def test_log_str_prop(self): """Test the log string property""" adr = ActivityDetailRecord(ActivityDetailType.CODE_EXECUTED, key="my_key", show=False, importance=233) assert adr.log_str == "3,0,233,my_key,0" adr = ActivityDetailRecord(ActivityDetailType.OUTPUT_DATA, key="my_key", show=True, importance=25) assert adr.log_str == "1,1,25,my_key,0" adr = ActivityDetailRecord(ActivityDetailType.OUTPUT_DATA, key="my_key", show=True, importance=25, action=ActivityAction.EDIT) assert adr.log_str == "1,1,25,my_key,2"
def test_to_json(self): """Test converting to json""" adr = ActivityDetailRecord(ActivityDetailType.ENVIRONMENT, key="my_key3", show=True, importance=225) adr.add_value("text/plain", "this is some data") adr_in_json = adr.to_json() assert type(adr_in_json) == str json_dict = json.loads(adr_in_json) assert json_dict['type'] == 0 assert json_dict['importance'] == 225 assert json_dict['show'] is True assert json_dict['data'] == {'text/plain': "this is some data"}
def test_log_str_prop(self): """Test the log string property""" ar = ActivityRecord(ActivityType.CODE, show=True, message="added some code", importance=50, linked_commit='aaaaaaaa') adr = ActivityDetailRecord(ActivityDetailType.CODE) adr.add_value("text/plain", "this is some data") adr.key = "my_fake_detail_key" ar.add_detail_object(adr) assert ar.log_str == """_GTM_ACTIVITY_START_**
def test_add_detail_object(self): """Test adding values to the detail object""" ar = ActivityRecord(ActivityType.CODE, show=True, message="added some code", importance=50, linked_commit='aaaaaaaa') adr = ActivityDetailRecord(ActivityDetailType.CODE) adr.add_value("text/plain", "this is some data") ar.add_detail_object(adr) assert len(ar._detail_objects) == 1 assert ar._detail_objects[0][0] is True assert ar._detail_objects[0][1] == ActivityDetailType.CODE.value assert ar._detail_objects[0][2] == 0 assert ar._detail_objects[0][3] == adr
def test_add_value(self): """Test adding values to the detail object""" adr = ActivityDetailRecord(ActivityDetailType.CODE_EXECUTED, key="my_key2") adr.add_value("text/plain", "this is some data") adr.add_value("text/html", "<p>this is some data<\p>") assert len(adr.data.keys()) == 2 assert adr.data["text/plain"] == "this is some data" assert adr.data["text/html"] == "<p>this is some data<\p>" with pytest.raises(ValueError): adr.add_value("text/html", "<p>this is some data<\p>")
def test_from_log_str(self): """Test the creating from a log string""" adr = ActivityDetailRecord.from_log_str("2,1,25,my_key,3") assert type(adr) == ActivityDetailRecord assert adr.type == ActivityDetailType.RESULT assert adr.key == "my_key" assert adr.show is True assert adr.importance == 25 assert adr.tags == [] assert adr.data == {} assert adr.action == ActivityAction.DELETE
def test_constructor(self): """Test the constructor""" adr = ActivityDetailRecord(ActivityDetailType.CODE) assert type(adr) == ActivityDetailRecord assert adr.type == ActivityDetailType.CODE assert adr.key is None assert adr.show is True assert adr.importance == 0 assert adr.tags == [] assert adr.data == {} adr = ActivityDetailRecord(ActivityDetailType.CODE, key="my_key", show=False, importance=23) assert type(adr) == ActivityDetailRecord assert adr.type == ActivityDetailType.CODE assert adr.key == "my_key" assert adr.show is False assert adr.importance == 23 assert adr.tags == [] assert adr.data == {}
def test_jsonify_data(self): """Test converting to json""" adr = ActivityDetailRecord(ActivityDetailType.ENVIRONMENT, key="my_key3fgjg", show=True, importance=45) adr.add_value("text/plain", "this is some data to jsonify") adr.add_value("text/markdown", "this is some data to `jsonify`") data = adr.jsonify_data() assert type(data) == dict assert len(data.keys()) == 2 assert data['text/plain'] == "this is some data to jsonify" assert data['text/markdown'] == "this is some data to `jsonify`"
def test_log_str_prop_errors(self): """Test the log string property""" ar = ActivityRecord(ActivityType.CODE, show=True, importance=50, linked_commit='aaaaaaaa') with pytest.raises(ValueError): # Missing message _ = ar.log_str ar = ActivityRecord(ActivityType.CODE, show=True, message="added some code", importance=50, linked_commit='aaaaaaaa') adr = ActivityDetailRecord(ActivityDetailType.CODE) adr.add_value("text/plain", "this is some data") ar.add_detail_object(adr) with pytest.raises(ValueError): # Missing detail key for detail record ar.log_str
def put_detail_record( self, detail_obj: ActivityDetailRecord) -> ActivityDetailRecord: """Method to write a detail record to the activity detail db Args: detail_obj(ActivityDetailRecord): The detail record to write Returns: ActivityDetailRecord: the detail record updated with the key """ # Set compression option based on config and objects size compress = False if self.compress_details: if detail_obj.data_size >= self.compress_min_bytes: compress = True # Write record and store key detail_obj.key = self.detaildb.put( self._encode_write_options(compress=compress) + detail_obj.to_bytes(compress)) logger.debug( f"Successfully wrote ActivityDetailRecord {detail_obj.key}") return detail_obj
def test_to_dict(self): """Test converting to a dictionary""" adr = ActivityDetailRecord(ActivityDetailType.CODE_EXECUTED, key="my_key2", show=True, importance=25) adr.add_value("text/plain", "this is some data") adr.add_value("text/html", "<p>this is some data<\p>") dict_obj = adr.to_dict() assert type(dict_obj) == dict assert dict_obj['type'] == 3 assert dict_obj['action'] == 0 assert dict_obj['importance'] == 25 assert dict_obj['show'] == 1 assert dict_obj['data'] == {"text/plain": "this is some data", "text/html": "<p>this is some data<\p>"}
def test_data_size(self): """Test getting the size of details stored""" adr = ActivityDetailRecord(ActivityDetailType.CODE_EXECUTED, key="my_key2") assert adr.data_size == 0 adr.add_value("text/plain", "0000000000") assert adr.data_size == 10 adr.add_value("text/html", "0000000000") assert adr.data_size == 20
def test_put_get_activity_record_with_tag(self, mock_config_with_activitystore): """Method to test creating and getting an individual activity record with a tag""" adr1 = ActivityDetailRecord(ActivityDetailType.CODE) adr1.show = True adr1.importance = 100 adr1.add_value("text/plain", "this is a thing" * 1000) adr1.tags = ['tag1', 'tag2'] linked_commit = helper_create_labbook_change(mock_config_with_activitystore[1], 1) ar = ActivityRecord(ActivityType.CODE, show=True, message="added some code", importance=50, linked_commit=linked_commit.hexsha) ar.add_detail_object(adr1) ar.tags = ['tag1', 'tag2'] assert ar.commit is None # Create Activity Record ar_written = mock_config_with_activitystore[0].create_activity_record(ar) assert ar.commit is not None # Get Note and check stored_ar = mock_config_with_activitystore[0].get_activity_record(ar_written.commit) assert ar.commit == stored_ar.commit assert ar.importance == stored_ar.importance assert ar.linked_commit == stored_ar.linked_commit assert ar.log_str == stored_ar.log_str assert ar.message == stored_ar.message assert ar.show == stored_ar.show assert ar.tags == stored_ar.tags assert ar.type == stored_ar.type assert len(ar._detail_objects) == len(stored_ar._detail_objects) assert stored_ar.username == 'default' assert stored_ar.email == '*****@*****.**' assert stored_ar.username == ar.username assert stored_ar.email == ar.email assert ar._detail_objects[0][0] == stored_ar._detail_objects[0][0] assert ar._detail_objects[0][1] == stored_ar._detail_objects[0][1] assert ar._detail_objects[0][2] == stored_ar._detail_objects[0][2] assert ar._detail_objects[0][3].is_loaded is True assert stored_ar._detail_objects[0][3].is_loaded is False
def get_detail_record(self, detail_key: str) -> ActivityDetailRecord: """Method to fetch a detail entry from the activity detail db Args: detail_key : the key returned from the activity detail DB when storing. Returns: ActivityDetailRecord """ # Get value from key-value store detail_bytes = self.detaildb.get(detail_key) # Remove header options = self._decode_write_options(detail_bytes[:1]) # Create object record = ActivityDetailRecord.from_bytes( detail_bytes[1:], decompress=options['compress']) record.key = detail_key return record
def test_to_bytes_from_bytes(self): """Test converting to a byte array""" adr = ActivityDetailRecord(ActivityDetailType.CODE_EXECUTED, key="my_key3", show=True, importance=225, action=ActivityAction.CREATE) adr.add_value("text/plain", "this is some data") byte_array_no_compression = adr.to_bytes(compress=False) assert type(byte_array_no_compression) == bytes adr2 = ActivityDetailRecord.from_bytes(byte_array_no_compression, decompress=False) assert type(adr2) == ActivityDetailRecord assert adr2.type == ActivityDetailType.CODE_EXECUTED assert adr2.action == ActivityAction.CREATE assert adr2.key is None assert adr2.show is True assert adr2.importance == 225 assert adr2.tags == [] assert adr2.data == {"text/plain": "this is some data"}
def test_add_detail_objects_sort(self): """Test adding values to the detail object""" ar = ActivityRecord(ActivityType.CODE, show=True, message="added some code", importance=50, linked_commit='aaaaaaaa') adr = ActivityDetailRecord(ActivityDetailType.CODE) adr.show = True adr.importance = 100 adr.add_value("text/plain", "second") ar.add_detail_object(adr) adr = ActivityDetailRecord(ActivityDetailType.CODE) adr.show = True adr.importance = 200 adr.add_value("text/plain", "first") ar.add_detail_object(adr) adr = ActivityDetailRecord(ActivityDetailType.CODE) adr.show = False adr.importance = 0 adr.add_value("text/plain", "sixth") ar.add_detail_object(adr) adr = ActivityDetailRecord(ActivityDetailType.OUTPUT_DATA) adr.show = True adr.importance = 201 adr.add_value("text/plain", "forth") ar.add_detail_object(adr) adr = ActivityDetailRecord(ActivityDetailType.RESULT) adr.show = True adr.importance = 201 adr.add_value("text/plain", "third") ar.add_detail_object(adr) adr = ActivityDetailRecord(ActivityDetailType.INPUT_DATA) adr.show = False adr.importance = 201 adr.add_value("text/plain", "fifth") ar.add_detail_object(adr) assert len(ar._detail_objects) == 6 assert ar._detail_objects[0][3].data['text/plain'] == 'first' assert ar._detail_objects[1][3].data['text/plain'] == 'second' assert ar._detail_objects[2][3].data['text/plain'] == 'third' assert ar._detail_objects[3][3].data['text/plain'] == 'forth' assert ar._detail_objects[4][3].data['text/plain'] == 'fifth' assert ar._detail_objects[5][3].data['text/plain'] == 'sixth'
def test_trim_detail_objects(self): """""" ar = ActivityRecord(ActivityType.CODE, show=True, message="added some code", importance=50, linked_commit='aaaaaaaa') adr1 = ActivityDetailRecord(ActivityDetailType.CODE) adr1.show = True adr1.importance = 100 adr1.add_value("text/plain", "first") ar.add_detail_object(adr1) adr2 = ActivityDetailRecord(ActivityDetailType.CODE) adr2.show = True adr2.importance = 0 adr2.add_value("text/plain", "second") ar.add_detail_object(adr2) adr3 = ActivityDetailRecord(ActivityDetailType.CODE) adr3.show = True adr3.importance = 0 adr3.add_value("text/plain", "third") ar.add_detail_object(adr3) assert len(ar._detail_objects) == 3 assert ar._detail_objects[0][3].data['text/plain'] == 'first' assert ar._detail_objects[1][3].data['text/plain'] == 'second' assert ar._detail_objects[2][3].data['text/plain'] == 'third' with ar.inspect_detail_objects(): ar.trim_detail_objects(2) assert len(ar._detail_objects) == 2 assert ar._detail_objects[0][3].data['text/plain'] == 'first' assert ar._detail_objects[1][3].data['text/plain'] == 'second' with pytest.raises(ValueError): with ar.inspect_detail_objects(): ar.trim_detail_objects(0)
def test_update_detail_object(self): """Test converting to a dictionary""" ar = ActivityRecord(ActivityType.CODE, show=True, message="added some code", importance=50, linked_commit='aaaaaaaa') adr1 = ActivityDetailRecord(ActivityDetailType.CODE) adr1.show = True adr1.importance = 100 adr1.add_value("text/plain", "first") ar.add_detail_object(adr1) adr2 = ActivityDetailRecord(ActivityDetailType.CODE) adr2.show = True adr2.importance = 0 adr2.add_value("text/plain", "second") ar.add_detail_object(adr2) assert len(ar._detail_objects) == 2 assert ar._detail_objects[0][3].data['text/plain'] == 'first' assert ar._detail_objects[1][3].data['text/plain'] == 'second' adr2.importance = 200 with ar.inspect_detail_objects(): ar.update_detail_object(adr2, 1) assert len(ar._detail_objects) == 2 assert ar._detail_objects[0][3].data['text/plain'] == 'second' assert ar._detail_objects[1][3].data['text/plain'] == 'first'
def test_compression(self): """Test compression on large objects""" adr = ActivityDetailRecord(ActivityDetailType.INPUT_DATA, key="my_ke3", show=True, importance=125) adr.add_value("text/plain", "this is some data00000000000000000000000000000000000" * 1000) byte_array_no_compression = adr.to_bytes(compress=False) assert type(byte_array_no_compression) == bytes adr2 = ActivityDetailRecord(ActivityDetailType.INPUT_DATA, key="my_ke3", show=True, importance=125) adr2.add_value("text/plain", "this is some data00000000000000000000000000000000000" * 1000) byte_array_compression = adr2.to_bytes(compress=True) assert type(byte_array_compression) == bytes assert len(byte_array_compression) < len(byte_array_no_compression) adr3 = ActivityDetailRecord.from_bytes(byte_array_compression, decompress=True) assert type(adr3) == ActivityDetailRecord assert adr3.type == ActivityDetailType.INPUT_DATA assert adr3.action == ActivityAction.NOACTION assert adr3.key is None assert adr3.show is True assert adr3.importance == 125 assert adr3.tags == [] assert adr3.data == adr.data
def test_put_get_detail_record_with_tags(self, mock_config_with_activitystore): """Test to test storing and retrieving data from the activity detail db""" # Create test values adr1 = ActivityDetailRecord(ActivityDetailType.CODE, action=ActivityAction.CREATE) adr1.show = True adr1.importance = 100 adr1.add_value("text/plain", "first") adr1.tags = ['test1'] adr2 = ActivityDetailRecord(ActivityDetailType.CODE_EXECUTED, action=ActivityAction.EXECUTE) adr2.show = True adr2.importance = 0 adr2.add_value("text/plain", "second") adr2.tags = ['test2', 'test:3'] adr1 = mock_config_with_activitystore[0].put_detail_record(adr1) adr2 = mock_config_with_activitystore[0].put_detail_record(adr2) assert adr1.key is not None assert adr2.key is not None assert type(adr1.key) == str assert type(adr2.key) == str # Load adr1_loaded = mock_config_with_activitystore[0].get_detail_record(adr1.key) adr2_loaded = mock_config_with_activitystore[0].get_detail_record(adr2.key) assert adr1.key == adr1_loaded.key assert adr1.importance == adr1_loaded.importance assert adr1.type == adr1_loaded.type assert adr1.is_loaded == adr1_loaded.is_loaded is True assert adr1.data == adr1_loaded.data assert adr1.tags == adr1_loaded.tags assert adr1.action == adr1_loaded.action == ActivityAction.CREATE assert adr2.key == adr2_loaded.key assert adr2.importance == adr2_loaded.importance assert adr2.type == adr2_loaded.type assert adr2.is_loaded == adr2_loaded.is_loaded is True assert adr2.data == adr2_loaded.data assert adr2.tags == adr2_loaded.tags assert adr2.action == adr2_loaded.action == ActivityAction.EXECUTE
def helper_create_activitydetailobject(): """Helper to create a random ActivityDetailRecord""" adr = ActivityDetailRecord(ActivityDetailType(random.randint(0, 6)), key=f"my_key_{random.randint(0, 99999)}") adr.add_value("text/plain", ''.join(random.choice('abcdefghijklmnopqrstuvwxyz ') for _ in range(50)))
def test_put_get_detail_record_with_compression(self, mock_config_with_activitystore): """Test to test storing and retrieving data from the activity detail db w/ compression""" # Create test values adr1 = ActivityDetailRecord(ActivityDetailType.CODE) adr1.show = True adr1.importance = 100 adr1.add_value("text/plain", "first" * 1000) adr2 = ActivityDetailRecord(ActivityDetailType.CODE) adr2.show = True adr2.importance = 0 adr2.add_value("text/plain", "second" * 1000) adr1 = mock_config_with_activitystore[0].put_detail_record(adr1) adr2 = mock_config_with_activitystore[0].put_detail_record(adr2) assert adr1.key is not None assert adr2.key is not None assert type(adr1.key) == str assert type(adr2.key) == str # Load adr1_loaded = mock_config_with_activitystore[0].get_detail_record(adr1.key) adr2_loaded = mock_config_with_activitystore[0].get_detail_record(adr2.key) assert adr1.key == adr1_loaded.key assert adr1.importance == adr1_loaded.importance assert adr1.type == adr1_loaded.type assert adr1.is_loaded == adr1_loaded.is_loaded is True assert adr1.data == adr1_loaded.data assert adr2.key == adr2_loaded.key assert adr2.importance == adr2_loaded.importance assert adr2.type == adr2_loaded.type assert adr2.is_loaded == adr2_loaded.is_loaded is True assert adr2.data == adr2_loaded.data