def test_log_str_prop(self):
"""Test the log string property"""
adr = ActivityDetailRecord(ActivityDetailType.CODE_EXECUTED, key="my_key", show=False, importance=233)
assert adr.log_str == "3,0,233,my_key,0"
adr = ActivityDetailRecord(ActivityDetailType.OUTPUT_DATA, key="my_key", show=True, importance=25)
assert adr.log_str == "1,1,25,my_key,0"
adr = ActivityDetailRecord(ActivityDetailType.OUTPUT_DATA, key="my_key", show=True, importance=25,
action=ActivityAction.EDIT)
assert adr.log_str == "1,1,25,my_key,2"
def test_to_json(self):
"""Test converting to json"""
adr = ActivityDetailRecord(ActivityDetailType.ENVIRONMENT, key="my_key3", show=True, importance=225)
adr.add_value("text/plain", "this is some data")
adr_in_json = adr.to_json()
assert type(adr_in_json) == str
json_dict = json.loads(adr_in_json)
assert json_dict['type'] == 0
assert json_dict['importance'] == 225
assert json_dict['show'] is True
assert json_dict['data'] == {'text/plain': "this is some data"}
def test_log_str_prop(self):
"""Test the log string property"""
ar = ActivityRecord(ActivityType.CODE,
show=True,
message="added some code",
importance=50,
linked_commit='aaaaaaaa')
adr = ActivityDetailRecord(ActivityDetailType.CODE)
adr.add_value("text/plain", "this is some data")
adr.key = "my_fake_detail_key"
ar.add_detail_object(adr)
assert ar.log_str == """_GTM_ACTIVITY_START_**
def test_add_detail_object(self):
"""Test adding values to the detail object"""
ar = ActivityRecord(ActivityType.CODE,
show=True,
message="added some code",
importance=50,
linked_commit='aaaaaaaa')
adr = ActivityDetailRecord(ActivityDetailType.CODE)
adr.add_value("text/plain", "this is some data")
ar.add_detail_object(adr)
assert len(ar._detail_objects) == 1
assert ar._detail_objects[0][0] is True
assert ar._detail_objects[0][1] == ActivityDetailType.CODE.value
assert ar._detail_objects[0][2] == 0
assert ar._detail_objects[0][3] == adr
def test_add_value(self):
"""Test adding values to the detail object"""
adr = ActivityDetailRecord(ActivityDetailType.CODE_EXECUTED, key="my_key2")
adr.add_value("text/plain", "this is some data")
adr.add_value("text/html", "<p>this is some data<\p>")
assert len(adr.data.keys()) == 2
assert adr.data["text/plain"] == "this is some data"
assert adr.data["text/html"] == "<p>this is some data<\p>"
with pytest.raises(ValueError):
adr.add_value("text/html", "<p>this is some data<\p>")
def test_from_log_str(self):
"""Test the creating from a log string"""
adr = ActivityDetailRecord.from_log_str("2,1,25,my_key,3")
assert type(adr) == ActivityDetailRecord
assert adr.type == ActivityDetailType.RESULT
assert adr.key == "my_key"
assert adr.show is True
assert adr.importance == 25
assert adr.tags == []
assert adr.data == {}
assert adr.action == ActivityAction.DELETE
def test_constructor(self):
"""Test the constructor"""
adr = ActivityDetailRecord(ActivityDetailType.CODE)
assert type(adr) == ActivityDetailRecord
assert adr.type == ActivityDetailType.CODE
assert adr.key is None
assert adr.show is True
assert adr.importance == 0
assert adr.tags == []
assert adr.data == {}
adr = ActivityDetailRecord(ActivityDetailType.CODE, key="my_key", show=False, importance=23)
assert type(adr) == ActivityDetailRecord
assert adr.type == ActivityDetailType.CODE
assert adr.key == "my_key"
assert adr.show is False
assert adr.importance == 23
assert adr.tags == []
assert adr.data == {}
def test_jsonify_data(self):
"""Test converting to json"""
adr = ActivityDetailRecord(ActivityDetailType.ENVIRONMENT, key="my_key3fgjg", show=True, importance=45)
adr.add_value("text/plain", "this is some data to jsonify")
adr.add_value("text/markdown", "this is some data to `jsonify`")
data = adr.jsonify_data()
assert type(data) == dict
assert len(data.keys()) == 2
assert data['text/plain'] == "this is some data to jsonify"
assert data['text/markdown'] == "this is some data to `jsonify`"
def test_log_str_prop_errors(self):
"""Test the log string property"""
ar = ActivityRecord(ActivityType.CODE,
show=True,
importance=50,
linked_commit='aaaaaaaa')
with pytest.raises(ValueError):
# Missing message
_ = ar.log_str
ar = ActivityRecord(ActivityType.CODE,
show=True,
message="added some code",
importance=50,
linked_commit='aaaaaaaa')
adr = ActivityDetailRecord(ActivityDetailType.CODE)
adr.add_value("text/plain", "this is some data")
ar.add_detail_object(adr)
with pytest.raises(ValueError):
# Missing detail key for detail record
ar.log_str
def put_detail_record(
self, detail_obj: ActivityDetailRecord) -> ActivityDetailRecord:
"""Method to write a detail record to the activity detail db
Args:
detail_obj(ActivityDetailRecord): The detail record to write
Returns:
ActivityDetailRecord: the detail record updated with the key
"""
# Set compression option based on config and objects size
compress = False
if self.compress_details:
if detail_obj.data_size >= self.compress_min_bytes:
compress = True
# Write record and store key
detail_obj.key = self.detaildb.put(
self._encode_write_options(compress=compress) +
detail_obj.to_bytes(compress))
logger.debug(
f"Successfully wrote ActivityDetailRecord {detail_obj.key}")
return detail_obj
def test_to_dict(self):
"""Test converting to a dictionary"""
adr = ActivityDetailRecord(ActivityDetailType.CODE_EXECUTED, key="my_key2", show=True, importance=25)
adr.add_value("text/plain", "this is some data")
adr.add_value("text/html", "<p>this is some data<\p>")
dict_obj = adr.to_dict()
assert type(dict_obj) == dict
assert dict_obj['type'] == 3
assert dict_obj['action'] == 0
assert dict_obj['importance'] == 25
assert dict_obj['show'] == 1
assert dict_obj['data'] == {"text/plain": "this is some data",
"text/html": "<p>this is some data<\p>"}
def test_data_size(self):
"""Test getting the size of details stored"""
adr = ActivityDetailRecord(ActivityDetailType.CODE_EXECUTED, key="my_key2")
assert adr.data_size == 0
adr.add_value("text/plain", "0000000000")
assert adr.data_size == 10
adr.add_value("text/html", "0000000000")
assert adr.data_size == 20
def test_put_get_activity_record_with_tag(self, mock_config_with_activitystore):
"""Method to test creating and getting an individual activity record with a tag"""
adr1 = ActivityDetailRecord(ActivityDetailType.CODE)
adr1.show = True
adr1.importance = 100
adr1.add_value("text/plain", "this is a thing" * 1000)
adr1.tags = ['tag1', 'tag2']
linked_commit = helper_create_labbook_change(mock_config_with_activitystore[1], 1)
ar = ActivityRecord(ActivityType.CODE,
show=True,
message="added some code",
importance=50,
linked_commit=linked_commit.hexsha)
ar.add_detail_object(adr1)
ar.tags = ['tag1', 'tag2']
assert ar.commit is None
# Create Activity Record
ar_written = mock_config_with_activitystore[0].create_activity_record(ar)
assert ar.commit is not None
# Get Note and check
stored_ar = mock_config_with_activitystore[0].get_activity_record(ar_written.commit)
assert ar.commit == stored_ar.commit
assert ar.importance == stored_ar.importance
assert ar.linked_commit == stored_ar.linked_commit
assert ar.log_str == stored_ar.log_str
assert ar.message == stored_ar.message
assert ar.show == stored_ar.show
assert ar.tags == stored_ar.tags
assert ar.type == stored_ar.type
assert len(ar._detail_objects) == len(stored_ar._detail_objects)
assert stored_ar.username == 'default'
assert stored_ar.email == '*****@*****.**'
assert stored_ar.username == ar.username
assert stored_ar.email == ar.email
assert ar._detail_objects[0][0] == stored_ar._detail_objects[0][0]
assert ar._detail_objects[0][1] == stored_ar._detail_objects[0][1]
assert ar._detail_objects[0][2] == stored_ar._detail_objects[0][2]
assert ar._detail_objects[0][3].is_loaded is True
assert stored_ar._detail_objects[0][3].is_loaded is False
def get_detail_record(self, detail_key: str) -> ActivityDetailRecord:
"""Method to fetch a detail entry from the activity detail db
Args:
detail_key : the key returned from the activity detail DB when storing.
Returns:
ActivityDetailRecord
"""
# Get value from key-value store
detail_bytes = self.detaildb.get(detail_key)
# Remove header
options = self._decode_write_options(detail_bytes[:1])
# Create object
record = ActivityDetailRecord.from_bytes(
detail_bytes[1:], decompress=options['compress'])
record.key = detail_key
return record
def test_to_bytes_from_bytes(self):
"""Test converting to a byte array"""
adr = ActivityDetailRecord(ActivityDetailType.CODE_EXECUTED, key="my_key3", show=True, importance=225,
action=ActivityAction.CREATE)
adr.add_value("text/plain", "this is some data")
byte_array_no_compression = adr.to_bytes(compress=False)
assert type(byte_array_no_compression) == bytes
adr2 = ActivityDetailRecord.from_bytes(byte_array_no_compression, decompress=False)
assert type(adr2) == ActivityDetailRecord
assert adr2.type == ActivityDetailType.CODE_EXECUTED
assert adr2.action == ActivityAction.CREATE
assert adr2.key is None
assert adr2.show is True
assert adr2.importance == 225
assert adr2.tags == []
assert adr2.data == {"text/plain": "this is some data"}
def test_add_detail_objects_sort(self):
"""Test adding values to the detail object"""
ar = ActivityRecord(ActivityType.CODE,
show=True,
message="added some code",
importance=50,
linked_commit='aaaaaaaa')
adr = ActivityDetailRecord(ActivityDetailType.CODE)
adr.show = True
adr.importance = 100
adr.add_value("text/plain", "second")
ar.add_detail_object(adr)
adr = ActivityDetailRecord(ActivityDetailType.CODE)
adr.show = True
adr.importance = 200
adr.add_value("text/plain", "first")
ar.add_detail_object(adr)
adr = ActivityDetailRecord(ActivityDetailType.CODE)
adr.show = False
adr.importance = 0
adr.add_value("text/plain", "sixth")
ar.add_detail_object(adr)
adr = ActivityDetailRecord(ActivityDetailType.OUTPUT_DATA)
adr.show = True
adr.importance = 201
adr.add_value("text/plain", "forth")
ar.add_detail_object(adr)
adr = ActivityDetailRecord(ActivityDetailType.RESULT)
adr.show = True
adr.importance = 201
adr.add_value("text/plain", "third")
ar.add_detail_object(adr)
adr = ActivityDetailRecord(ActivityDetailType.INPUT_DATA)
adr.show = False
adr.importance = 201
adr.add_value("text/plain", "fifth")
ar.add_detail_object(adr)
assert len(ar._detail_objects) == 6
assert ar._detail_objects[0][3].data['text/plain'] == 'first'
assert ar._detail_objects[1][3].data['text/plain'] == 'second'
assert ar._detail_objects[2][3].data['text/plain'] == 'third'
assert ar._detail_objects[3][3].data['text/plain'] == 'forth'
assert ar._detail_objects[4][3].data['text/plain'] == 'fifth'
assert ar._detail_objects[5][3].data['text/plain'] == 'sixth'
def test_trim_detail_objects(self):
""""""
ar = ActivityRecord(ActivityType.CODE,
show=True,
message="added some code",
importance=50,
linked_commit='aaaaaaaa')
adr1 = ActivityDetailRecord(ActivityDetailType.CODE)
adr1.show = True
adr1.importance = 100
adr1.add_value("text/plain", "first")
ar.add_detail_object(adr1)
adr2 = ActivityDetailRecord(ActivityDetailType.CODE)
adr2.show = True
adr2.importance = 0
adr2.add_value("text/plain", "second")
ar.add_detail_object(adr2)
adr3 = ActivityDetailRecord(ActivityDetailType.CODE)
adr3.show = True
adr3.importance = 0
adr3.add_value("text/plain", "third")
ar.add_detail_object(adr3)
assert len(ar._detail_objects) == 3
assert ar._detail_objects[0][3].data['text/plain'] == 'first'
assert ar._detail_objects[1][3].data['text/plain'] == 'second'
assert ar._detail_objects[2][3].data['text/plain'] == 'third'
with ar.inspect_detail_objects():
ar.trim_detail_objects(2)
assert len(ar._detail_objects) == 2
assert ar._detail_objects[0][3].data['text/plain'] == 'first'
assert ar._detail_objects[1][3].data['text/plain'] == 'second'
with pytest.raises(ValueError):
with ar.inspect_detail_objects():
ar.trim_detail_objects(0)
def test_update_detail_object(self):
"""Test converting to a dictionary"""
ar = ActivityRecord(ActivityType.CODE,
show=True,
message="added some code",
importance=50,
linked_commit='aaaaaaaa')
adr1 = ActivityDetailRecord(ActivityDetailType.CODE)
adr1.show = True
adr1.importance = 100
adr1.add_value("text/plain", "first")
ar.add_detail_object(adr1)
adr2 = ActivityDetailRecord(ActivityDetailType.CODE)
adr2.show = True
adr2.importance = 0
adr2.add_value("text/plain", "second")
ar.add_detail_object(adr2)
assert len(ar._detail_objects) == 2
assert ar._detail_objects[0][3].data['text/plain'] == 'first'
assert ar._detail_objects[1][3].data['text/plain'] == 'second'
adr2.importance = 200
with ar.inspect_detail_objects():
ar.update_detail_object(adr2, 1)
assert len(ar._detail_objects) == 2
assert ar._detail_objects[0][3].data['text/plain'] == 'second'
assert ar._detail_objects[1][3].data['text/plain'] == 'first'
def test_compression(self):
"""Test compression on large objects"""
adr = ActivityDetailRecord(ActivityDetailType.INPUT_DATA, key="my_ke3", show=True, importance=125)
adr.add_value("text/plain", "this is some data00000000000000000000000000000000000" * 1000)
byte_array_no_compression = adr.to_bytes(compress=False)
assert type(byte_array_no_compression) == bytes
adr2 = ActivityDetailRecord(ActivityDetailType.INPUT_DATA, key="my_ke3", show=True, importance=125)
adr2.add_value("text/plain", "this is some data00000000000000000000000000000000000" * 1000)
byte_array_compression = adr2.to_bytes(compress=True)
assert type(byte_array_compression) == bytes
assert len(byte_array_compression) < len(byte_array_no_compression)
adr3 = ActivityDetailRecord.from_bytes(byte_array_compression, decompress=True)
assert type(adr3) == ActivityDetailRecord
assert adr3.type == ActivityDetailType.INPUT_DATA
assert adr3.action == ActivityAction.NOACTION
assert adr3.key is None
assert adr3.show is True
assert adr3.importance == 125
assert adr3.tags == []
assert adr3.data == adr.data
def test_put_get_detail_record_with_tags(self, mock_config_with_activitystore):
"""Test to test storing and retrieving data from the activity detail db"""
# Create test values
adr1 = ActivityDetailRecord(ActivityDetailType.CODE, action=ActivityAction.CREATE)
adr1.show = True
adr1.importance = 100
adr1.add_value("text/plain", "first")
adr1.tags = ['test1']
adr2 = ActivityDetailRecord(ActivityDetailType.CODE_EXECUTED, action=ActivityAction.EXECUTE)
adr2.show = True
adr2.importance = 0
adr2.add_value("text/plain", "second")
adr2.tags = ['test2', 'test:3']
adr1 = mock_config_with_activitystore[0].put_detail_record(adr1)
adr2 = mock_config_with_activitystore[0].put_detail_record(adr2)
assert adr1.key is not None
assert adr2.key is not None
assert type(adr1.key) == str
assert type(adr2.key) == str
# Load
adr1_loaded = mock_config_with_activitystore[0].get_detail_record(adr1.key)
adr2_loaded = mock_config_with_activitystore[0].get_detail_record(adr2.key)
assert adr1.key == adr1_loaded.key
assert adr1.importance == adr1_loaded.importance
assert adr1.type == adr1_loaded.type
assert adr1.is_loaded == adr1_loaded.is_loaded is True
assert adr1.data == adr1_loaded.data
assert adr1.tags == adr1_loaded.tags
assert adr1.action == adr1_loaded.action == ActivityAction.CREATE
assert adr2.key == adr2_loaded.key
assert adr2.importance == adr2_loaded.importance
assert adr2.type == adr2_loaded.type
assert adr2.is_loaded == adr2_loaded.is_loaded is True
assert adr2.data == adr2_loaded.data
assert adr2.tags == adr2_loaded.tags
assert adr2.action == adr2_loaded.action == ActivityAction.EXECUTE
def helper_create_activitydetailobject():
"""Helper to create a random ActivityDetailRecord"""
adr = ActivityDetailRecord(ActivityDetailType(random.randint(0, 6)), key=f"my_key_{random.randint(0, 99999)}")
adr.add_value("text/plain", ''.join(random.choice('abcdefghijklmnopqrstuvwxyz ') for _ in range(50)))
def test_put_get_detail_record_with_compression(self, mock_config_with_activitystore):
"""Test to test storing and retrieving data from the activity detail db w/ compression"""
# Create test values
adr1 = ActivityDetailRecord(ActivityDetailType.CODE)
adr1.show = True
adr1.importance = 100
adr1.add_value("text/plain", "first" * 1000)
adr2 = ActivityDetailRecord(ActivityDetailType.CODE)
adr2.show = True
adr2.importance = 0
adr2.add_value("text/plain", "second" * 1000)
adr1 = mock_config_with_activitystore[0].put_detail_record(adr1)
adr2 = mock_config_with_activitystore[0].put_detail_record(adr2)
assert adr1.key is not None
assert adr2.key is not None
assert type(adr1.key) == str
assert type(adr2.key) == str
# Load
adr1_loaded = mock_config_with_activitystore[0].get_detail_record(adr1.key)
adr2_loaded = mock_config_with_activitystore[0].get_detail_record(adr2.key)
assert adr1.key == adr1_loaded.key
assert adr1.importance == adr1_loaded.importance
assert adr1.type == adr1_loaded.type
assert adr1.is_loaded == adr1_loaded.is_loaded is True
assert adr1.data == adr1_loaded.data
assert adr2.key == adr2_loaded.key
assert adr2.importance == adr2_loaded.importance
assert adr2.type == adr2_loaded.type
assert adr2.is_loaded == adr2_loaded.is_loaded is True
assert adr2.data == adr2_loaded.data