def test_sample_build():
sn = SampleNode('foo')
sn2 = SampleNode('bat')
sn3 = SampleNode('bar', type_=SubSampleType.TECHNICAL_REPLICATE, parent='foo')
sn4 = SampleNode('baz', type_=SubSampleType.SUB_SAMPLE, parent='foo')
sndup = SampleNode('foo')
s = Sample([sn])
assert s.nodes == (sndup,)
assert s.name is None
s = Sample([sn, sn2, sn4, sn3], ' \t foo ')
assert s.nodes == (sndup, sn2, sn4, sn3)
assert s.name == 'foo'
s = Sample([sn], 'a' * 256)
assert s.nodes == (sndup,)
assert s.name == 'a' * 256
id_ = uuid.UUID('1234567890abcdef1234567890abcdef')
s = SavedSample(id_, UserID('user'), [sn], dt(6))
assert s.id == uuid.UUID('1234567890abcdef1234567890abcdef')
assert s.user == UserID('user')
assert s.nodes == (sndup,)
assert s.savetime == dt(6)
assert s.name is None
assert s.version is None
s = SavedSample(id_, UserID('user2'), [sn], dt(6), 'foo')
assert s.id == uuid.UUID('1234567890abcdef1234567890abcdef')
assert s.user == UserID('user2')
assert s.nodes == (sndup,)
assert s.savetime == dt(6)
assert s.name == 'foo'
assert s.version is None
s = SavedSample(id_, UserID('user'), [sn], dt(6), 'foo', 1)
assert s.id == uuid.UUID('1234567890abcdef1234567890abcdef')
assert s.user == UserID('user')
assert s.nodes == (sndup,)
assert s.savetime == dt(6)
assert s.name == 'foo'
assert s.version == 1
s = SavedSample(id_, UserID('user'), [sn], dt(6), 'foo', 8)
assert s.id == uuid.UUID('1234567890abcdef1234567890abcdef')
assert s.user == UserID('user')
assert s.nodes == (sndup,)
assert s.savetime == dt(6)
assert s.name == 'foo'
assert s.version == 8
s = SavedSample(id_, UserID('user'), [sn], dt(6), version=8)
assert s.id == uuid.UUID('1234567890abcdef1234567890abcdef')
assert s.user == UserID('user')
assert s.nodes == (sndup,)
assert s.savetime == dt(6)
assert s.name is None
assert s.version == 8
def save_sample(self,
sample: Sample,
user: UserID,
id_: UUID = None,
prior_version: Optional[int] = None,
as_admin: bool = False) -> Tuple[UUID, int]:
'''
Save a sample.
:param sample: the sample to save.
:param user: the username of the user saving the sample.
:param id_: if the sample is a new version of a sample, the ID of the sample which will
get a new version.
:param prior_version: if id_ is included, specifying prior_version will ensure that the new
sample is saved with version prior_version + 1 or not at all.
:param as_admin: skip ACL checks for new versions.
:returns a tuple of the sample ID and version.
:raises IllegalParameterError: if the prior version is < 1
:raises UnauthorizedError: if the user does not have write permission to the sample when
saving a new version.
:raises NoSuchSampleError: if the sample does not exist when saving a new version.
:raises SampleStorageError: if the sample could not be retrieved when saving a new version
or if the sample fails to save.
:raises ConcurrencyError: if the sample's version is not equal to prior_version.
'''
_not_falsy(sample, 'sample')
_not_falsy(user, 'user')
self._validate_metadata(sample)
if id_:
if prior_version is not None and prior_version < 1:
raise _IllegalParameterError('Prior version must be > 0')
self._check_perms(id_,
user,
_SampleAccessType.WRITE,
as_admin=as_admin)
swid = SavedSample(id_, user, list(sample.nodes), self._now(),
sample.name)
ver = self._storage.save_sample_version(swid, prior_version)
else:
id_ = self._uuid_gen()
swid = SavedSample(id_, user, list(sample.nodes), self._now(),
sample.name)
# don't bother checking output since we created uuid
self._storage.save_sample(swid)
ver = 1
if self._kafka:
self._kafka.notify_new_sample_version(id_, ver)
return (id_, ver)
def _sample_build_fail(nodes, name, expected):
with raises(Exception) as got:
Sample(nodes, name)
assert_exception_correct(got.value, expected)
id_ = uuid.UUID('1234567890abcdef1234567890abcdef')
with raises(Exception) as got:
SavedSample(id_, UserID('u'), nodes, dt(8), name)
assert_exception_correct(got.value, expected)
def test_sample_build_fail_sample_count():
nodes = [SampleNode('s' + str(i)) for i in range(10000)]
s = Sample(nodes)
assert s.nodes == tuple(nodes)
assert s.name is None
id_ = uuid.UUID('1234567890abcdef1234567890abcdef')
s = SavedSample(id_, UserID('u'), nodes, dt(8))
assert s.id == uuid.UUID('1234567890abcdef1234567890abcdef')
assert s.user == UserID('u')
assert s.nodes == tuple(nodes)
assert s.savetime == dt(8)
assert s.name is None
assert s.version is None
nodes.append(SampleNode('s10000'))
_sample_build_fail(nodes, None, IllegalParameterError(
'At most 10000 nodes are allowed per sample'))
def test_sample_to_dict_maximal():
expected = {'node_tree': [{'id': 'foo',
'type': 'BioReplicate',
'meta_controlled': {},
'meta_user': {},
'parent': None
},
{'id': 'bar',
'type': 'TechReplicate',
'meta_controlled': {'a': {'b': 'c', 'm': 6.7}},
'meta_user': {'d': {'e': True}, 'g': {'h': 1}},
'parent': 'foo'
}],
'id': 'f5bd78c3-823e-40b2-9f93-20e78680e41e',
'user': '******',
'save_date': 87897,
'name': 'myname',
'version': 23,
}
id_ = UUID('f5bd78c3-823e-40b2-9f93-20e78680e41e')
s = sample_to_dict(
SavedSample(
id_,
UserID('user3'),
[SampleNode('foo'),
SampleNode(
'bar',
SubSampleType.TECHNICAL_REPLICATE,
'foo',
{'a': {'b': 'c', 'm': 6.7}},
{'d': {'e': True}, 'g': {'h': 1}})
],
dt(87.8971),
'myname',
23))
assert s == expected
# ensure that the result is jsonable. The data structure includes frozen maps which are not
json.dumps(s)
def test_sample_to_dict_minimal():
expected = {'node_tree': [{'id': 'foo',
'type': 'BioReplicate',
'meta_controlled': {},
'meta_user': {},
'parent': None
}],
'id': 'f5bd78c3-823e-40b2-9f93-20e78680e41e',
'user': '******',
'save_date': 87897,
'name': None,
'version': None,
}
id_ = UUID('f5bd78c3-823e-40b2-9f93-20e78680e41e')
s = sample_to_dict(SavedSample(id_, UserID('user2'), [SampleNode('foo')], dt(87.8971)))
assert s == expected
# ensure that the result is jsonable. The data structure includes frozen maps which are not
json.dumps(s)
def test_sample_hash():
# hashes will change from instance to instance of the python interpreter, and therefore
# tests can't be written that directly test the hash value. See
# https://docs.python.org/3/reference/datamodel.html#object.__hash__
sn = SampleNode('foo')
sn2 = SampleNode('bar')
id1 = uuid.UUID('1234567890abcdef1234567890abcdef')
id2 = uuid.UUID('1234567890abcdef1234567890abcdea')
dt1 = dt(5)
dt2 = dt(8)
u = UserID('u')
u2 = UserID('u2')
assert hash(Sample([sn], 'yay')) == hash(Sample([sn], 'yay'))
assert hash(Sample([sn], 'foo')) == hash(Sample([sn], 'foo'))
assert hash(Sample([sn], 'yay')) != hash(Sample([sn2], 'yay'))
assert hash(Sample([sn], 'yay')) != hash(Sample([sn], 'yo'))
assert hash(SavedSample(id1, u, [sn], dt1, 'yay')) == hash(SavedSample(
id1, u, [sn], dt(5), 'yay'))
assert hash(SavedSample(id2, u, [sn], dt1, 'foo')) == hash(SavedSample(
id2, u, [sn], dt1, 'foo'))
assert hash(SavedSample(id1, u, [sn], dt1, 'foo')) != hash(SavedSample(
id2, u, [sn], dt1, 'foo'))
assert hash(SavedSample(id1, u, [sn], dt1, 'foo')) != hash(SavedSample(
id1, u2, [sn], dt1, 'foo'))
assert hash(SavedSample(id2, u, [sn], dt1, 'foo')) != hash(SavedSample(
id2, u, [sn2], dt1, 'foo'))
assert hash(SavedSample(id2, u, [sn], dt1, 'foo')) != hash(SavedSample(
id2, u, [sn], dt2, 'foo'))
assert hash(SavedSample(id2, u, [sn], dt1, 'foo')) != hash(SavedSample(
id2, u, [sn], dt1, 'bar'))
assert hash(SavedSample(id1, u, [sn], dt1, 'foo', 6)) == hash(SavedSample(
id1, u, [sn], dt1, 'foo', 6))
assert hash(SavedSample(id1, u, [sn], dt1, 'foo', 6)) != hash(SavedSample(
id1, u, [sn], dt1, 'foo', 7))
def test_sample_eq():
sn = SampleNode('foo')
sn2 = SampleNode('bar')
assert Sample([sn], 'yay') == Sample([sn], 'yay')
assert Sample([sn], 'yay') != Sample([sn2], 'yay')
assert Sample([sn], 'yay') != Sample([sn], 'yooo')
id1 = uuid.UUID('1234567890abcdef1234567890abcdef')
id2 = uuid.UUID('1234567890abcdef1234567890abcdea')
dt1 = dt(5)
dt2 = dt(8)
u = UserID('u')
u2 = UserID('u2')
assert SavedSample(id1, u, [sn], dt1) == SavedSample(id1, u, [sn], dt(5))
assert SavedSample(id1, u, [sn], dt1) != SavedSample(id2, u, [sn], dt1)
assert SavedSample(id1, u, [sn], dt1) != SavedSample(id1, u2, [sn], dt1)
assert SavedSample(id1, u, [sn], dt1) != SavedSample(id1, u, [sn2], dt1)
assert SavedSample(id1, u, [sn], dt1) != SavedSample(id1, u, [sn], dt2)
assert SavedSample(id1, u, [sn], dt1, 'yay') == SavedSample(id1, u, [sn], dt1, 'yay')
assert SavedSample(id1, u, [sn], dt1, 'yay') != SavedSample(id1, u, [sn], dt1, 'yooo')
assert SavedSample(id1, u, [sn], dt2, 'yay', 6) == SavedSample(id1, u, [sn], dt2, 'yay', 6)
assert SavedSample(id1, u, [sn], dt1, 'yay', 6) != SavedSample(id1, u, [sn], dt1, 'yay', 7)
assert SavedSample(id1, u, [sn], dt1, 'yay') != Sample([sn], 'yay')
assert Sample([sn], 'yay') != SavedSample(id1, u, [sn], dt1, 'yay')
def _sample_with_id_build_fail(id_, user, nodes, savetime, name, version, expected):
with raises(Exception) as got:
SavedSample(id_, user, nodes, savetime, name, version)
Sample(nodes, name)
assert_exception_correct(got.value, expected)
def test_timestamp_seconds_to_milliseconds(samplestorage):
ts1=1614958000000 # milliseconds
ts2=1614958000 # seconds
ts3=1614958 # seconds
ts4=9007199254740.991 # seconds
id1 = uuid.UUID('1234567890abcdef1234567890abcdef')
id2 = uuid.UUID('1234567890abcdef1234567890abcdee')
assert samplestorage.save_sample(
SavedSample(id1, UserID('user'), [SampleNode('mynode')], dt(ts3), 'foo')) is True
assert samplestorage.save_sample_version(
SavedSample(id1, UserID('user'), [SampleNode('mynode1')], dt(ts2), 'foo')) == 2
assert samplestorage.save_sample(
SavedSample(id2, UserID('user'), [SampleNode('mynode2')], dt(ts3), 'foo')) is True
lid1=uuid.UUID('1234567890abcdef1234567890abcde2')
lid2=uuid.UUID('1234567890abcdef1234567890abcde3')
lid3=uuid.UUID('1234567890abcdef1234567890abcde4')
samplestorage.create_data_link(DataLink(
lid1,
DataUnitID(UPA('42/42/42'), 'dataunit1'),
SampleNodeAddress(SampleAddress(id1, 1), 'mynode'),
dt(ts2),
UserID('user'))
)
samplestorage.create_data_link(DataLink(
lid2,
DataUnitID(UPA('5/89/32'), 'dataunit2'),
SampleNodeAddress(SampleAddress(id2, 1), 'mynode2'),
dt(ts3),
UserID('user'))
)
_create_and_expire_data_link(
samplestorage,
DataLink(
lid3,
DataUnitID(UPA('5/89/33'), 'dataunit1'),
SampleNodeAddress(SampleAddress(id1, 1), 'mynode'),
dt(ts3),
UserID('user')),
dt(ts3+100),
UserID('user')
)
assert samplestorage.get_sample(id1, 1).savetime == dt(ts3)
assert samplestorage.get_sample(id1, 2).savetime == dt(ts2)
assert samplestorage.get_sample(id2).savetime == dt(ts3)
assert samplestorage.get_data_link(lid1).created == dt(ts2)
assert samplestorage.get_data_link(lid2).created == dt(ts3)
assert samplestorage.get_data_link(lid3).created == dt(ts3)
assert samplestorage.get_data_link(lid3).expired == dt(ts3+100)
threshold=1000000000000 # current timestamp in milliseconds is above 1600000000000
query="""
FOR sample1 IN samples_nodes
FILTER sample1.saved < @threshold
UPDATE sample1 WITH { saved: ROUND(sample1.saved * 1000) } IN samples_nodes
FOR sample2 IN samples_version
FILTER sample2.saved < @threshold
UPDATE sample2 WITH { saved: ROUND(sample2.saved * 1000) } IN samples_version
FOR link IN samples_data_link
FILTER link.expired < @threshold OR link.created < @threshold
UPDATE link WITH {
expired: link.expired < @threshold ? ROUND(link.expired * 1000) : link.expired,
created: link.created < @threshold ? ROUND(link.created * 1000) : link.created
} IN samples_data_link
"""
samplestorage._db.aql.execute(query, bind_vars={'threshold': threshold})
assert samplestorage.get_sample(id1, 1).savetime == dt(ts2)
assert samplestorage.get_sample(id1, 2).savetime == dt(ts2)
assert samplestorage.get_sample(id2).savetime == dt(ts2)
assert samplestorage.get_data_link(lid1).created == dt(ts2)
assert samplestorage.get_data_link(lid2).created == dt(ts2)
assert samplestorage.get_data_link(lid3).created == dt(ts2)
assert samplestorage.get_data_link(lid3).expired == dt((ts3+100) * 1000)
samplestorage._db.aql.execute(query, bind_vars={'threshold': threshold})
assert samplestorage.get_sample(id1, 1).savetime == dt(ts2)
assert samplestorage.get_sample(id1, 2).savetime == dt(ts2)
assert samplestorage.get_sample(id2).savetime == dt(ts2)
assert samplestorage.get_data_link(lid1).created == dt(ts2)
assert samplestorage.get_data_link(lid2).created == dt(ts2)
assert samplestorage.get_data_link(lid3).created == dt(ts2)
assert samplestorage.get_data_link(lid3).expired == dt((ts3+100) * 1000)