def get_datum_pages(*args, **kwargs):
datum_cursor = get_datum_cursor(*args, **kwargs)
while True:
result = list(itertools.islice(datum_cursor, page_size))
if result:
yield event_model.pack_datum_page(*result)
else:
break
def test_round_trip_datum_page_with_empty_data():
datum_page = {
'datum_id': ['a', 'b', 'c'],
'resource': 'd',
'datum_kwargs': {}
}
datums = list(event_model.unpack_datum_page(datum_page))
assert len(datums) == 3
page_again = event_model.pack_datum_page(*datums)
assert page_again == datum_page
def test_bulk_datum_to_datum_page():
run_bundle = event_model.compose_run()
res_bundle = run_bundle.compose_resource(
spec='TIFF', root='/tmp', resource_path='stack.tiff',
resource_kwargs={})
datum1 = res_bundle.compose_datum(datum_kwargs={'slice': 5})
datum2 = res_bundle.compose_datum(datum_kwargs={'slice': 10})
actual = event_model.pack_datum_page(datum1, datum2)
bulk_datum = {'resource': res_bundle.resource_doc['uid'],
'datum_kwarg_list': [datum1['datum_kwargs'],
datum2['datum_kwargs']],
'datum_ids': [datum1['datum_id'], datum2['datum_id']]}
expected = event_model.bulk_datum_to_datum_page(bulk_datum)
assert actual == expected
def test_round_trip_pagination():
run_bundle = event_model.compose_run()
desc_bundle = run_bundle.compose_descriptor(data_keys={
'motor': {
'shape': [],
'dtype': 'number',
'source': '...'
},
'image': {
'shape': [512, 512],
'dtype': 'number',
'source': '...',
'external': 'FILESTORE:'
}
},
name='primary')
res_bundle = run_bundle.compose_resource(spec='TIFF',
root='/tmp',
resource_path='stack.tiff',
resource_kwargs={})
datum_doc1 = res_bundle.compose_datum(datum_kwargs={'slice': 5})
datum_doc2 = res_bundle.compose_datum(datum_kwargs={'slice': 10})
datum_doc3 = res_bundle.compose_datum(datum_kwargs={'slice': 15})
event_doc1 = desc_bundle.compose_event(data={
'motor': 0,
'image': datum_doc1['datum_id']
},
timestamps={
'motor': 0,
'image': 0
},
filled={'image': False},
seq_num=1)
event_doc2 = desc_bundle.compose_event(data={
'motor': 1,
'image': datum_doc2['datum_id']
},
timestamps={
'motor': 0,
'image': 0
},
filled={'image': False},
seq_num=1)
event_doc3 = desc_bundle.compose_event(data={
'motor': 2,
'image': datum_doc3['datum_id']
},
timestamps={
'motor': 0,
'image': 0
},
filled={'image': False},
seq_num=1)
# Round trip single event -> event_page -> event.
expected = event_doc1
actual, = event_model.unpack_event_page(
event_model.pack_event_page(expected))
assert actual == expected
# Round trip two events -> event_page -> events.
expected = [event_doc1, event_doc2]
actual = list(
event_model.unpack_event_page(event_model.pack_event_page(*expected)))
assert actual == expected
# Round trip three events -> event_page -> events.
expected = [event_doc1, event_doc2, event_doc3]
actual = list(
event_model.unpack_event_page(event_model.pack_event_page(*expected)))
assert actual == expected
# Round trip on docs that don't have a filled key
unfilled_doc1 = event_doc1
unfilled_doc1.pop('filled')
unfilled_doc2 = event_doc2
unfilled_doc2.pop('filled')
unfilled_doc3 = event_doc3
unfilled_doc3.pop('filled')
expected = [unfilled_doc1, unfilled_doc2, unfilled_doc3]
actual = list(
event_model.unpack_event_page(event_model.pack_event_page(*expected)))
for doc in actual:
doc.pop('filled')
assert actual == expected
# Round trip one datum -> datum_page -> datum.
expected = datum_doc1
actual, = event_model.unpack_datum_page(
event_model.pack_datum_page(expected))
assert actual == expected
# Round trip two datum -> datum_page -> datum.
expected = [datum_doc1, datum_doc2]
actual = list(
event_model.unpack_datum_page(event_model.pack_datum_page(*expected)))
assert actual == expected
# Round trip three datum -> datum_page -> datum.
expected = [datum_doc1, datum_doc2, datum_doc3]
actual = list(
event_model.unpack_datum_page(event_model.pack_datum_page(*expected)))
assert actual == expected
# Check edge case where datum_kwargs are empty.
datum_doc1 = res_bundle.compose_datum(datum_kwargs={})
datum_doc2 = res_bundle.compose_datum(datum_kwargs={})
datum_doc3 = res_bundle.compose_datum(datum_kwargs={})
# Round trip one datum -> datum_page -> datum.
expected = datum_doc1
actual, = event_model.unpack_datum_page(
event_model.pack_datum_page(expected))
assert actual == expected
# Round trip two datum -> datum_page -> datum.
expected = [datum_doc1, datum_doc2]
actual = list(
event_model.unpack_datum_page(event_model.pack_datum_page(*expected)))
assert actual == expected
# Round trip three datum -> datum_page -> datum.
expected = [datum_doc1, datum_doc2, datum_doc3]
actual = list(
event_model.unpack_datum_page(event_model.pack_datum_page(*expected)))
assert actual == expected
def test_pack_empty_raises():
with pytest.raises(ValueError):
event_model.pack_event_page()
with pytest.raises(ValueError):
event_model.pack_datum_page()
def test_document_router_dispatch_datum():
datum_calls = [] # used for counting calls
datum_page_calls = [] # used for counting calls
# example documents
datum1 = {
'datum_id': 'placeholder/1',
'resource': 'placeholder',
'datum_kwargs': {
'index': 1
}
}
datum2 = {
'datum_id': 'placholder/2',
'resource': 'placeholder',
'datum_kwargs': {
'index': 2
}
}
datum_page = event_model.pack_datum_page(datum1, datum2)
def check(ret, original=None):
name, doc = ret
assert doc is not None
assert doc is not NotImplemented
if original is not None:
# Verify that a copy is returned.
assert doc is not original # ret is such a poser, dude.
assert doc == original
class DefinesNeitherDatumNorDatumPage(event_model.DocumentRouter):
def datum(self, doc):
datum_calls.append(object())
# This returns NotImplemented.
return super().datum(doc)
def datum_page(self, doc):
datum_page_calls.append(object())
# This returns NotImplemented.
return super().datum_page(doc)
dr = DefinesNeitherDatumNorDatumPage()
# Test that Datum is routed to Datum and DatumPage.
check(dr('datum', datum1))
assert len(datum_calls) == 1
assert len(datum_page_calls) == 1
datum_calls.clear()
datum_page_calls.clear()
# Test that DatumPage is routed to DatumPage and Datum *once* before giving
# up.
check(dr('datum_page', datum_page))
assert len(datum_page_calls) == 1
assert len(datum_calls) == 1
datum_calls.clear()
datum_page_calls.clear()
class DefinesDatumNotDatumPage(event_model.DocumentRouter):
def datum(self, doc):
# Just a dumb test that check something particular to these example
# documents.
assert doc['datum_kwargs']['index'] == int(doc['datum_id'][-1])
datum_calls.append(object())
return dict(doc)
def datum_page(self, doc):
datum_page_calls.append(object())
# This returns NotImplemented.
return super().datum_page(doc)
dr = DefinesDatumNotDatumPage()
# Test that Datum is routed to Datum.
check(dr('datum', datum1), datum1)
assert len(datum_calls) == 1
assert len(datum_page_calls) == 0
datum_calls.clear()
datum_page_calls.clear()
# Test that DatumPage is unpacked and routed to Datum one at a time.
check(dr('datum_page', datum_page), datum_page)
assert len(datum_page_calls) == 1
assert len(datum_calls) == 2
datum_calls.clear()
datum_page_calls.clear()
class DefinesDatumPageNotDatum(event_model.DocumentRouter):
def datum(self, doc):
datum_calls.append(object())
# This returns NotImplemented.
return super().datum_page(doc)
def datum_page(self, doc):
# Just a dumb test that check something particular to these example
# documents.
assert doc['datum_kwargs']['index'][0] == int(
doc['datum_id'][0][-1])
datum_page_calls.append(object())
return dict(doc)
dr = DefinesDatumPageNotDatum()
# Test that Datum is packed and routed to DatumPage.
check(dr('datum', datum1), datum1)
assert len(datum_calls) == 1
assert len(datum_page_calls) == 1
datum_calls.clear()
datum_page_calls.clear()
# Test that DatumPage is routed to DatumPage.
check(dr('datum_page', datum_page), datum_page)
assert len(datum_page_calls) == 1
assert len(datum_calls) == 0
datum_calls.clear()
datum_page_calls.clear()
# Test that DatumPage is routed to DatumPage.
class DefinesDatumPageAndDatum(event_model.DocumentRouter):
def datum(self, doc):
# Just a dumb test that check something particular to these example
# documents.
assert doc['datum_kwargs']['index'] == int(doc['datum_id'][-1])
datum_calls.append(object())
return dict(doc)
def datum_page(self, doc):
# Just a dumb test that check something particular to these example
# documents.
assert doc['datum_kwargs']['index'][0] == int(
doc['datum_id'][0][-1])
datum_page_calls.append(object())
return dict(doc)
dr = DefinesDatumPageAndDatum()
# Test that Datum is routed to Datum.
check(dr('datum', datum1), datum1)
assert len(datum_calls) == 1
assert len(datum_page_calls) == 0
datum_calls.clear()
datum_page_calls.clear()
# Test that DatumPage is routed to DatumPage.
check(dr('datum_page', datum_page), datum_page)
assert len(datum_page_calls) == 1
assert len(datum_calls) == 0
datum_calls.clear()
datum_page_calls.clear()
