def get_datum_pages(*args, **kwargs): datum_cursor = get_datum_cursor(*args, **kwargs) while True: result = list(itertools.islice(datum_cursor, page_size)) if result: yield event_model.pack_datum_page(*result) else: break
def test_round_trip_datum_page_with_empty_data(): datum_page = { 'datum_id': ['a', 'b', 'c'], 'resource': 'd', 'datum_kwargs': {} } datums = list(event_model.unpack_datum_page(datum_page)) assert len(datums) == 3 page_again = event_model.pack_datum_page(*datums) assert page_again == datum_page
def test_bulk_datum_to_datum_page(): run_bundle = event_model.compose_run() res_bundle = run_bundle.compose_resource( spec='TIFF', root='/tmp', resource_path='stack.tiff', resource_kwargs={}) datum1 = res_bundle.compose_datum(datum_kwargs={'slice': 5}) datum2 = res_bundle.compose_datum(datum_kwargs={'slice': 10}) actual = event_model.pack_datum_page(datum1, datum2) bulk_datum = {'resource': res_bundle.resource_doc['uid'], 'datum_kwarg_list': [datum1['datum_kwargs'], datum2['datum_kwargs']], 'datum_ids': [datum1['datum_id'], datum2['datum_id']]} expected = event_model.bulk_datum_to_datum_page(bulk_datum) assert actual == expected
def test_round_trip_pagination(): run_bundle = event_model.compose_run() desc_bundle = run_bundle.compose_descriptor(data_keys={ 'motor': { 'shape': [], 'dtype': 'number', 'source': '...' }, 'image': { 'shape': [512, 512], 'dtype': 'number', 'source': '...', 'external': 'FILESTORE:' } }, name='primary') res_bundle = run_bundle.compose_resource(spec='TIFF', root='/tmp', resource_path='stack.tiff', resource_kwargs={}) datum_doc1 = res_bundle.compose_datum(datum_kwargs={'slice': 5}) datum_doc2 = res_bundle.compose_datum(datum_kwargs={'slice': 10}) datum_doc3 = res_bundle.compose_datum(datum_kwargs={'slice': 15}) event_doc1 = desc_bundle.compose_event(data={ 'motor': 0, 'image': datum_doc1['datum_id'] }, timestamps={ 'motor': 0, 'image': 0 }, filled={'image': False}, seq_num=1) event_doc2 = desc_bundle.compose_event(data={ 'motor': 1, 'image': datum_doc2['datum_id'] }, timestamps={ 'motor': 0, 'image': 0 }, filled={'image': False}, seq_num=1) event_doc3 = desc_bundle.compose_event(data={ 'motor': 2, 'image': datum_doc3['datum_id'] }, timestamps={ 'motor': 0, 'image': 0 }, filled={'image': False}, seq_num=1) # Round trip single event -> event_page -> event. expected = event_doc1 actual, = event_model.unpack_event_page( event_model.pack_event_page(expected)) assert actual == expected # Round trip two events -> event_page -> events. expected = [event_doc1, event_doc2] actual = list( event_model.unpack_event_page(event_model.pack_event_page(*expected))) assert actual == expected # Round trip three events -> event_page -> events. expected = [event_doc1, event_doc2, event_doc3] actual = list( event_model.unpack_event_page(event_model.pack_event_page(*expected))) assert actual == expected # Round trip on docs that don't have a filled key unfilled_doc1 = event_doc1 unfilled_doc1.pop('filled') unfilled_doc2 = event_doc2 unfilled_doc2.pop('filled') unfilled_doc3 = event_doc3 unfilled_doc3.pop('filled') expected = [unfilled_doc1, unfilled_doc2, unfilled_doc3] actual = list( event_model.unpack_event_page(event_model.pack_event_page(*expected))) for doc in actual: doc.pop('filled') assert actual == expected # Round trip one datum -> datum_page -> datum. expected = datum_doc1 actual, = event_model.unpack_datum_page( event_model.pack_datum_page(expected)) assert actual == expected # Round trip two datum -> datum_page -> datum. expected = [datum_doc1, datum_doc2] actual = list( event_model.unpack_datum_page(event_model.pack_datum_page(*expected))) assert actual == expected # Round trip three datum -> datum_page -> datum. expected = [datum_doc1, datum_doc2, datum_doc3] actual = list( event_model.unpack_datum_page(event_model.pack_datum_page(*expected))) assert actual == expected # Check edge case where datum_kwargs are empty. datum_doc1 = res_bundle.compose_datum(datum_kwargs={}) datum_doc2 = res_bundle.compose_datum(datum_kwargs={}) datum_doc3 = res_bundle.compose_datum(datum_kwargs={}) # Round trip one datum -> datum_page -> datum. expected = datum_doc1 actual, = event_model.unpack_datum_page( event_model.pack_datum_page(expected)) assert actual == expected # Round trip two datum -> datum_page -> datum. expected = [datum_doc1, datum_doc2] actual = list( event_model.unpack_datum_page(event_model.pack_datum_page(*expected))) assert actual == expected # Round trip three datum -> datum_page -> datum. expected = [datum_doc1, datum_doc2, datum_doc3] actual = list( event_model.unpack_datum_page(event_model.pack_datum_page(*expected))) assert actual == expected
def test_pack_empty_raises(): with pytest.raises(ValueError): event_model.pack_event_page() with pytest.raises(ValueError): event_model.pack_datum_page()
def test_document_router_dispatch_datum(): datum_calls = [] # used for counting calls datum_page_calls = [] # used for counting calls # example documents datum1 = { 'datum_id': 'placeholder/1', 'resource': 'placeholder', 'datum_kwargs': { 'index': 1 } } datum2 = { 'datum_id': 'placholder/2', 'resource': 'placeholder', 'datum_kwargs': { 'index': 2 } } datum_page = event_model.pack_datum_page(datum1, datum2) def check(ret, original=None): name, doc = ret assert doc is not None assert doc is not NotImplemented if original is not None: # Verify that a copy is returned. assert doc is not original # ret is such a poser, dude. assert doc == original class DefinesNeitherDatumNorDatumPage(event_model.DocumentRouter): def datum(self, doc): datum_calls.append(object()) # This returns NotImplemented. return super().datum(doc) def datum_page(self, doc): datum_page_calls.append(object()) # This returns NotImplemented. return super().datum_page(doc) dr = DefinesNeitherDatumNorDatumPage() # Test that Datum is routed to Datum and DatumPage. check(dr('datum', datum1)) assert len(datum_calls) == 1 assert len(datum_page_calls) == 1 datum_calls.clear() datum_page_calls.clear() # Test that DatumPage is routed to DatumPage and Datum *once* before giving # up. check(dr('datum_page', datum_page)) assert len(datum_page_calls) == 1 assert len(datum_calls) == 1 datum_calls.clear() datum_page_calls.clear() class DefinesDatumNotDatumPage(event_model.DocumentRouter): def datum(self, doc): # Just a dumb test that check something particular to these example # documents. assert doc['datum_kwargs']['index'] == int(doc['datum_id'][-1]) datum_calls.append(object()) return dict(doc) def datum_page(self, doc): datum_page_calls.append(object()) # This returns NotImplemented. return super().datum_page(doc) dr = DefinesDatumNotDatumPage() # Test that Datum is routed to Datum. check(dr('datum', datum1), datum1) assert len(datum_calls) == 1 assert len(datum_page_calls) == 0 datum_calls.clear() datum_page_calls.clear() # Test that DatumPage is unpacked and routed to Datum one at a time. check(dr('datum_page', datum_page), datum_page) assert len(datum_page_calls) == 1 assert len(datum_calls) == 2 datum_calls.clear() datum_page_calls.clear() class DefinesDatumPageNotDatum(event_model.DocumentRouter): def datum(self, doc): datum_calls.append(object()) # This returns NotImplemented. return super().datum_page(doc) def datum_page(self, doc): # Just a dumb test that check something particular to these example # documents. assert doc['datum_kwargs']['index'][0] == int( doc['datum_id'][0][-1]) datum_page_calls.append(object()) return dict(doc) dr = DefinesDatumPageNotDatum() # Test that Datum is packed and routed to DatumPage. check(dr('datum', datum1), datum1) assert len(datum_calls) == 1 assert len(datum_page_calls) == 1 datum_calls.clear() datum_page_calls.clear() # Test that DatumPage is routed to DatumPage. check(dr('datum_page', datum_page), datum_page) assert len(datum_page_calls) == 1 assert len(datum_calls) == 0 datum_calls.clear() datum_page_calls.clear() # Test that DatumPage is routed to DatumPage. class DefinesDatumPageAndDatum(event_model.DocumentRouter): def datum(self, doc): # Just a dumb test that check something particular to these example # documents. assert doc['datum_kwargs']['index'] == int(doc['datum_id'][-1]) datum_calls.append(object()) return dict(doc) def datum_page(self, doc): # Just a dumb test that check something particular to these example # documents. assert doc['datum_kwargs']['index'][0] == int( doc['datum_id'][0][-1]) datum_page_calls.append(object()) return dict(doc) dr = DefinesDatumPageAndDatum() # Test that Datum is routed to Datum. check(dr('datum', datum1), datum1) assert len(datum_calls) == 1 assert len(datum_page_calls) == 0 datum_calls.clear() datum_page_calls.clear() # Test that DatumPage is routed to DatumPage. check(dr('datum_page', datum_page), datum_page) assert len(datum_page_calls) == 1 assert len(datum_calls) == 0 datum_calls.clear() datum_page_calls.clear()