def test_bulk_events_to_event_page(): run_bundle = event_model.compose_run() desc_bundle = run_bundle.compose_descriptor( data_keys={'motor': {'shape': [], 'dtype': 'number', 'source': '...'}, 'image': {'shape': [512, 512], 'dtype': 'number', 'source': '...', 'external': 'FILESTORE:'}}, name='primary') desc_bundle_baseline = run_bundle.compose_descriptor( data_keys={'motor': {'shape': [], 'dtype': 'number', 'source': '...'}}, name='baseline') res_bundle = run_bundle.compose_resource( spec='TIFF', root='/tmp', resource_path='stack.tiff', resource_kwargs={}) datum_doc1 = res_bundle.compose_datum(datum_kwargs={'slice': 5}) datum_doc2 = res_bundle.compose_datum(datum_kwargs={'slice': 10}) event1 = desc_bundle.compose_event( data={'motor': 0, 'image': datum_doc1['datum_id']}, timestamps={'motor': 0, 'image': 0}, filled={'image': False}, seq_num=1) event2 = desc_bundle.compose_event( data={'motor': 0, 'image': datum_doc2['datum_id']}, timestamps={'motor': 0, 'image': 0}, filled={'image': False}, seq_num=2) event3 = desc_bundle_baseline.compose_event( data={'motor': 0}, timestamps={'motor': 0}, seq_num=1) primary_event_page = event_model.pack_event_page(event1, event2) baseline_event_page = event_model.pack_event_page(event3) bulk_events = {'primary': [event1, event2], 'baseline': [event3]} pages = event_model.bulk_events_to_event_pages(bulk_events) assert tuple(pages) == (primary_event_page, baseline_event_page)
def test_verify_filled(filler): "Test the utility function verify_filled." with pytest.raises(event_model.UnfilledData): event_model.verify_filled(event_model.pack_event_page(raw_event)) event = copy.deepcopy(raw_event) name, doc = filler('event', event) event_model.verify_filled(event_model.pack_event_page(event))
def test_fill_event_page(RE, hw): docs = [] def callback(name, doc): docs.append((name, doc)) RE(count([hw.img]), callback) docs dask_filler = Filler({'NPY_SEQ': NumpySeqHandler}, coerce='delayed', inplace=False) filled_docs = [] _, event = docs[-2] event_page = event_model.pack_event_page(event) docs[-2] = ('event_page', event_page) dask_filler = Filler({'NPY_SEQ': NumpySeqHandler}, coerce='delayed', inplace=False) filled_docs = [] for name, doc in docs: filled_docs.append(dask_filler(name, doc)) _, dask_filled_event_page = filled_docs[-2] arr = dask_filled_event_page['data']['img'][0].compute() assert arr.shape == (10, 10) assert isinstance(arr, numpy.ndarray)
def test_sanitize_doc(): run_bundle = event_model.compose_run() desc_bundle = run_bundle.compose_descriptor( data_keys={'motor': {'shape': [], 'dtype': 'number', 'source': '...'}, 'image': {'shape': [512, 512], 'dtype': 'number', 'source': '...', 'external': 'FILESTORE:'}}, name='primary') desc_bundle_baseline = run_bundle.compose_descriptor( data_keys={'motor': {'shape': [], 'dtype': 'number', 'source': '...'}}, name='baseline') event1 = desc_bundle.compose_event( data={'motor': 0, 'image': numpy.ones((512, 512))}, timestamps={'motor': 0, 'image': 0}, filled={'image': True}, seq_num=1) event2 = desc_bundle.compose_event( data={'motor': 0, 'image': numpy.ones((512, 512))}, timestamps={'motor': 0, 'image': 0}, filled={'image': True}, seq_num=2) event3 = desc_bundle_baseline.compose_event( data={'motor': 0}, timestamps={'motor': 0}, seq_num=1) event_page = event_model.pack_event_page(event1, event2) bulk_events = {'primary': [event1, event2], 'baseline': [event3]} json.dumps(event_model.sanitize_doc(event_page)) json.dumps(event_model.sanitize_doc(bulk_events)) json.dumps(event_model.sanitize_doc(event1))
def get_event_pages(*args, **kwargs): event_cursor = get_event_cursor(*args, **kwargs) while True: result = list(itertools.islice(event_cursor, page_size)) if result: yield event_model.pack_event_page(*result) else: break
def collect(name, doc): if name == 'event': events.append(doc) elif name == 'stop': collector.append( ('event_page', event_model.pack_event_page(*events))) collector.append((name, doc)) else: collector.append((name, doc))
def event_page(self, doc): event = self.event # Avoid attribute lookup in hot loop. filled_events = [] for event_doc in event_model.unpack_event_page(doc): filled_events.append(event(event_doc)) new_event_page = event_model.pack_event_page(*filled_events) # Modify original doc in place, as we do with 'event'. doc['data'] = new_event_page['data'] return doc
def event(self, doc): '''Add event document information to a ".tiff" file. This method adds event document information to a ".tiff" file, creating it if necessary. .. warning:: All non 2D 'image-like' data is explicitly ignored. .. note:: The data in Events might be structured as an Event, an EventPage, or a "bulk event" (deprecated). The DocumentRouter base class takes care of first transforming the other representations into an EventPage and then routing them through here, as we require Event documents _in this case_ we overwrite both the `event` method and the `event_page` method so we can assume we will always receive an Event. Parameters: ----------- doc : dict Event document ''' event_model.verify_filled(event_model.pack_event_page(*[doc])) descriptor = self._descriptors[doc['descriptor']] stream_name = descriptor.get('name') for field in doc['data']: img = doc['data'][field] # Check that the data is 2D or 3D; if not ignore it. data_key = descriptor['data_keys'][field] ndim = len(data_key['shape'] or []) if data_key['dtype'] == 'array' and 1 < ndim < 4: img_asarray = numpy.asarray(img, dtype=self._astype) if ndim == 2: # handle 2D data just like 3D data # by adding a 3rd dimension img_asarray = numpy.expand_dims(img_asarray, axis=0) for i in range(img_asarray.shape[0]): img_asarray_2d = img_asarray[i, :] num = next(self._counter[stream_name][field]) filename = get_prefixed_filename( file_prefix=self._file_prefix, start_doc=self._start, descriptor_doc=descriptor, event_doc=doc, num=num, stream_name=stream_name, field=field) file = self._manager.open('stream_data', filename, 'xb') tw = TiffWriter(file, **self._init_kwargs) self._tiff_writers[stream_name][field + f'-{num}'] = tw tw.save(img_asarray_2d, *self._kwargs)
def test_context_manager_with_event_page(): with event_model.Filler(reg, inplace=True) as filler: filler('start', run_bundle.start_doc) filler('descriptor', desc_bundle.descriptor_doc) filler('descriptor', desc_bundle_baseline.descriptor_doc) filler('resource', res_bundle.resource_doc) filler('datum', datum_doc) event_page = event_model.pack_event_page(copy.deepcopy(raw_event)) name, doc = filler('event_page', event_page) assert name == 'event_page' assert doc is event_page filler('stop', stop_doc) assert not filler.closed assert event_page['data']['image'][0].shape == (5, 5) assert filler.closed
def test_round_trip_event_page_with_empty_data(): event_page = { 'time': [1, 2, 3], 'seq_num': [1, 2, 3], 'uid': ['a', 'b', 'c'], 'descriptor': 'd', 'data': {}, 'timestamps': {}, 'filled': {} } events = list(event_model.unpack_event_page(event_page)) assert len(events) == 3 page_again = event_model.pack_event_page(*events) assert page_again == event_page
def event(self, doc): '''Add event document information to a ".tiff" file. This method adds event document information to a ".tiff" file, creating it if necessary. .. warning:: All non 2D 'image-like' data is explicitly ignored. .. note:: The data in Events might be structured as an Event, an EventPage, or a "bulk event" (deprecated). The DocumentRouter base class takes care of first transforming the other repsentations into an EventPage and then routing them through here, as we require Event documents _in this case_ we overwrite both the `event` method and the `event_page` method so we can assume we will always receive an Event. Parameters: ----------- doc : dict Event document ''' event_model.verify_filled(event_model.pack_event_page(*[doc])) descriptor = self._descriptors[doc['descriptor']] streamname = descriptor.get('name') for field in doc['data']: img = doc['data'][field] # check that the data is 2D, if not ignore it img_asarray = numpy.asarray(img, dtype=self._astype) if img_asarray.ndim == 2: # template the file name. self._templated_file_prefix = self._file_prefix.format( start=self._start, descriptor=descriptor, event=doc) num = next(self._counter[streamname][field]) filename = (f'{self._templated_file_prefix}' f'{streamname}-{field}-{num}.tiff') file = self._manager.open('stream_data', filename, 'xb') tw = TiffWriter(file, **self._init_kwargs) self._tiff_writers[streamname][field + f'-{num}'] = tw tw.save(img_asarray, *self._kwargs)
def test_filler(tmp_path): class DummyHandler: def __init__(self, resource_path, a, b): assert a == 1 assert b == 2 assert resource_path == str(tmp_path / "stack.tiff") def __call__(self, c, d): assert c == 3 assert d == 4 return numpy.ones((5, 5)) path_root = str(tmp_path) reg = {'DUMMY': DummyHandler} filler = event_model.Filler(reg, inplace=True) run_bundle = event_model.compose_run() desc_bundle = run_bundle.compose_descriptor(data_keys={ 'motor': { 'shape': [], 'dtype': 'number', 'source': '...' }, 'image': { 'shape': [512, 512], 'dtype': 'number', 'source': '...', 'external': 'FILESTORE:' } }, name='primary') desc_bundle_baseline = run_bundle.compose_descriptor( data_keys={'motor': { 'shape': [], 'dtype': 'number', 'source': '...' }}, name='baseline') res_bundle = run_bundle.compose_resource(spec='DUMMY', root=path_root, resource_path='stack.tiff', resource_kwargs={ 'a': 1, 'b': 2 }) datum_doc = res_bundle.compose_datum(datum_kwargs={'c': 3, 'd': 4}) raw_event = desc_bundle.compose_event(data={ 'motor': 0, 'image': datum_doc['datum_id'] }, timestamps={ 'motor': 0, 'image': 0 }, filled={'image': False}, seq_num=1) filler('start', run_bundle.start_doc) filler('descriptor', desc_bundle.descriptor_doc) filler('descriptor', desc_bundle_baseline.descriptor_doc) filler('resource', res_bundle.resource_doc) filler('datum', datum_doc) event = copy.deepcopy(raw_event) assert isinstance(event['data']['image'], str) filler('event', event) stop_doc = run_bundle.compose_stop() filler('stop', stop_doc) assert event['data']['image'].shape == (5, 5) assert not filler._closed # Test NoFiller. filler = event_model.NoFiller(reg) filler('start', run_bundle.start_doc) filler('descriptor', desc_bundle.descriptor_doc) filler('descriptor', desc_bundle_baseline.descriptor_doc) filler('resource', res_bundle.resource_doc) filler('datum', datum_doc) event = copy.deepcopy(raw_event) assert isinstance(event['data']['image'], str) filler('event', event) # Check that it *hasn't* been filled. assert isinstance(event['data']['image'], str) filler('stop', stop_doc) # Test get_handler() method. handler = filler.get_handler(res_bundle.resource_doc) # The method does not expose the internal cache of handlers, so it should # not return the same instance when called repeatedly. assert filler.get_handler(res_bundle.resource_doc) is not handler # Test closing. filler.close() with pytest.raises(event_model.EventModelRuntimeError): filler.get_handler(res_bundle.resource_doc) with pytest.raises(event_model.EventModelRuntimeError): filler('stop', stop_doc) # Test context manager with Event. with event_model.Filler(reg, inplace=True) as filler: filler('start', run_bundle.start_doc) filler('descriptor', desc_bundle.descriptor_doc) filler('descriptor', desc_bundle_baseline.descriptor_doc) filler('resource', res_bundle.resource_doc) filler('datum', datum_doc) event = copy.deepcopy(raw_event) name, doc = filler('event', event) assert name == 'event' assert doc is event filler('stop', stop_doc) assert not filler._closed assert event['data']['image'].shape == (5, 5) assert filler._closed # Test context manager with EventPage. with event_model.Filler(reg, inplace=True) as filler: filler('start', run_bundle.start_doc) filler('descriptor', desc_bundle.descriptor_doc) filler('descriptor', desc_bundle_baseline.descriptor_doc) filler('resource', res_bundle.resource_doc) filler('datum', datum_doc) event_page = event_model.pack_event_page(copy.deepcopy(raw_event)) name, doc = filler('event_page', event_page) assert name == 'event_page' assert doc is event_page filler('stop', stop_doc) assert not filler._closed assert event_page['data']['image'][0].shape == (5, 5) assert filler._closed # Test undefined handler spec with event_model.Filler({}, inplace=True) as filler: filler('start', run_bundle.start_doc) filler('descriptor', desc_bundle.descriptor_doc) filler('descriptor', desc_bundle_baseline.descriptor_doc) filler('resource', res_bundle.resource_doc) filler('datum', datum_doc) event = copy.deepcopy(raw_event) assert isinstance(event['data']['image'], str) with pytest.raises(event_model.UndefinedAssetSpecification): filler('event', event) # Test exclude and include. with pytest.raises(ValueError): event_model.Filler({}, include=[], exclude=[], inplace=True) with pytest.warns(DeprecationWarning): with event_model.Filler(reg, exclude=['image'], inplace=True) as filler: filler('start', run_bundle.start_doc) filler('descriptor', desc_bundle.descriptor_doc) filler('descriptor', desc_bundle_baseline.descriptor_doc) filler('resource', res_bundle.resource_doc) filler('datum', datum_doc) event = copy.deepcopy(raw_event) assert isinstance(event['data']['image'], str) filler('event', event) filler('stop', stop_doc) with pytest.warns(DeprecationWarning): with event_model.Filler(reg, include=['image'], inplace=True) as filler: filler('start', run_bundle.start_doc) filler('descriptor', desc_bundle.descriptor_doc) filler('descriptor', desc_bundle_baseline.descriptor_doc) filler('resource', res_bundle.resource_doc) filler('datum', datum_doc) event = copy.deepcopy(raw_event) filler('event', event) filler('stop', stop_doc) assert not filler._closed assert event['data']['image'].shape == (5, 5) with pytest.warns(DeprecationWarning): with event_model.Filler(reg, include=['image', 'EXTRA THING'], inplace=True) as filler: filler('start', run_bundle.start_doc) filler('descriptor', desc_bundle.descriptor_doc) filler('descriptor', desc_bundle_baseline.descriptor_doc) filler('resource', res_bundle.resource_doc) filler('datum', datum_doc) event = copy.deepcopy(raw_event) filler('event', event) filler('stop', stop_doc) assert not filler._closed assert event['data']['image'].shape == (5, 5) class DummyHandlerRootMapTest: def __init__(self, resource_path, a, b): assert a == 1 assert b == 2 assert resource_path == str(tmp_path / "moved" / "stack.tiff") def __call__(self, c, d): assert c == 3 assert d == 4 return numpy.ones((5, 5)) with event_model.Filler({'DUMMY': DummyHandlerRootMapTest}, root_map={path_root: str(tmp_path / "moved")}, inplace=True) as filler: filler('start', run_bundle.start_doc) filler('descriptor', desc_bundle.descriptor_doc) filler('descriptor', desc_bundle_baseline.descriptor_doc) filler('resource', res_bundle.resource_doc) filler('datum', datum_doc) event = copy.deepcopy(raw_event) filler('event', event) filler('stop', stop_doc) assert not filler._closed assert event['data']['image'].shape == (5, 5) # Test verify_filled. with pytest.raises(event_model.UnfilledData): event_model.verify_filled(event_model.pack_event_page(raw_event)) event_model.verify_filled(event_model.pack_event_page(event)) # Test inplace. with event_model.Filler(reg, inplace=True) as filler: filler('start', run_bundle.start_doc) filler('descriptor', desc_bundle.descriptor_doc) filler('descriptor', desc_bundle_baseline.descriptor_doc) filler('resource', res_bundle.resource_doc) filler('datum', datum_doc) # Test event() event = copy.deepcopy(raw_event) name, filled_event = filler('event', event) assert filled_event is event event = copy.deepcopy(raw_event) # Test fill_event() filled_event = filler.fill_event(event) assert filled_event is event # Test event_page() event_page = event_model.pack_event_page(copy.deepcopy(raw_event)) _, filled_event_page = filler('event_page', event_page) assert filled_event_page is event_page # Test fill_event_page() event_page = event_model.pack_event_page(copy.deepcopy(raw_event)) filled_event_page = filler.fill_event_page(event_page) assert filled_event_page is event_page # Test fill_event and fill_event_page again with inplace=False. # Test fill_event() filled_event = filler.fill_event(event, inplace=False) assert filled_event is not event # Test fill_event_page() event_page = event_model.pack_event_page(copy.deepcopy(raw_event)) filled_event_page = filler.fill_event_page(event_page, inplace=False) assert filled_event_page is not event_page with event_model.Filler(reg, inplace=False) as filler: filler('start', run_bundle.start_doc) filler('descriptor', desc_bundle.descriptor_doc) filler('descriptor', desc_bundle_baseline.descriptor_doc) filler('resource', res_bundle.resource_doc) filler('datum', datum_doc) event = copy.deepcopy(raw_event) name, filled_event = filler('event', event) assert filled_event is not event assert isinstance(event['data']['image'], str) event = copy.deepcopy(raw_event) # Test fill_event() filled_event = filler.fill_event(event) assert filled_event is not event # Test event_page() event_page = event_model.pack_event_page(copy.deepcopy(raw_event)) _, filled_event_page = filler('event_page', event_page) assert filled_event_page is not event_page # Test fill_event_page() event_page = event_model.pack_event_page(copy.deepcopy(raw_event)) filled_event_page = filler.fill_event_page(event_page) assert filled_event_page is not event_page # Test fill_event and fill_event_page again with inplace=True. # Test fill_event() filled_event = filler.fill_event(event, inplace=True) assert filled_event is event # Test fill_event_page() event_page = event_model.pack_event_page(copy.deepcopy(raw_event)) filled_event_page = filler.fill_event_page(event_page, inplace=True) assert filled_event_page is event_page with pytest.warns(UserWarning): filler = event_model.Filler(reg) class OtherDummyHandler: "Same as DummyHandler, but a different object to test mutating reg" def __init__(self, resource_path, a, b): assert a == 1 assert b == 2 assert resource_path == str(tmp_path / "stack.tiff") def __call__(self, c, d): assert c == 3 assert d == 4 return numpy.ones((5, 5)) with event_model.Filler(reg, inplace=False) as filler: with pytest.raises(event_model.EventModelTypeError): # Updating an existing key fails. filler.handler_registry['DUMMY'] = OtherDummyHandler with pytest.raises(event_model.EventModelTypeError): # Setting a new key fails. filler.handler_registry['SOMETHING_ELSE'] = OtherDummyHandler with pytest.raises(event_model.EventModelTypeError): # Deleting a item fails. del filler.handler_registry['DUMMY'] filler('start', run_bundle.start_doc) filler('descriptor', desc_bundle.descriptor_doc) filler('descriptor', desc_bundle_baseline.descriptor_doc) filler('resource', res_bundle.resource_doc) filler('datum', datum_doc) event = copy.deepcopy(raw_event) name, filled_event = filler('event', event) assert filled_event is not event assert isinstance(event['data']['image'], str) # Now there should be a handler instance in the cache. assert filler._handler_cache # implementation detail with pytest.raises(event_model.DuplicateHandler): filler.register_handler('DUMMY', OtherDummyHandler) filler.register_handler('DUMMY', OtherDummyHandler, overwrite=True) assert filler.handler_registry['DUMMY'] is OtherDummyHandler # Replacing the handler for a given spec should clear the cache. assert not filler._handler_cache # implementation detail # Filling should work the same.... filler('start', run_bundle.start_doc) filler('descriptor', desc_bundle.descriptor_doc) filler('descriptor', desc_bundle_baseline.descriptor_doc) filler('resource', res_bundle.resource_doc) filler('datum', datum_doc) event = copy.deepcopy(raw_event) name, filled_event = filler('event', event) assert filled_event is not event assert isinstance(event['data']['image'], str) filler.deregister_handler('DUMMY') assert not filler.handler_registry assert not filler._handler_cache # implementation detail
def test_inplace(): "Test the behavior of the 'inplace' parameter." with event_model.Filler(reg, inplace=True) as filler: filler('start', run_bundle.start_doc) filler('descriptor', desc_bundle.descriptor_doc) filler('descriptor', desc_bundle_baseline.descriptor_doc) filler('resource', res_bundle.resource_doc) filler('datum', datum_doc) # Test event() event = copy.deepcopy(raw_event) name, filled_event = filler('event', event) assert filled_event is event event = copy.deepcopy(raw_event) # Test fill_event() filled_event = filler.fill_event(event) assert filled_event is event # Test event_page() event_page = event_model.pack_event_page(copy.deepcopy(raw_event)) _, filled_event_page = filler('event_page', event_page) assert filled_event_page is event_page # Test fill_event_page() event_page = event_model.pack_event_page(copy.deepcopy(raw_event)) filled_event_page = filler.fill_event_page(event_page) assert filled_event_page is event_page # Test fill_event and fill_event_page again with inplace=False. # Test fill_event() filled_event = filler.fill_event(event, inplace=False) assert filled_event is not event # Test fill_event_page() event_page = event_model.pack_event_page(copy.deepcopy(raw_event)) filled_event_page = filler.fill_event_page(event_page, inplace=False) assert filled_event_page is not event_page with event_model.Filler(reg, inplace=False) as filler: filler('start', run_bundle.start_doc) filler('descriptor', desc_bundle.descriptor_doc) filler('descriptor', desc_bundle_baseline.descriptor_doc) filler('resource', res_bundle.resource_doc) filler('datum', datum_doc) event = copy.deepcopy(raw_event) name, filled_event = filler('event', event) assert filled_event is not event assert isinstance(event['data']['image'], str) event = copy.deepcopy(raw_event) # Test fill_event() filled_event = filler.fill_event(event) assert filled_event is not event # Test event_page() event_page = event_model.pack_event_page(copy.deepcopy(raw_event)) _, filled_event_page = filler('event_page', event_page) assert filled_event_page is not event_page # Test fill_event_page() event_page = event_model.pack_event_page(copy.deepcopy(raw_event)) filled_event_page = filler.fill_event_page(event_page) assert filled_event_page is not event_page # Test fill_event and fill_event_page again with inplace=True. # Test fill_event() filled_event = filler.fill_event(event, inplace=True) assert filled_event is event # Test fill_event_page() event_page = event_model.pack_event_page(copy.deepcopy(raw_event)) filled_event_page = filler.fill_event_page(event_page, inplace=True) assert filled_event_page is event_page with pytest.warns(UserWarning): # warnings because inplace is not specified filler = event_model.Filler(reg)
def test_filler(tmp_path): class DummyHandler: def __init__(self, resource_path, a, b): assert a == 1 assert b == 2 assert resource_path == str(tmp_path / "stack.tiff") def __call__(self, c, d): assert c == 3 assert d == 4 return numpy.ones((5, 5)) path_root = str(tmp_path) reg = {'DUMMY': DummyHandler} filler = event_model.Filler(reg) run_bundle = event_model.compose_run() desc_bundle = run_bundle.compose_descriptor(data_keys={ 'motor': { 'shape': [], 'dtype': 'number', 'source': '...' }, 'image': { 'shape': [512, 512], 'dtype': 'number', 'source': '...', 'external': 'FILESTORE:' } }, name='primary') desc_bundle_baseline = run_bundle.compose_descriptor( data_keys={'motor': { 'shape': [], 'dtype': 'number', 'source': '...' }}, name='baseline') res_bundle = run_bundle.compose_resource(spec='DUMMY', root=path_root, resource_path='stack.tiff', resource_kwargs={ 'a': 1, 'b': 2 }) datum_doc = res_bundle.compose_datum(datum_kwargs={'c': 3, 'd': 4}) raw_event = desc_bundle.compose_event(data={ 'motor': 0, 'image': datum_doc['datum_id'] }, timestamps={ 'motor': 0, 'image': 0 }, filled={'image': False}, seq_num=1) filler('start', run_bundle.start_doc) filler('descriptor', desc_bundle.descriptor_doc) filler('descriptor', desc_bundle_baseline.descriptor_doc) filler('resource', res_bundle.resource_doc) filler('datum', datum_doc) event = copy.deepcopy(raw_event) assert isinstance(event['data']['image'], str) filler('event', event) stop_doc = run_bundle.compose_stop() filler('stop', stop_doc) assert event['data']['image'].shape == (5, 5) assert not filler._closed filler.close() assert filler._closed # Test context manager with Event. with event_model.Filler(reg) as filler: filler('start', run_bundle.start_doc) filler('descriptor', desc_bundle.descriptor_doc) filler('descriptor', desc_bundle_baseline.descriptor_doc) filler('resource', res_bundle.resource_doc) filler('datum', datum_doc) event = copy.deepcopy(raw_event) name, doc = filler('event', event) assert name == 'event' assert doc is event filler('stop', stop_doc) assert not filler._closed assert event['data']['image'].shape == (5, 5) assert filler._closed # Test context manager with EventPage. with event_model.Filler(reg) as filler: filler('start', run_bundle.start_doc) filler('descriptor', desc_bundle.descriptor_doc) filler('descriptor', desc_bundle_baseline.descriptor_doc) filler('resource', res_bundle.resource_doc) filler('datum', datum_doc) event_page = event_model.pack_event_page(copy.deepcopy(raw_event)) name, doc = filler('event_page', event_page) assert name == 'event_page' assert doc is event_page filler('stop', stop_doc) assert not filler._closed assert event_page['data']['image'][0].shape == (5, 5) assert filler._closed # Test undefined handler spec with event_model.Filler({}) as filler: filler('start', run_bundle.start_doc) filler('descriptor', desc_bundle.descriptor_doc) filler('descriptor', desc_bundle_baseline.descriptor_doc) filler('resource', res_bundle.resource_doc) filler('datum', datum_doc) event = copy.deepcopy(raw_event) assert isinstance(event['data']['image'], str) with pytest.raises(event_model.UndefinedAssetSpecification): filler('event', event) # Test exclude and include. with pytest.raises(ValueError): event_model.Filler({}, include=[], exclude=[]) with event_model.Filler(reg, exclude=['image']) as filler: filler('start', run_bundle.start_doc) filler('descriptor', desc_bundle.descriptor_doc) filler('descriptor', desc_bundle_baseline.descriptor_doc) filler('resource', res_bundle.resource_doc) filler('datum', datum_doc) event = copy.deepcopy(raw_event) assert isinstance(event['data']['image'], str) filler('event', event) filler('stop', stop_doc) with event_model.Filler(reg, include=['image']) as filler: filler('start', run_bundle.start_doc) filler('descriptor', desc_bundle.descriptor_doc) filler('descriptor', desc_bundle_baseline.descriptor_doc) filler('resource', res_bundle.resource_doc) filler('datum', datum_doc) event = copy.deepcopy(raw_event) filler('event', event) filler('stop', stop_doc) assert not filler._closed assert event['data']['image'].shape == (5, 5) with event_model.Filler(reg, include=['image', 'EXTRA THING']) as filler: filler('start', run_bundle.start_doc) filler('descriptor', desc_bundle.descriptor_doc) filler('descriptor', desc_bundle_baseline.descriptor_doc) filler('resource', res_bundle.resource_doc) filler('datum', datum_doc) event = copy.deepcopy(raw_event) filler('event', event) filler('stop', stop_doc) assert not filler._closed assert event['data']['image'].shape == (5, 5) class DummyHandlerRootMapTest: def __init__(self, resource_path, a, b): assert a == 1 assert b == 2 assert resource_path == str(tmp_path / "moved" / "stack.tiff") def __call__(self, c, d): assert c == 3 assert d == 4 return numpy.ones((5, 5)) with event_model.Filler({'DUMMY': DummyHandlerRootMapTest}, root_map={path_root: str(tmp_path / "moved")}) as filler: filler('start', run_bundle.start_doc) filler('descriptor', desc_bundle.descriptor_doc) filler('descriptor', desc_bundle_baseline.descriptor_doc) filler('resource', res_bundle.resource_doc) filler('datum', datum_doc) event = copy.deepcopy(raw_event) filler('event', event) filler('stop', stop_doc) assert not filler._closed assert event['data']['image'].shape == (5, 5) # Test verify_filled. with pytest.raises(event_model.UnfilledData): event_model.verify_filled(event_model.pack_event_page(raw_event)) event_model.verify_filled(event_model.pack_event_page(event))
def test_document_router_dispatch_event(): event_calls = [] # used for counting calls event_page_calls = [] # used for counting calls # example documents event1 = { 'data': { 'x': 1 }, 'timestamps': { 'x': 0. }, 'uid': 'placeholder X', 'descriptor': 'placeholder Y', 'time': 0., 'seq_num': 1 } event2 = { 'data': { 'x': 2 }, 'timestamps': { 'x': 1. }, 'uid': 'placeholder X', 'descriptor': 'placeholder Y', 'time': 1., 'seq_num': 2 } event_page = event_model.pack_event_page(event1, event2) def check(ret, original=None): name, doc = ret assert doc is not None assert doc is not NotImplemented if original is not None: # Verify that a copy is returned. assert doc is not original # ret is such a poser, dude. doc.pop('filled', None) original.pop('filled', None) assert doc == original class DefinesNeitherEventNorEventPage(event_model.DocumentRouter): def event(self, doc): event_calls.append(object()) # This returns NotImplemented. return super().event_page(doc) def event_page(self, doc): event_page_calls.append(object()) # This returns NotImplemented. return super().event_page(doc) dr = DefinesNeitherEventNorEventPage() # Test that Event is routed to Event and EventPage. check(dr('event', event1)) assert len(event_calls) == 1 assert len(event_page_calls) == 1 event_calls.clear() event_page_calls.clear() # Test that EventPage is routed to EventPage and Event *once* before # giving up. check(dr('event_page', event_page)) assert len(event_page_calls) == 1 assert len(event_calls) == 1 event_calls.clear() event_page_calls.clear() class DefinesEventNotEventPage(event_model.DocumentRouter): def event(self, doc): # Just a dumb test that check something particular to these example # documents. assert doc['data']['x'] == doc['seq_num'] event_calls.append(object()) return dict(doc) def event_page(self, doc): event_page_calls.append(object()) # This returns NotImplemented. return super().event_page(doc) dr = DefinesEventNotEventPage() # Test that Event is routed to Event. check(dr('event', event1), event1) assert len(event_calls) == 1 assert len(event_page_calls) == 0 event_calls.clear() event_page_calls.clear() # Test that EventPage is unpacked and routed to Event one at a time. check(dr('event_page', event_page), event_page) assert len(event_page_calls) == 1 assert len(event_calls) == 2 event_calls.clear() event_page_calls.clear() class DefinesEventPageNotEvent(event_model.DocumentRouter): def event(self, doc): event_calls.append(object()) # This returns NotImplemented. return super().event(doc) def event_page(self, doc): # Just a dumb test that check something particular to these example # documents. assert doc['data']['x'][0] == 1 event_page_calls.append(object()) return dict(doc) dr = DefinesEventPageNotEvent() # Test that Event is packed and routed to EventPage. check(dr('event', event1), event1) assert len(event_calls) == 1 assert len(event_page_calls) == 1 event_calls.clear() event_page_calls.clear() # Test that EventPage is routed to EventPage. check(dr('event_page', event_page), event_page) assert len(event_page_calls) == 1 assert len(event_calls) == 0 event_calls.clear() event_page_calls.clear() class DefinesEventPageAndEvent(event_model.DocumentRouter): def event(self, doc): # Just a dumb test that check something particular to these example # documents. assert doc['data']['x'] == doc['seq_num'] event_calls.append(object()) return dict(doc) def event_page(self, doc): # Just a dumb test that check something particular to these example # documents. assert doc['data']['x'][0] == 1 event_page_calls.append(object()) return dict(doc) dr = DefinesEventPageAndEvent() # Test that Event is routed to Event. check(dr('event', event1), event1) assert len(event_calls) == 1 assert len(event_page_calls) == 0 event_calls.clear() event_page_calls.clear() # Test that EventPage is routed to EventPage. check(dr('event_page', event_page), event_page) assert len(event_page_calls) == 1 assert len(event_calls) == 0 event_calls.clear() event_page_calls.clear()
def test_round_trip_pagination(): run_bundle = event_model.compose_run() desc_bundle = run_bundle.compose_descriptor(data_keys={ 'motor': { 'shape': [], 'dtype': 'number', 'source': '...' }, 'image': { 'shape': [512, 512], 'dtype': 'number', 'source': '...', 'external': 'FILESTORE:' } }, name='primary') res_bundle = run_bundle.compose_resource(spec='TIFF', root='/tmp', resource_path='stack.tiff', resource_kwargs={}) datum_doc1 = res_bundle.compose_datum(datum_kwargs={'slice': 5}) datum_doc2 = res_bundle.compose_datum(datum_kwargs={'slice': 10}) datum_doc3 = res_bundle.compose_datum(datum_kwargs={'slice': 15}) event_doc1 = desc_bundle.compose_event(data={ 'motor': 0, 'image': datum_doc1['datum_id'] }, timestamps={ 'motor': 0, 'image': 0 }, filled={'image': False}, seq_num=1) event_doc2 = desc_bundle.compose_event(data={ 'motor': 1, 'image': datum_doc2['datum_id'] }, timestamps={ 'motor': 0, 'image': 0 }, filled={'image': False}, seq_num=1) event_doc3 = desc_bundle.compose_event(data={ 'motor': 2, 'image': datum_doc3['datum_id'] }, timestamps={ 'motor': 0, 'image': 0 }, filled={'image': False}, seq_num=1) # Round trip single event -> event_page -> event. expected = event_doc1 actual, = event_model.unpack_event_page( event_model.pack_event_page(expected)) assert actual == expected # Round trip two events -> event_page -> events. expected = [event_doc1, event_doc2] actual = list( event_model.unpack_event_page(event_model.pack_event_page(*expected))) assert actual == expected # Round trip three events -> event_page -> events. expected = [event_doc1, event_doc2, event_doc3] actual = list( event_model.unpack_event_page(event_model.pack_event_page(*expected))) assert actual == expected # Round trip on docs that don't have a filled key unfilled_doc1 = event_doc1 unfilled_doc1.pop('filled') unfilled_doc2 = event_doc2 unfilled_doc2.pop('filled') unfilled_doc3 = event_doc3 unfilled_doc3.pop('filled') expected = [unfilled_doc1, unfilled_doc2, unfilled_doc3] actual = list( event_model.unpack_event_page(event_model.pack_event_page(*expected))) for doc in actual: doc.pop('filled') assert actual == expected # Round trip one datum -> datum_page -> datum. expected = datum_doc1 actual, = event_model.unpack_datum_page( event_model.pack_datum_page(expected)) assert actual == expected # Round trip two datum -> datum_page -> datum. expected = [datum_doc1, datum_doc2] actual = list( event_model.unpack_datum_page(event_model.pack_datum_page(*expected))) assert actual == expected # Round trip three datum -> datum_page -> datum. expected = [datum_doc1, datum_doc2, datum_doc3] actual = list( event_model.unpack_datum_page(event_model.pack_datum_page(*expected))) assert actual == expected # Check edge case where datum_kwargs are empty. datum_doc1 = res_bundle.compose_datum(datum_kwargs={}) datum_doc2 = res_bundle.compose_datum(datum_kwargs={}) datum_doc3 = res_bundle.compose_datum(datum_kwargs={}) # Round trip one datum -> datum_page -> datum. expected = datum_doc1 actual, = event_model.unpack_datum_page( event_model.pack_datum_page(expected)) assert actual == expected # Round trip two datum -> datum_page -> datum. expected = [datum_doc1, datum_doc2] actual = list( event_model.unpack_datum_page(event_model.pack_datum_page(*expected))) assert actual == expected # Round trip three datum -> datum_page -> datum. expected = [datum_doc1, datum_doc2, datum_doc3] actual = list( event_model.unpack_datum_page(event_model.pack_datum_page(*expected))) assert actual == expected
def test_pack_empty_raises(): with pytest.raises(ValueError): event_model.pack_event_page() with pytest.raises(ValueError): event_model.pack_datum_page()
def export(gen, filepath, **kwargs): """ Export a stream of documents to CSV file(s) and one JSON file of metadata. Creates {filepath}_meta.json and then {filepath}_{stream_name}.csv for every Event stream. The structure of the json is:: {'start': {...}, 'descriptors': {'<stream_name>': [{...}, {...}, ...], ...}, 'stop': {...}} Parameters ---------- gen : generator expected to yield (name, document) pairs filepath : str the filepath and filename suffix to use in the output files. **kwargs : kwargs kwargs to be passed to pandas.Dataframe.to_csv. Returns ------- dest : tuple filepaths of generated files """ meta = {} # to be exported as JSON at the end meta['descriptors'] = defaultdict(list) # map stream_name to descriptors files = {} # map descriptor uid to file handle of CSV file desc_counters = defaultdict(itertools.count) has_header = set() # a set of uids indicating if the file has a header kwargs.setdefault('header', True) initial_header_kwarg = kwargs['header'] # used later to set the headers kwargs.setdefault('index_label', 'time') kwargs.setdefault('mode', 'a') try: for name, doc in gen: if name == 'start': if 'start' in meta: raise RuntimeError("This exporter expects documents from " "one run only.") meta['start'] = doc elif name == 'stop': meta['stop'] = doc elif name == 'descriptor': stream_name = doc.get('name') meta['descriptors'][stream_name].append(doc) filepath_ = (f"{filepath}_{stream_name}_" f"{next(desc_counters[doc['uid']])}.csv") files[doc['uid']] = open(filepath_, 'w+') elif (name == 'event' or name == 'bulk_event' or name == 'event_page'): if name == 'event': # convert event to an event_pages list event_pages = [event_model.pack_event_page(doc)] elif name == 'bulk_event': # convert bulk_event to event_pages event_pages = event_model.bulk_events_to_event_pages(doc) else: # convert an event_page to an event_pages list. event_pages = [doc] for event_page in event_pages: if not all(event_page['filled'].values()): # check that all event_page data is filled unfilled_data = [] for field in event_page['filled']: if not event_page['filled'][field]: unfilled_data.append(field) # Note: As of this writing, this is a slightly # aspirational error message, as event_model.Filler has # not been merged yet. May need to be revisited if it # is renamed or kept elsewhere in the end. raise UnfilledData('unfilled data found in' '{}. Try passing the parameter ' '"gen" through "event_model.Filler"' ' first'.format(unfilled_data)) else: event_data = pandas.DataFrame(event_page['data'], index=event_page['time']) event_data['seq_num'] = event_page['seq_num'] if initial_header_kwarg: kwargs['header'] = event_page['descriptor'] \ not in has_header event_data.to_csv(files[event_page['descriptor']], **kwargs) has_header.add(event_page['descriptor']) finally: for f in files.values(): f.close() with open(f"{filepath}_meta.json", 'w') as f: json.dump(meta, f) return (f.name, ) + tuple(f.name for f in files.values())