def test_bulk_events_to_event_page():
run_bundle = event_model.compose_run()
desc_bundle = run_bundle.compose_descriptor(
data_keys={'motor': {'shape': [], 'dtype': 'number', 'source': '...'},
'image': {'shape': [512, 512], 'dtype': 'number',
'source': '...', 'external': 'FILESTORE:'}},
name='primary')
desc_bundle_baseline = run_bundle.compose_descriptor(
data_keys={'motor': {'shape': [], 'dtype': 'number', 'source': '...'}},
name='baseline')
res_bundle = run_bundle.compose_resource(
spec='TIFF', root='/tmp', resource_path='stack.tiff',
resource_kwargs={})
datum_doc1 = res_bundle.compose_datum(datum_kwargs={'slice': 5})
datum_doc2 = res_bundle.compose_datum(datum_kwargs={'slice': 10})
event1 = desc_bundle.compose_event(
data={'motor': 0, 'image': datum_doc1['datum_id']},
timestamps={'motor': 0, 'image': 0}, filled={'image': False},
seq_num=1)
event2 = desc_bundle.compose_event(
data={'motor': 0, 'image': datum_doc2['datum_id']},
timestamps={'motor': 0, 'image': 0}, filled={'image': False},
seq_num=2)
event3 = desc_bundle_baseline.compose_event(
data={'motor': 0},
timestamps={'motor': 0},
seq_num=1)
primary_event_page = event_model.pack_event_page(event1, event2)
baseline_event_page = event_model.pack_event_page(event3)
bulk_events = {'primary': [event1, event2], 'baseline': [event3]}
pages = event_model.bulk_events_to_event_pages(bulk_events)
assert tuple(pages) == (primary_event_page, baseline_event_page)
def test_verify_filled(filler):
"Test the utility function verify_filled."
with pytest.raises(event_model.UnfilledData):
event_model.verify_filled(event_model.pack_event_page(raw_event))
event = copy.deepcopy(raw_event)
name, doc = filler('event', event)
event_model.verify_filled(event_model.pack_event_page(event))
def test_fill_event_page(RE, hw):
docs = []
def callback(name, doc):
docs.append((name, doc))
RE(count([hw.img]), callback)
docs
dask_filler = Filler({'NPY_SEQ': NumpySeqHandler},
coerce='delayed',
inplace=False)
filled_docs = []
_, event = docs[-2]
event_page = event_model.pack_event_page(event)
docs[-2] = ('event_page', event_page)
dask_filler = Filler({'NPY_SEQ': NumpySeqHandler},
coerce='delayed',
inplace=False)
filled_docs = []
for name, doc in docs:
filled_docs.append(dask_filler(name, doc))
_, dask_filled_event_page = filled_docs[-2]
arr = dask_filled_event_page['data']['img'][0].compute()
assert arr.shape == (10, 10)
assert isinstance(arr, numpy.ndarray)
def test_sanitize_doc():
run_bundle = event_model.compose_run()
desc_bundle = run_bundle.compose_descriptor(
data_keys={'motor': {'shape': [], 'dtype': 'number', 'source': '...'},
'image': {'shape': [512, 512], 'dtype': 'number',
'source': '...', 'external': 'FILESTORE:'}},
name='primary')
desc_bundle_baseline = run_bundle.compose_descriptor(
data_keys={'motor': {'shape': [], 'dtype': 'number', 'source': '...'}},
name='baseline')
event1 = desc_bundle.compose_event(
data={'motor': 0, 'image': numpy.ones((512, 512))},
timestamps={'motor': 0, 'image': 0}, filled={'image': True},
seq_num=1)
event2 = desc_bundle.compose_event(
data={'motor': 0, 'image': numpy.ones((512, 512))},
timestamps={'motor': 0, 'image': 0}, filled={'image': True},
seq_num=2)
event3 = desc_bundle_baseline.compose_event(
data={'motor': 0},
timestamps={'motor': 0},
seq_num=1)
event_page = event_model.pack_event_page(event1, event2)
bulk_events = {'primary': [event1, event2], 'baseline': [event3]}
json.dumps(event_model.sanitize_doc(event_page))
json.dumps(event_model.sanitize_doc(bulk_events))
json.dumps(event_model.sanitize_doc(event1))
def get_event_pages(*args, **kwargs):
event_cursor = get_event_cursor(*args, **kwargs)
while True:
result = list(itertools.islice(event_cursor, page_size))
if result:
yield event_model.pack_event_page(*result)
else:
break
def collect(name, doc):
if name == 'event':
events.append(doc)
elif name == 'stop':
collector.append(
('event_page', event_model.pack_event_page(*events)))
collector.append((name, doc))
else:
collector.append((name, doc))
def event_page(self, doc):
event = self.event # Avoid attribute lookup in hot loop.
filled_events = []
for event_doc in event_model.unpack_event_page(doc):
filled_events.append(event(event_doc))
new_event_page = event_model.pack_event_page(*filled_events)
# Modify original doc in place, as we do with 'event'.
doc['data'] = new_event_page['data']
return doc
def event(self, doc):
'''Add event document information to a ".tiff" file.
This method adds event document information to a ".tiff" file,
creating it if necessary.
.. warning::
All non 2D 'image-like' data is explicitly ignored.
.. note::
The data in Events might be structured as an Event, an EventPage,
or a "bulk event" (deprecated). The DocumentRouter base class takes
care of first transforming the other representations into an
EventPage and then routing them through here, as we require Event
documents _in this case_ we overwrite both the `event` method and
the `event_page` method so we can assume we will always receive an
Event.
Parameters:
-----------
doc : dict
Event document
'''
event_model.verify_filled(event_model.pack_event_page(*[doc]))
descriptor = self._descriptors[doc['descriptor']]
stream_name = descriptor.get('name')
for field in doc['data']:
img = doc['data'][field]
# Check that the data is 2D or 3D; if not ignore it.
data_key = descriptor['data_keys'][field]
ndim = len(data_key['shape'] or [])
if data_key['dtype'] == 'array' and 1 < ndim < 4:
img_asarray = numpy.asarray(img, dtype=self._astype)
if ndim == 2:
# handle 2D data just like 3D data
# by adding a 3rd dimension
img_asarray = numpy.expand_dims(img_asarray, axis=0)
for i in range(img_asarray.shape[0]):
img_asarray_2d = img_asarray[i, :]
num = next(self._counter[stream_name][field])
filename = get_prefixed_filename(
file_prefix=self._file_prefix,
start_doc=self._start,
descriptor_doc=descriptor,
event_doc=doc,
num=num,
stream_name=stream_name,
field=field)
file = self._manager.open('stream_data', filename, 'xb')
tw = TiffWriter(file, **self._init_kwargs)
self._tiff_writers[stream_name][field + f'-{num}'] = tw
tw.save(img_asarray_2d, *self._kwargs)
def test_context_manager_with_event_page():
with event_model.Filler(reg, inplace=True) as filler:
filler('start', run_bundle.start_doc)
filler('descriptor', desc_bundle.descriptor_doc)
filler('descriptor', desc_bundle_baseline.descriptor_doc)
filler('resource', res_bundle.resource_doc)
filler('datum', datum_doc)
event_page = event_model.pack_event_page(copy.deepcopy(raw_event))
name, doc = filler('event_page', event_page)
assert name == 'event_page'
assert doc is event_page
filler('stop', stop_doc)
assert not filler.closed
assert event_page['data']['image'][0].shape == (5, 5)
assert filler.closed
def test_round_trip_event_page_with_empty_data():
event_page = {
'time': [1, 2, 3],
'seq_num': [1, 2, 3],
'uid': ['a', 'b', 'c'],
'descriptor': 'd',
'data': {},
'timestamps': {},
'filled': {}
}
events = list(event_model.unpack_event_page(event_page))
assert len(events) == 3
page_again = event_model.pack_event_page(*events)
assert page_again == event_page
def event(self, doc):
'''Add event document information to a ".tiff" file.
This method adds event document information to a ".tiff" file,
creating it if necessary.
.. warning::
All non 2D 'image-like' data is explicitly ignored.
.. note::
The data in Events might be structured as an Event, an EventPage,
or a "bulk event" (deprecated). The DocumentRouter base class takes
care of first transforming the other repsentations into an
EventPage and then routing them through here, as we require Event
documents _in this case_ we overwrite both the `event` method and
the `event_page` method so we can assume we will always receive an
Event.
Parameters:
-----------
doc : dict
Event document
'''
event_model.verify_filled(event_model.pack_event_page(*[doc]))
descriptor = self._descriptors[doc['descriptor']]
streamname = descriptor.get('name')
for field in doc['data']:
img = doc['data'][field]
# check that the data is 2D, if not ignore it
img_asarray = numpy.asarray(img, dtype=self._astype)
if img_asarray.ndim == 2:
# template the file name.
self._templated_file_prefix = self._file_prefix.format(
start=self._start, descriptor=descriptor, event=doc)
num = next(self._counter[streamname][field])
filename = (f'{self._templated_file_prefix}'
f'{streamname}-{field}-{num}.tiff')
file = self._manager.open('stream_data', filename, 'xb')
tw = TiffWriter(file, **self._init_kwargs)
self._tiff_writers[streamname][field + f'-{num}'] = tw
tw.save(img_asarray, *self._kwargs)
def test_filler(tmp_path):
class DummyHandler:
def __init__(self, resource_path, a, b):
assert a == 1
assert b == 2
assert resource_path == str(tmp_path / "stack.tiff")
def __call__(self, c, d):
assert c == 3
assert d == 4
return numpy.ones((5, 5))
path_root = str(tmp_path)
reg = {'DUMMY': DummyHandler}
filler = event_model.Filler(reg, inplace=True)
run_bundle = event_model.compose_run()
desc_bundle = run_bundle.compose_descriptor(data_keys={
'motor': {
'shape': [],
'dtype': 'number',
'source': '...'
},
'image': {
'shape': [512, 512],
'dtype': 'number',
'source': '...',
'external': 'FILESTORE:'
}
},
name='primary')
desc_bundle_baseline = run_bundle.compose_descriptor(
data_keys={'motor': {
'shape': [],
'dtype': 'number',
'source': '...'
}},
name='baseline')
res_bundle = run_bundle.compose_resource(spec='DUMMY',
root=path_root,
resource_path='stack.tiff',
resource_kwargs={
'a': 1,
'b': 2
})
datum_doc = res_bundle.compose_datum(datum_kwargs={'c': 3, 'd': 4})
raw_event = desc_bundle.compose_event(data={
'motor': 0,
'image': datum_doc['datum_id']
},
timestamps={
'motor': 0,
'image': 0
},
filled={'image': False},
seq_num=1)
filler('start', run_bundle.start_doc)
filler('descriptor', desc_bundle.descriptor_doc)
filler('descriptor', desc_bundle_baseline.descriptor_doc)
filler('resource', res_bundle.resource_doc)
filler('datum', datum_doc)
event = copy.deepcopy(raw_event)
assert isinstance(event['data']['image'], str)
filler('event', event)
stop_doc = run_bundle.compose_stop()
filler('stop', stop_doc)
assert event['data']['image'].shape == (5, 5)
assert not filler._closed
# Test NoFiller.
filler = event_model.NoFiller(reg)
filler('start', run_bundle.start_doc)
filler('descriptor', desc_bundle.descriptor_doc)
filler('descriptor', desc_bundle_baseline.descriptor_doc)
filler('resource', res_bundle.resource_doc)
filler('datum', datum_doc)
event = copy.deepcopy(raw_event)
assert isinstance(event['data']['image'], str)
filler('event', event)
# Check that it *hasn't* been filled.
assert isinstance(event['data']['image'], str)
filler('stop', stop_doc)
# Test get_handler() method.
handler = filler.get_handler(res_bundle.resource_doc)
# The method does not expose the internal cache of handlers, so it should
# not return the same instance when called repeatedly.
assert filler.get_handler(res_bundle.resource_doc) is not handler
# Test closing.
filler.close()
with pytest.raises(event_model.EventModelRuntimeError):
filler.get_handler(res_bundle.resource_doc)
with pytest.raises(event_model.EventModelRuntimeError):
filler('stop', stop_doc)
# Test context manager with Event.
with event_model.Filler(reg, inplace=True) as filler:
filler('start', run_bundle.start_doc)
filler('descriptor', desc_bundle.descriptor_doc)
filler('descriptor', desc_bundle_baseline.descriptor_doc)
filler('resource', res_bundle.resource_doc)
filler('datum', datum_doc)
event = copy.deepcopy(raw_event)
name, doc = filler('event', event)
assert name == 'event'
assert doc is event
filler('stop', stop_doc)
assert not filler._closed
assert event['data']['image'].shape == (5, 5)
assert filler._closed
# Test context manager with EventPage.
with event_model.Filler(reg, inplace=True) as filler:
filler('start', run_bundle.start_doc)
filler('descriptor', desc_bundle.descriptor_doc)
filler('descriptor', desc_bundle_baseline.descriptor_doc)
filler('resource', res_bundle.resource_doc)
filler('datum', datum_doc)
event_page = event_model.pack_event_page(copy.deepcopy(raw_event))
name, doc = filler('event_page', event_page)
assert name == 'event_page'
assert doc is event_page
filler('stop', stop_doc)
assert not filler._closed
assert event_page['data']['image'][0].shape == (5, 5)
assert filler._closed
# Test undefined handler spec
with event_model.Filler({}, inplace=True) as filler:
filler('start', run_bundle.start_doc)
filler('descriptor', desc_bundle.descriptor_doc)
filler('descriptor', desc_bundle_baseline.descriptor_doc)
filler('resource', res_bundle.resource_doc)
filler('datum', datum_doc)
event = copy.deepcopy(raw_event)
assert isinstance(event['data']['image'], str)
with pytest.raises(event_model.UndefinedAssetSpecification):
filler('event', event)
# Test exclude and include.
with pytest.raises(ValueError):
event_model.Filler({}, include=[], exclude=[], inplace=True)
with pytest.warns(DeprecationWarning):
with event_model.Filler(reg, exclude=['image'],
inplace=True) as filler:
filler('start', run_bundle.start_doc)
filler('descriptor', desc_bundle.descriptor_doc)
filler('descriptor', desc_bundle_baseline.descriptor_doc)
filler('resource', res_bundle.resource_doc)
filler('datum', datum_doc)
event = copy.deepcopy(raw_event)
assert isinstance(event['data']['image'], str)
filler('event', event)
filler('stop', stop_doc)
with pytest.warns(DeprecationWarning):
with event_model.Filler(reg, include=['image'],
inplace=True) as filler:
filler('start', run_bundle.start_doc)
filler('descriptor', desc_bundle.descriptor_doc)
filler('descriptor', desc_bundle_baseline.descriptor_doc)
filler('resource', res_bundle.resource_doc)
filler('datum', datum_doc)
event = copy.deepcopy(raw_event)
filler('event', event)
filler('stop', stop_doc)
assert not filler._closed
assert event['data']['image'].shape == (5, 5)
with pytest.warns(DeprecationWarning):
with event_model.Filler(reg,
include=['image', 'EXTRA THING'],
inplace=True) as filler:
filler('start', run_bundle.start_doc)
filler('descriptor', desc_bundle.descriptor_doc)
filler('descriptor', desc_bundle_baseline.descriptor_doc)
filler('resource', res_bundle.resource_doc)
filler('datum', datum_doc)
event = copy.deepcopy(raw_event)
filler('event', event)
filler('stop', stop_doc)
assert not filler._closed
assert event['data']['image'].shape == (5, 5)
class DummyHandlerRootMapTest:
def __init__(self, resource_path, a, b):
assert a == 1
assert b == 2
assert resource_path == str(tmp_path / "moved" / "stack.tiff")
def __call__(self, c, d):
assert c == 3
assert d == 4
return numpy.ones((5, 5))
with event_model.Filler({'DUMMY': DummyHandlerRootMapTest},
root_map={path_root: str(tmp_path / "moved")},
inplace=True) as filler:
filler('start', run_bundle.start_doc)
filler('descriptor', desc_bundle.descriptor_doc)
filler('descriptor', desc_bundle_baseline.descriptor_doc)
filler('resource', res_bundle.resource_doc)
filler('datum', datum_doc)
event = copy.deepcopy(raw_event)
filler('event', event)
filler('stop', stop_doc)
assert not filler._closed
assert event['data']['image'].shape == (5, 5)
# Test verify_filled.
with pytest.raises(event_model.UnfilledData):
event_model.verify_filled(event_model.pack_event_page(raw_event))
event_model.verify_filled(event_model.pack_event_page(event))
# Test inplace.
with event_model.Filler(reg, inplace=True) as filler:
filler('start', run_bundle.start_doc)
filler('descriptor', desc_bundle.descriptor_doc)
filler('descriptor', desc_bundle_baseline.descriptor_doc)
filler('resource', res_bundle.resource_doc)
filler('datum', datum_doc)
# Test event()
event = copy.deepcopy(raw_event)
name, filled_event = filler('event', event)
assert filled_event is event
event = copy.deepcopy(raw_event)
# Test fill_event()
filled_event = filler.fill_event(event)
assert filled_event is event
# Test event_page()
event_page = event_model.pack_event_page(copy.deepcopy(raw_event))
_, filled_event_page = filler('event_page', event_page)
assert filled_event_page is event_page
# Test fill_event_page()
event_page = event_model.pack_event_page(copy.deepcopy(raw_event))
filled_event_page = filler.fill_event_page(event_page)
assert filled_event_page is event_page
# Test fill_event and fill_event_page again with inplace=False.
# Test fill_event()
filled_event = filler.fill_event(event, inplace=False)
assert filled_event is not event
# Test fill_event_page()
event_page = event_model.pack_event_page(copy.deepcopy(raw_event))
filled_event_page = filler.fill_event_page(event_page, inplace=False)
assert filled_event_page is not event_page
with event_model.Filler(reg, inplace=False) as filler:
filler('start', run_bundle.start_doc)
filler('descriptor', desc_bundle.descriptor_doc)
filler('descriptor', desc_bundle_baseline.descriptor_doc)
filler('resource', res_bundle.resource_doc)
filler('datum', datum_doc)
event = copy.deepcopy(raw_event)
name, filled_event = filler('event', event)
assert filled_event is not event
assert isinstance(event['data']['image'], str)
event = copy.deepcopy(raw_event)
# Test fill_event()
filled_event = filler.fill_event(event)
assert filled_event is not event
# Test event_page()
event_page = event_model.pack_event_page(copy.deepcopy(raw_event))
_, filled_event_page = filler('event_page', event_page)
assert filled_event_page is not event_page
# Test fill_event_page()
event_page = event_model.pack_event_page(copy.deepcopy(raw_event))
filled_event_page = filler.fill_event_page(event_page)
assert filled_event_page is not event_page
# Test fill_event and fill_event_page again with inplace=True.
# Test fill_event()
filled_event = filler.fill_event(event, inplace=True)
assert filled_event is event
# Test fill_event_page()
event_page = event_model.pack_event_page(copy.deepcopy(raw_event))
filled_event_page = filler.fill_event_page(event_page, inplace=True)
assert filled_event_page is event_page
with pytest.warns(UserWarning):
filler = event_model.Filler(reg)
class OtherDummyHandler:
"Same as DummyHandler, but a different object to test mutating reg"
def __init__(self, resource_path, a, b):
assert a == 1
assert b == 2
assert resource_path == str(tmp_path / "stack.tiff")
def __call__(self, c, d):
assert c == 3
assert d == 4
return numpy.ones((5, 5))
with event_model.Filler(reg, inplace=False) as filler:
with pytest.raises(event_model.EventModelTypeError):
# Updating an existing key fails.
filler.handler_registry['DUMMY'] = OtherDummyHandler
with pytest.raises(event_model.EventModelTypeError):
# Setting a new key fails.
filler.handler_registry['SOMETHING_ELSE'] = OtherDummyHandler
with pytest.raises(event_model.EventModelTypeError):
# Deleting a item fails.
del filler.handler_registry['DUMMY']
filler('start', run_bundle.start_doc)
filler('descriptor', desc_bundle.descriptor_doc)
filler('descriptor', desc_bundle_baseline.descriptor_doc)
filler('resource', res_bundle.resource_doc)
filler('datum', datum_doc)
event = copy.deepcopy(raw_event)
name, filled_event = filler('event', event)
assert filled_event is not event
assert isinstance(event['data']['image'], str)
# Now there should be a handler instance in the cache.
assert filler._handler_cache # implementation detail
with pytest.raises(event_model.DuplicateHandler):
filler.register_handler('DUMMY', OtherDummyHandler)
filler.register_handler('DUMMY', OtherDummyHandler, overwrite=True)
assert filler.handler_registry['DUMMY'] is OtherDummyHandler
# Replacing the handler for a given spec should clear the cache.
assert not filler._handler_cache # implementation detail
# Filling should work the same....
filler('start', run_bundle.start_doc)
filler('descriptor', desc_bundle.descriptor_doc)
filler('descriptor', desc_bundle_baseline.descriptor_doc)
filler('resource', res_bundle.resource_doc)
filler('datum', datum_doc)
event = copy.deepcopy(raw_event)
name, filled_event = filler('event', event)
assert filled_event is not event
assert isinstance(event['data']['image'], str)
filler.deregister_handler('DUMMY')
assert not filler.handler_registry
assert not filler._handler_cache # implementation detail
def test_inplace():
"Test the behavior of the 'inplace' parameter."
with event_model.Filler(reg, inplace=True) as filler:
filler('start', run_bundle.start_doc)
filler('descriptor', desc_bundle.descriptor_doc)
filler('descriptor', desc_bundle_baseline.descriptor_doc)
filler('resource', res_bundle.resource_doc)
filler('datum', datum_doc)
# Test event()
event = copy.deepcopy(raw_event)
name, filled_event = filler('event', event)
assert filled_event is event
event = copy.deepcopy(raw_event)
# Test fill_event()
filled_event = filler.fill_event(event)
assert filled_event is event
# Test event_page()
event_page = event_model.pack_event_page(copy.deepcopy(raw_event))
_, filled_event_page = filler('event_page', event_page)
assert filled_event_page is event_page
# Test fill_event_page()
event_page = event_model.pack_event_page(copy.deepcopy(raw_event))
filled_event_page = filler.fill_event_page(event_page)
assert filled_event_page is event_page
# Test fill_event and fill_event_page again with inplace=False.
# Test fill_event()
filled_event = filler.fill_event(event, inplace=False)
assert filled_event is not event
# Test fill_event_page()
event_page = event_model.pack_event_page(copy.deepcopy(raw_event))
filled_event_page = filler.fill_event_page(event_page, inplace=False)
assert filled_event_page is not event_page
with event_model.Filler(reg, inplace=False) as filler:
filler('start', run_bundle.start_doc)
filler('descriptor', desc_bundle.descriptor_doc)
filler('descriptor', desc_bundle_baseline.descriptor_doc)
filler('resource', res_bundle.resource_doc)
filler('datum', datum_doc)
event = copy.deepcopy(raw_event)
name, filled_event = filler('event', event)
assert filled_event is not event
assert isinstance(event['data']['image'], str)
event = copy.deepcopy(raw_event)
# Test fill_event()
filled_event = filler.fill_event(event)
assert filled_event is not event
# Test event_page()
event_page = event_model.pack_event_page(copy.deepcopy(raw_event))
_, filled_event_page = filler('event_page', event_page)
assert filled_event_page is not event_page
# Test fill_event_page()
event_page = event_model.pack_event_page(copy.deepcopy(raw_event))
filled_event_page = filler.fill_event_page(event_page)
assert filled_event_page is not event_page
# Test fill_event and fill_event_page again with inplace=True.
# Test fill_event()
filled_event = filler.fill_event(event, inplace=True)
assert filled_event is event
# Test fill_event_page()
event_page = event_model.pack_event_page(copy.deepcopy(raw_event))
filled_event_page = filler.fill_event_page(event_page, inplace=True)
assert filled_event_page is event_page
with pytest.warns(UserWarning):
# warnings because inplace is not specified
filler = event_model.Filler(reg)
def test_filler(tmp_path):
class DummyHandler:
def __init__(self, resource_path, a, b):
assert a == 1
assert b == 2
assert resource_path == str(tmp_path / "stack.tiff")
def __call__(self, c, d):
assert c == 3
assert d == 4
return numpy.ones((5, 5))
path_root = str(tmp_path)
reg = {'DUMMY': DummyHandler}
filler = event_model.Filler(reg)
run_bundle = event_model.compose_run()
desc_bundle = run_bundle.compose_descriptor(data_keys={
'motor': {
'shape': [],
'dtype': 'number',
'source': '...'
},
'image': {
'shape': [512, 512],
'dtype': 'number',
'source': '...',
'external': 'FILESTORE:'
}
},
name='primary')
desc_bundle_baseline = run_bundle.compose_descriptor(
data_keys={'motor': {
'shape': [],
'dtype': 'number',
'source': '...'
}},
name='baseline')
res_bundle = run_bundle.compose_resource(spec='DUMMY',
root=path_root,
resource_path='stack.tiff',
resource_kwargs={
'a': 1,
'b': 2
})
datum_doc = res_bundle.compose_datum(datum_kwargs={'c': 3, 'd': 4})
raw_event = desc_bundle.compose_event(data={
'motor': 0,
'image': datum_doc['datum_id']
},
timestamps={
'motor': 0,
'image': 0
},
filled={'image': False},
seq_num=1)
filler('start', run_bundle.start_doc)
filler('descriptor', desc_bundle.descriptor_doc)
filler('descriptor', desc_bundle_baseline.descriptor_doc)
filler('resource', res_bundle.resource_doc)
filler('datum', datum_doc)
event = copy.deepcopy(raw_event)
assert isinstance(event['data']['image'], str)
filler('event', event)
stop_doc = run_bundle.compose_stop()
filler('stop', stop_doc)
assert event['data']['image'].shape == (5, 5)
assert not filler._closed
filler.close()
assert filler._closed
# Test context manager with Event.
with event_model.Filler(reg) as filler:
filler('start', run_bundle.start_doc)
filler('descriptor', desc_bundle.descriptor_doc)
filler('descriptor', desc_bundle_baseline.descriptor_doc)
filler('resource', res_bundle.resource_doc)
filler('datum', datum_doc)
event = copy.deepcopy(raw_event)
name, doc = filler('event', event)
assert name == 'event'
assert doc is event
filler('stop', stop_doc)
assert not filler._closed
assert event['data']['image'].shape == (5, 5)
assert filler._closed
# Test context manager with EventPage.
with event_model.Filler(reg) as filler:
filler('start', run_bundle.start_doc)
filler('descriptor', desc_bundle.descriptor_doc)
filler('descriptor', desc_bundle_baseline.descriptor_doc)
filler('resource', res_bundle.resource_doc)
filler('datum', datum_doc)
event_page = event_model.pack_event_page(copy.deepcopy(raw_event))
name, doc = filler('event_page', event_page)
assert name == 'event_page'
assert doc is event_page
filler('stop', stop_doc)
assert not filler._closed
assert event_page['data']['image'][0].shape == (5, 5)
assert filler._closed
# Test undefined handler spec
with event_model.Filler({}) as filler:
filler('start', run_bundle.start_doc)
filler('descriptor', desc_bundle.descriptor_doc)
filler('descriptor', desc_bundle_baseline.descriptor_doc)
filler('resource', res_bundle.resource_doc)
filler('datum', datum_doc)
event = copy.deepcopy(raw_event)
assert isinstance(event['data']['image'], str)
with pytest.raises(event_model.UndefinedAssetSpecification):
filler('event', event)
# Test exclude and include.
with pytest.raises(ValueError):
event_model.Filler({}, include=[], exclude=[])
with event_model.Filler(reg, exclude=['image']) as filler:
filler('start', run_bundle.start_doc)
filler('descriptor', desc_bundle.descriptor_doc)
filler('descriptor', desc_bundle_baseline.descriptor_doc)
filler('resource', res_bundle.resource_doc)
filler('datum', datum_doc)
event = copy.deepcopy(raw_event)
assert isinstance(event['data']['image'], str)
filler('event', event)
filler('stop', stop_doc)
with event_model.Filler(reg, include=['image']) as filler:
filler('start', run_bundle.start_doc)
filler('descriptor', desc_bundle.descriptor_doc)
filler('descriptor', desc_bundle_baseline.descriptor_doc)
filler('resource', res_bundle.resource_doc)
filler('datum', datum_doc)
event = copy.deepcopy(raw_event)
filler('event', event)
filler('stop', stop_doc)
assert not filler._closed
assert event['data']['image'].shape == (5, 5)
with event_model.Filler(reg, include=['image', 'EXTRA THING']) as filler:
filler('start', run_bundle.start_doc)
filler('descriptor', desc_bundle.descriptor_doc)
filler('descriptor', desc_bundle_baseline.descriptor_doc)
filler('resource', res_bundle.resource_doc)
filler('datum', datum_doc)
event = copy.deepcopy(raw_event)
filler('event', event)
filler('stop', stop_doc)
assert not filler._closed
assert event['data']['image'].shape == (5, 5)
class DummyHandlerRootMapTest:
def __init__(self, resource_path, a, b):
assert a == 1
assert b == 2
assert resource_path == str(tmp_path / "moved" / "stack.tiff")
def __call__(self, c, d):
assert c == 3
assert d == 4
return numpy.ones((5, 5))
with event_model.Filler({'DUMMY': DummyHandlerRootMapTest},
root_map={path_root:
str(tmp_path / "moved")}) as filler:
filler('start', run_bundle.start_doc)
filler('descriptor', desc_bundle.descriptor_doc)
filler('descriptor', desc_bundle_baseline.descriptor_doc)
filler('resource', res_bundle.resource_doc)
filler('datum', datum_doc)
event = copy.deepcopy(raw_event)
filler('event', event)
filler('stop', stop_doc)
assert not filler._closed
assert event['data']['image'].shape == (5, 5)
# Test verify_filled.
with pytest.raises(event_model.UnfilledData):
event_model.verify_filled(event_model.pack_event_page(raw_event))
event_model.verify_filled(event_model.pack_event_page(event))
def test_document_router_dispatch_event():
event_calls = [] # used for counting calls
event_page_calls = [] # used for counting calls
# example documents
event1 = {
'data': {
'x': 1
},
'timestamps': {
'x': 0.
},
'uid': 'placeholder X',
'descriptor': 'placeholder Y',
'time': 0.,
'seq_num': 1
}
event2 = {
'data': {
'x': 2
},
'timestamps': {
'x': 1.
},
'uid': 'placeholder X',
'descriptor': 'placeholder Y',
'time': 1.,
'seq_num': 2
}
event_page = event_model.pack_event_page(event1, event2)
def check(ret, original=None):
name, doc = ret
assert doc is not None
assert doc is not NotImplemented
if original is not None:
# Verify that a copy is returned.
assert doc is not original # ret is such a poser, dude.
doc.pop('filled', None)
original.pop('filled', None)
assert doc == original
class DefinesNeitherEventNorEventPage(event_model.DocumentRouter):
def event(self, doc):
event_calls.append(object())
# This returns NotImplemented.
return super().event_page(doc)
def event_page(self, doc):
event_page_calls.append(object())
# This returns NotImplemented.
return super().event_page(doc)
dr = DefinesNeitherEventNorEventPage()
# Test that Event is routed to Event and EventPage.
check(dr('event', event1))
assert len(event_calls) == 1
assert len(event_page_calls) == 1
event_calls.clear()
event_page_calls.clear()
# Test that EventPage is routed to EventPage and Event *once* before
# giving up.
check(dr('event_page', event_page))
assert len(event_page_calls) == 1
assert len(event_calls) == 1
event_calls.clear()
event_page_calls.clear()
class DefinesEventNotEventPage(event_model.DocumentRouter):
def event(self, doc):
# Just a dumb test that check something particular to these example
# documents.
assert doc['data']['x'] == doc['seq_num']
event_calls.append(object())
return dict(doc)
def event_page(self, doc):
event_page_calls.append(object())
# This returns NotImplemented.
return super().event_page(doc)
dr = DefinesEventNotEventPage()
# Test that Event is routed to Event.
check(dr('event', event1), event1)
assert len(event_calls) == 1
assert len(event_page_calls) == 0
event_calls.clear()
event_page_calls.clear()
# Test that EventPage is unpacked and routed to Event one at a time.
check(dr('event_page', event_page), event_page)
assert len(event_page_calls) == 1
assert len(event_calls) == 2
event_calls.clear()
event_page_calls.clear()
class DefinesEventPageNotEvent(event_model.DocumentRouter):
def event(self, doc):
event_calls.append(object())
# This returns NotImplemented.
return super().event(doc)
def event_page(self, doc):
# Just a dumb test that check something particular to these example
# documents.
assert doc['data']['x'][0] == 1
event_page_calls.append(object())
return dict(doc)
dr = DefinesEventPageNotEvent()
# Test that Event is packed and routed to EventPage.
check(dr('event', event1), event1)
assert len(event_calls) == 1
assert len(event_page_calls) == 1
event_calls.clear()
event_page_calls.clear()
# Test that EventPage is routed to EventPage.
check(dr('event_page', event_page), event_page)
assert len(event_page_calls) == 1
assert len(event_calls) == 0
event_calls.clear()
event_page_calls.clear()
class DefinesEventPageAndEvent(event_model.DocumentRouter):
def event(self, doc):
# Just a dumb test that check something particular to these example
# documents.
assert doc['data']['x'] == doc['seq_num']
event_calls.append(object())
return dict(doc)
def event_page(self, doc):
# Just a dumb test that check something particular to these example
# documents.
assert doc['data']['x'][0] == 1
event_page_calls.append(object())
return dict(doc)
dr = DefinesEventPageAndEvent()
# Test that Event is routed to Event.
check(dr('event', event1), event1)
assert len(event_calls) == 1
assert len(event_page_calls) == 0
event_calls.clear()
event_page_calls.clear()
# Test that EventPage is routed to EventPage.
check(dr('event_page', event_page), event_page)
assert len(event_page_calls) == 1
assert len(event_calls) == 0
event_calls.clear()
event_page_calls.clear()
def test_round_trip_pagination():
run_bundle = event_model.compose_run()
desc_bundle = run_bundle.compose_descriptor(data_keys={
'motor': {
'shape': [],
'dtype': 'number',
'source': '...'
},
'image': {
'shape': [512, 512],
'dtype': 'number',
'source': '...',
'external': 'FILESTORE:'
}
},
name='primary')
res_bundle = run_bundle.compose_resource(spec='TIFF',
root='/tmp',
resource_path='stack.tiff',
resource_kwargs={})
datum_doc1 = res_bundle.compose_datum(datum_kwargs={'slice': 5})
datum_doc2 = res_bundle.compose_datum(datum_kwargs={'slice': 10})
datum_doc3 = res_bundle.compose_datum(datum_kwargs={'slice': 15})
event_doc1 = desc_bundle.compose_event(data={
'motor': 0,
'image': datum_doc1['datum_id']
},
timestamps={
'motor': 0,
'image': 0
},
filled={'image': False},
seq_num=1)
event_doc2 = desc_bundle.compose_event(data={
'motor': 1,
'image': datum_doc2['datum_id']
},
timestamps={
'motor': 0,
'image': 0
},
filled={'image': False},
seq_num=1)
event_doc3 = desc_bundle.compose_event(data={
'motor': 2,
'image': datum_doc3['datum_id']
},
timestamps={
'motor': 0,
'image': 0
},
filled={'image': False},
seq_num=1)
# Round trip single event -> event_page -> event.
expected = event_doc1
actual, = event_model.unpack_event_page(
event_model.pack_event_page(expected))
assert actual == expected
# Round trip two events -> event_page -> events.
expected = [event_doc1, event_doc2]
actual = list(
event_model.unpack_event_page(event_model.pack_event_page(*expected)))
assert actual == expected
# Round trip three events -> event_page -> events.
expected = [event_doc1, event_doc2, event_doc3]
actual = list(
event_model.unpack_event_page(event_model.pack_event_page(*expected)))
assert actual == expected
# Round trip on docs that don't have a filled key
unfilled_doc1 = event_doc1
unfilled_doc1.pop('filled')
unfilled_doc2 = event_doc2
unfilled_doc2.pop('filled')
unfilled_doc3 = event_doc3
unfilled_doc3.pop('filled')
expected = [unfilled_doc1, unfilled_doc2, unfilled_doc3]
actual = list(
event_model.unpack_event_page(event_model.pack_event_page(*expected)))
for doc in actual:
doc.pop('filled')
assert actual == expected
# Round trip one datum -> datum_page -> datum.
expected = datum_doc1
actual, = event_model.unpack_datum_page(
event_model.pack_datum_page(expected))
assert actual == expected
# Round trip two datum -> datum_page -> datum.
expected = [datum_doc1, datum_doc2]
actual = list(
event_model.unpack_datum_page(event_model.pack_datum_page(*expected)))
assert actual == expected
# Round trip three datum -> datum_page -> datum.
expected = [datum_doc1, datum_doc2, datum_doc3]
actual = list(
event_model.unpack_datum_page(event_model.pack_datum_page(*expected)))
assert actual == expected
# Check edge case where datum_kwargs are empty.
datum_doc1 = res_bundle.compose_datum(datum_kwargs={})
datum_doc2 = res_bundle.compose_datum(datum_kwargs={})
datum_doc3 = res_bundle.compose_datum(datum_kwargs={})
# Round trip one datum -> datum_page -> datum.
expected = datum_doc1
actual, = event_model.unpack_datum_page(
event_model.pack_datum_page(expected))
assert actual == expected
# Round trip two datum -> datum_page -> datum.
expected = [datum_doc1, datum_doc2]
actual = list(
event_model.unpack_datum_page(event_model.pack_datum_page(*expected)))
assert actual == expected
# Round trip three datum -> datum_page -> datum.
expected = [datum_doc1, datum_doc2, datum_doc3]
actual = list(
event_model.unpack_datum_page(event_model.pack_datum_page(*expected)))
assert actual == expected
def test_pack_empty_raises():
with pytest.raises(ValueError):
event_model.pack_event_page()
with pytest.raises(ValueError):
event_model.pack_datum_page()
def export(gen, filepath, **kwargs):
"""
Export a stream of documents to CSV file(s) and one JSON file of metadata.
Creates {filepath}_meta.json and then {filepath}_{stream_name}.csv
for every Event stream.
The structure of the json is::
{'start': {...},
'descriptors':
{'<stream_name>': [{...}, {...}, ...],
...},
'stop': {...}}
Parameters
----------
gen : generator
expected to yield (name, document) pairs
filepath : str
the filepath and filename suffix to use in the output files.
**kwargs : kwargs
kwargs to be passed to pandas.Dataframe.to_csv.
Returns
-------
dest : tuple
filepaths of generated files
"""
meta = {} # to be exported as JSON at the end
meta['descriptors'] = defaultdict(list) # map stream_name to descriptors
files = {} # map descriptor uid to file handle of CSV file
desc_counters = defaultdict(itertools.count)
has_header = set() # a set of uids indicating if the file has a header
kwargs.setdefault('header', True)
initial_header_kwarg = kwargs['header'] # used later to set the headers
kwargs.setdefault('index_label', 'time')
kwargs.setdefault('mode', 'a')
try:
for name, doc in gen:
if name == 'start':
if 'start' in meta:
raise RuntimeError("This exporter expects documents from "
"one run only.")
meta['start'] = doc
elif name == 'stop':
meta['stop'] = doc
elif name == 'descriptor':
stream_name = doc.get('name')
meta['descriptors'][stream_name].append(doc)
filepath_ = (f"{filepath}_{stream_name}_"
f"{next(desc_counters[doc['uid']])}.csv")
files[doc['uid']] = open(filepath_, 'w+')
elif (name == 'event' or name == 'bulk_event'
or name == 'event_page'):
if name == 'event': # convert event to an event_pages list
event_pages = [event_model.pack_event_page(doc)]
elif name == 'bulk_event': # convert bulk_event to event_pages
event_pages = event_model.bulk_events_to_event_pages(doc)
else: # convert an event_page to an event_pages list.
event_pages = [doc]
for event_page in event_pages:
if not all(event_page['filled'].values()):
# check that all event_page data is filled
unfilled_data = []
for field in event_page['filled']:
if not event_page['filled'][field]:
unfilled_data.append(field)
# Note: As of this writing, this is a slightly
# aspirational error message, as event_model.Filler has
# not been merged yet. May need to be revisited if it
# is renamed or kept elsewhere in the end.
raise UnfilledData('unfilled data found in'
'{}. Try passing the parameter '
'"gen" through "event_model.Filler"'
' first'.format(unfilled_data))
else:
event_data = pandas.DataFrame(event_page['data'],
index=event_page['time'])
event_data['seq_num'] = event_page['seq_num']
if initial_header_kwarg:
kwargs['header'] = event_page['descriptor'] \
not in has_header
event_data.to_csv(files[event_page['descriptor']],
**kwargs)
has_header.add(event_page['descriptor'])
finally:
for f in files.values():
f.close()
with open(f"{filepath}_meta.json", 'w') as f:
json.dump(meta, f)
return (f.name, ) + tuple(f.name for f in files.values())
