def test_canonical_unfilled(bundle):
run = bundle.cat['xyz']()[bundle.uid]
run.canonical_unfilled()
def sorted_actual():
for name_ in ('start', 'descriptor', 'resource', 'datum', 'event_page',
'event', 'stop'):
for name, doc in bundle.docs:
if name == name_ and name in ('start', 'descriptor', 'event',
'event_page', 'stop'):
yield name, doc
raw_run = [(name, doc) for name, doc in list(run.canonical_unfilled())
if name not in ('resource', 'datum', 'datum_page')]
for actual, expected in itertools.zip_longest(raw_run, sorted_actual()):
actual_name, actual_doc = actual
expected_name, expected_doc = expected
print(actual_name, expected_name)
try:
assert actual_name == expected_name
except ValueError:
assert numpy.array_equal(actual_doc, expected_doc)
# Passing the run through the filler to check resource and datum are
# received before corresponding event.
filler = event_model.Filler({'NPY_SEQ': ophyd.sim.NumpySeqHandler},
inplace=False)
for name, doc in run.canonical_unfilled():
filler(name, doc)
def __init__(self, handler_registry=None, query=None, **kwargs):
"""
This Catalog is backed by Python collections in memory.
Subclasses should define a ``_load`` method (same as any intake
Catalog) that calls this class's ``upsert`` method (which is particular
to this class).
Parameters
----------
handler_registry : dict, optional
Maps each asset spec to a handler class or a string specifying the
module name and class name, as in (for example)
``{'SOME_SPEC': 'module.submodule.class_name'}``.
query : dict, optional
Mongo query that filters entries' RunStart documents
**kwargs :
Additional keyword arguments are passed through to the base class,
Catalog.
"""
self._query = query or {}
if handler_registry is None:
handler_registry = {}
parsed_handler_registry = parse_handler_registry(handler_registry)
self.filler = event_model.Filler(parsed_handler_registry, inplace=True)
self._uid_to_run_start_doc = {}
super().__init__(**kwargs)
def test_canonical(bundle):
run = bundle.cat['xyz']()[bundle.uid]
# Smoke test for back-compat alias
with pytest.warns(UserWarning):
next(run.read_canonical())
filler = event_model.Filler({'NPY_SEQ': ophyd.sim.NumpySeqHandler},
inplace=False)
def sorted_actual():
for name_ in ('start', 'descriptor', 'resource', 'datum', 'event_page',
'event', 'stop'):
for name, doc in bundle.docs:
# Fill external data.
_, filled_doc = filler(name, doc)
if name == name_ and name in ('start', 'descriptor', 'event',
'event_page', 'stop'):
yield name, filled_doc
for actual, expected in itertools.zip_longest(run.canonical(),
sorted_actual()):
actual_name, actual_doc = actual
expected_name, expected_doc = expected
print(actual_name, expected_name)
try:
assert actual_name == expected_name
except ValueError:
assert numpy.array_equal(actual_doc, expected_doc)
def test_root_map():
new_path = pathlib.Path("/another_placeholder", "moved")
class DummyHandlerRootMapTest:
def __init__(self, resource_path, a, b):
assert a == 1
assert b == 2
assert resource_path == str(new_path / "stack.tiff")
def __call__(self, c, d):
assert c == 3
assert d == 4
return numpy.ones((5, 5))
with event_model.Filler({'DUMMY': DummyHandlerRootMapTest},
root_map={str(path_root): str(new_path)},
inplace=True) as filler:
filler('start', run_bundle.start_doc)
filler('descriptor', desc_bundle.descriptor_doc)
filler('descriptor', desc_bundle_baseline.descriptor_doc)
filler('resource', res_bundle.resource_doc)
filler('datum', datum_doc)
event = copy.deepcopy(raw_event)
filler('event', event)
filler('stop', stop_doc)
assert not filler.closed
assert event['data']['image'].shape == (5, 5)
def __init__(self, file_list, handler, *,
query=None, **kwargs):
"""
This Catalog is backed by a newline-delimited JSON (jsonl) file.
Each line of the file is expected to be a JSON list with two elements,
the document name (type) and the document itself. The documents are
expected to be in chronological order.
Parameters
----------
file_list : list
list of filepaths
handler_registry : dict, optional
Maps each asset spec to a handler class or a string specifying the
module name and class name, as in (for example)
``{'SOME_SPEC': 'module.submodule.class_name'}``.
**kwargs :
Additional keyword arguments are passed through to the base class,
Catalog.
"""
name = 'bluesky-files-catalog' # noqa
self._runs = {} # This maps run_start_uids to file paths.
self._run_starts = {} # This maps run_start_uids to run_start_docs.
self._query = query or {}
self.handler = handler
self.filler = event_model.Filler({handler.name: handler})
self._update_index(file_list)
super().__init__(**kwargs)
def test_include_and_exclude():
with pytest.raises(ValueError):
event_model.Filler({}, include=[], exclude=[], inplace=True)
with pytest.warns(DeprecationWarning):
with event_model.Filler(reg, exclude=['image'],
inplace=True) as filler:
filler('start', run_bundle.start_doc)
filler('descriptor', desc_bundle.descriptor_doc)
filler('descriptor', desc_bundle_baseline.descriptor_doc)
filler('resource', res_bundle.resource_doc)
filler('datum', datum_doc)
event = copy.deepcopy(raw_event)
assert isinstance(event['data']['image'], str)
filler('event', event)
filler('stop', stop_doc)
with pytest.warns(DeprecationWarning):
with event_model.Filler(reg, include=['image'],
inplace=True) as filler:
filler('start', run_bundle.start_doc)
filler('descriptor', desc_bundle.descriptor_doc)
filler('descriptor', desc_bundle_baseline.descriptor_doc)
filler('resource', res_bundle.resource_doc)
filler('datum', datum_doc)
event = copy.deepcopy(raw_event)
filler('event', event)
filler('stop', stop_doc)
assert not filler.closed
assert event['data']['image'].shape == (5, 5)
with pytest.warns(DeprecationWarning):
with event_model.Filler(reg,
include=['image', 'EXTRA THING'],
inplace=True) as filler:
filler('start', run_bundle.start_doc)
filler('descriptor', desc_bundle.descriptor_doc)
filler('descriptor', desc_bundle_baseline.descriptor_doc)
filler('resource', res_bundle.resource_doc)
filler('datum', datum_doc)
event = copy.deepcopy(raw_event)
filler('event', event)
filler('stop', stop_doc)
assert not filler.closed
assert event['data']['image'].shape == (5, 5)
def test_handler_registry_access():
"Test the handler_registery can be viewed but is immutable."
class OtherDummyHandler:
"Same as DummyHandler, but a different object to test mutating reg"
def __init__(self, resource_path, a, b):
assert a == 1
assert b == 2
assert resource_path == str(path_root / "stack.tiff")
def __call__(self, c, d):
assert c == 3
assert d == 4
return numpy.ones((5, 5))
with event_model.Filler(reg, inplace=False) as filler:
with pytest.raises(event_model.EventModelTypeError):
# Updating an existing key fails.
filler.handler_registry['DUMMY'] = OtherDummyHandler
with pytest.raises(event_model.EventModelTypeError):
# Setting a new key fails.
filler.handler_registry['SOMETHING_ELSE'] = OtherDummyHandler
with pytest.raises(event_model.EventModelTypeError):
# Deleting a item fails.
del filler.handler_registry['DUMMY']
filler('start', run_bundle.start_doc)
filler('descriptor', desc_bundle.descriptor_doc)
filler('descriptor', desc_bundle_baseline.descriptor_doc)
filler('resource', res_bundle.resource_doc)
filler('datum', datum_doc)
event = copy.deepcopy(raw_event)
name, filled_event = filler('event', event)
assert filled_event is not event
assert isinstance(event['data']['image'], str)
# Now there should be a handler instance in the cache.
assert filler._handler_cache # implementation detail
with pytest.raises(event_model.DuplicateHandler):
filler.register_handler('DUMMY', OtherDummyHandler)
filler.register_handler('DUMMY', OtherDummyHandler, overwrite=True)
assert filler.handler_registry['DUMMY'] is OtherDummyHandler
# Replacing the handler for a given spec should clear the cache.
assert not filler._handler_cache # implementation detail
# Filling should work the same....
filler('start', run_bundle.start_doc)
filler('descriptor', desc_bundle.descriptor_doc)
filler('descriptor', desc_bundle_baseline.descriptor_doc)
filler('resource', res_bundle.resource_doc)
filler('datum', datum_doc)
event = copy.deepcopy(raw_event)
name, filled_event = filler('event', event)
assert filled_event is not event
assert isinstance(event['data']['image'], str)
handler = filler.deregister_handler('DUMMY')
assert handler is OtherDummyHandler
assert not filler.handler_registry
assert not filler._handler_cache # implementation detail
def test_fill(data_files):
import event_model
with event_model.Filler({TIFSeriesPlugin.name: TIFSeriesPlugin}) as filler:
doc_stream = TIFSeriesPlugin.ingest(data_files)
for name, doc in doc_stream:
name, doc = filler(name, doc)
doc['filled'] = {'image': False}
if name == 'event':
assert isinstance(doc['data']['image'], np.ndarray)
def test_undefined_handler_spec():
"Check failure path when an unknown spec is found in a resource."
with event_model.Filler({}, inplace=True) as filler:
filler('start', run_bundle.start_doc)
filler('descriptor', desc_bundle.descriptor_doc)
filler('descriptor', desc_bundle_baseline.descriptor_doc)
filler('resource', res_bundle.resource_doc)
filler('datum', datum_doc)
event = copy.deepcopy(raw_event)
assert isinstance(event['data']['image'], str)
with pytest.raises(event_model.UndefinedAssetSpecification):
filler('event', event)
def test_canonical(bundle):
run = bundle.cat['xyz']()[bundle.uid]
filler = event_model.Filler({'NPY_SEQ': ophyd.sim.NumpySeqHandler},
inplace=False)
# Smoke test for back-compat alias
with pytest.warns(UserWarning):
next(run.read_canonical())
compare(run.canonical(fill='yes'),
(filler(name, doc) for name, doc in bundle.docs))
def test_canonical_delayed(bundle):
run = bundle.cat['xyz']()[bundle.uid]
filler = event_model.Filler({'NPY_SEQ': ophyd.sim.NumpySeqHandler},
inplace=False)
if bundle.remote:
with pytest.raises(NotImplementedError):
next(run.canonical(fill='delayed'))
else:
compare(run.canonical(fill='delayed'),
(filler(name, doc) for name, doc in bundle.docs))
def test_no_descriptors():
run_bundle = event_model.compose_run()
start_doc = run_bundle.start_doc
stop_doc = run_bundle.compose_stop()
documents_to_xarray(start_doc=start_doc,
stop_doc=stop_doc,
descriptor_docs=[],
get_event_pages=no_event_pages,
filler=event_model.Filler({}, inplace=True),
get_resource=None,
lookup_resource_for_datum=None,
get_datum_pages=None)
def test_canonical_unfilled(bundle):
run = bundle.cat['xyz']()[bundle.uid]
run.canonical(fill='no')
compare(run.canonical(fill='no'), bundle.docs)
# Passing the run through the filler to check resource and datum are
# received before corresponding event.
filler = event_model.Filler({'NPY_SEQ': ophyd.sim.NumpySeqHandler},
inplace=False)
for name, doc in run.canonical(fill='no'):
filler(name, doc)
def __init__(self, gen_func, gen_args, gen_kwargs, filler=None, **kwargs):
if filler is None:
filler = event_model.Filler({}, inplace=True)
document_cache = DocumentCache()
for item in gen_func(*gen_args, **gen_kwargs):
document_cache(*item)
assert document_cache.start_doc is not None
def get_run_start():
return document_cache.start_doc
def get_run_stop():
return document_cache.stop_doc
def get_event_descriptors():
return document_cache.descriptors.values()
def get_event_pages(descriptor_uid, skip=0, limit=None):
if skip != 0 and limit is not None:
raise NotImplementedError
return document_cache.event_pages[descriptor_uid]
def get_event_count(descriptor_uid):
return sum(len(page['seq_num'])
for page in (document_cache.event_pages[descriptor_uid]))
def get_resource(uid):
return document_cache.resources[uid]
def lookup_resource_for_datum(datum_id):
return document_cache.resource_uid_by_datum_id[datum_id]
def get_datum_pages(resource_uid, skip=0, limit=None):
if skip != 0 and limit is not None:
raise NotImplementedError
return document_cache.datum_pages_by_resource[resource_uid]
super().__init__(
get_run_start=get_run_start,
get_run_stop=get_run_stop,
get_event_descriptors=get_event_descriptors,
get_event_pages=get_event_pages,
get_event_count=get_event_count,
get_resource=get_resource,
lookup_resource_for_datum=lookup_resource_for_datum,
get_datum_pages=get_datum_pages,
filler=filler,
**kwargs)
def test_canonical(bundle):
"This is a deprecated synonym for documents()."
run = bundle.cat['xyz']()[bundle.uid]
filler = event_model.Filler({'NPY_SEQ': ophyd.sim.NumpySeqHandler},
inplace=False)
# Smoke test for back-compat alias
with pytest.warns(UserWarning):
next(run.canonical(fill="yes"))
compare(run.documents(fill='yes'),
(filler(name, doc) for name, doc in bundle.docs))
def filler():
filler = event_model.Filler(reg, inplace=True)
filler('start', run_bundle.start_doc)
filler('descriptor', desc_bundle.descriptor_doc)
filler('descriptor', desc_bundle_baseline.descriptor_doc)
filler('resource', res_bundle.resource_doc)
filler('datum', datum_doc)
event = copy.deepcopy(raw_event)
assert isinstance(event['data']['image'], str)
filler('event', event)
assert event['data']['image'].shape == (5, 5)
filler('stop', stop_doc)
assert not filler.closed
return filler
def test_context_manager_with_event_page():
with event_model.Filler(reg, inplace=True) as filler:
filler('start', run_bundle.start_doc)
filler('descriptor', desc_bundle.descriptor_doc)
filler('descriptor', desc_bundle_baseline.descriptor_doc)
filler('resource', res_bundle.resource_doc)
filler('datum', datum_doc)
event_page = event_model.pack_event_page(copy.deepcopy(raw_event))
name, doc = filler('event_page', event_page)
assert name == 'event_page'
assert doc is event_page
filler('stop', stop_doc)
assert not filler.closed
assert event_page['data']['image'][0].shape == (5, 5)
assert filler.closed
def test_context_manager():
"Construct Filler as a context manager."
with event_model.Filler(reg, inplace=True) as filler:
filler('start', run_bundle.start_doc)
filler('descriptor', desc_bundle.descriptor_doc)
filler('descriptor', desc_bundle_baseline.descriptor_doc)
filler('resource', res_bundle.resource_doc)
filler('datum', datum_doc)
event = copy.deepcopy(raw_event)
name, doc = filler('event', event)
assert name == 'event'
assert doc is event
filler('stop', stop_doc)
assert not filler.closed
assert event['data']['image'].shape == (5, 5)
assert filler.closed
def test_no_events():
run_bundle = event_model.compose_run()
start_doc = run_bundle.start_doc
desc_bundle = run_bundle.compose_descriptor(
data_keys={'x': {'source': '...', 'shape': [], 'dtype': 'number'}},
name='primary')
descriptor_doc = desc_bundle.descriptor_doc
stop_doc = run_bundle.compose_stop()
documents_to_xarray(
start_doc=start_doc,
stop_doc=stop_doc,
descriptor_docs=[descriptor_doc],
get_event_pages=no_event_pages,
filler=event_model.Filler({}, inplace=True),
get_resource=None,
lookup_resource_for_datum=None,
get_datum_pages=None)
def __init__(self, datastore_db, *, handler_registry=None,
query=None, **kwargs):
"""
This Catalog is backed by a MongoDB with an embedded data model.
This embedded data model has three collections: header, event, datum.
The header collection includes start, stop, descriptor, and resource
documents. The event_pages are stored in the event colleciton, and
datum_pages are stored in the datum collection.
Parameters
----------
datastore_db : pymongo.database.Database or string
Must be a Database or a URI string that includes a database name.
handler_registry : dict, optional
Maps each asset spec to a handler class or a string specifying the
module name and class name, as in (for example)
``{'SOME_SPEC': 'module.submodule.class_name'}``.
query : dict, optional
MongoDB query. Used internally by the ``search()`` method.
**kwargs :
Additional keyword arguments are passed through to the base class,
Catalog.
"""
name = 'bluesky-mongo-embedded-catalog' # noqa
if isinstance(datastore_db, str):
self._db = _get_database(datastore_db)
else:
self._db = datastore_db
self._query = query or {}
if handler_registry is None:
handler_registry = {}
parsed_handler_registry = parse_handler_registry(handler_registry)
self.filler = event_model.Filler(parsed_handler_registry, inplace=True)
super().__init__(**kwargs)
def test_mismatched_data_keys():
"Test that we raise specifically when data keys do not match between event and descriptor."
with pytest.raises(event_model.MismatchedDataKeys):
with event_model.NoFiller(reg) as filler:
filler('start', run_bundle.start_doc)
filler('descriptor', desc_bundle.descriptor_doc)
filler('descriptor', desc_bundle_baseline.descriptor_doc)
filler('resource', res_bundle.resource_doc)
filler('datum', datum_doc)
event = copy.deepcopy(raw_event)
del event['data']['image']
filler('event', event)
with pytest.raises(event_model.MismatchedDataKeys):
with event_model.Filler(reg, inplace=False) as filler:
filler('start', run_bundle.start_doc)
filler('descriptor', desc_bundle.descriptor_doc)
filler('descriptor', desc_bundle_baseline.descriptor_doc)
filler('resource', res_bundle.resource_doc)
filler('datum', datum_doc)
event = copy.deepcopy(raw_event)
del event['data']['image']
filler('event', event)
def test_inplace():
"Test the behavior of the 'inplace' parameter."
with event_model.Filler(reg, inplace=True) as filler:
filler('start', run_bundle.start_doc)
filler('descriptor', desc_bundle.descriptor_doc)
filler('descriptor', desc_bundle_baseline.descriptor_doc)
filler('resource', res_bundle.resource_doc)
filler('datum', datum_doc)
# Test event()
event = copy.deepcopy(raw_event)
name, filled_event = filler('event', event)
assert filled_event is event
event = copy.deepcopy(raw_event)
# Test fill_event()
filled_event = filler.fill_event(event)
assert filled_event is event
# Test event_page()
event_page = event_model.pack_event_page(copy.deepcopy(raw_event))
_, filled_event_page = filler('event_page', event_page)
assert filled_event_page is event_page
# Test fill_event_page()
event_page = event_model.pack_event_page(copy.deepcopy(raw_event))
filled_event_page = filler.fill_event_page(event_page)
assert filled_event_page is event_page
# Test fill_event and fill_event_page again with inplace=False.
# Test fill_event()
filled_event = filler.fill_event(event, inplace=False)
assert filled_event is not event
# Test fill_event_page()
event_page = event_model.pack_event_page(copy.deepcopy(raw_event))
filled_event_page = filler.fill_event_page(event_page, inplace=False)
assert filled_event_page is not event_page
with event_model.Filler(reg, inplace=False) as filler:
filler('start', run_bundle.start_doc)
filler('descriptor', desc_bundle.descriptor_doc)
filler('descriptor', desc_bundle_baseline.descriptor_doc)
filler('resource', res_bundle.resource_doc)
filler('datum', datum_doc)
event = copy.deepcopy(raw_event)
name, filled_event = filler('event', event)
assert filled_event is not event
assert isinstance(event['data']['image'], str)
event = copy.deepcopy(raw_event)
# Test fill_event()
filled_event = filler.fill_event(event)
assert filled_event is not event
# Test event_page()
event_page = event_model.pack_event_page(copy.deepcopy(raw_event))
_, filled_event_page = filler('event_page', event_page)
assert filled_event_page is not event_page
# Test fill_event_page()
event_page = event_model.pack_event_page(copy.deepcopy(raw_event))
filled_event_page = filler.fill_event_page(event_page)
assert filled_event_page is not event_page
# Test fill_event and fill_event_page again with inplace=True.
# Test fill_event()
filled_event = filler.fill_event(event, inplace=True)
assert filled_event is event
# Test fill_event_page()
event_page = event_model.pack_event_page(copy.deepcopy(raw_event))
filled_event_page = filler.fill_event_page(event_page, inplace=True)
assert filled_event_page is event_page
with pytest.warns(UserWarning):
# warnings because inplace is not specified
filler = event_model.Filler(reg)
def test_pickle_filler():
filler = event_model.Filler({}, inplace=False)
serialized = pickle.dumps(filler)
deserialized = pickle.loads(serialized)
assert filler == deserialized
def test_retry_intervals_input_normalization(retry_intervals):
filler = event_model.Filler({},
retry_intervals=retry_intervals,
inplace=False)
assert isinstance(filler.retry_intervals, list)
def test_filler(tmp_path):
class DummyHandler:
def __init__(self, resource_path, a, b):
assert a == 1
assert b == 2
assert resource_path == str(tmp_path / "stack.tiff")
def __call__(self, c, d):
assert c == 3
assert d == 4
return numpy.ones((5, 5))
path_root = str(tmp_path)
reg = {'DUMMY': DummyHandler}
filler = event_model.Filler(reg)
run_bundle = event_model.compose_run()
desc_bundle = run_bundle.compose_descriptor(data_keys={
'motor': {
'shape': [],
'dtype': 'number',
'source': '...'
},
'image': {
'shape': [512, 512],
'dtype': 'number',
'source': '...',
'external': 'FILESTORE:'
}
},
name='primary')
desc_bundle_baseline = run_bundle.compose_descriptor(
data_keys={'motor': {
'shape': [],
'dtype': 'number',
'source': '...'
}},
name='baseline')
res_bundle = run_bundle.compose_resource(spec='DUMMY',
root=path_root,
resource_path='stack.tiff',
resource_kwargs={
'a': 1,
'b': 2
})
datum_doc = res_bundle.compose_datum(datum_kwargs={'c': 3, 'd': 4})
raw_event = desc_bundle.compose_event(data={
'motor': 0,
'image': datum_doc['datum_id']
},
timestamps={
'motor': 0,
'image': 0
},
filled={'image': False},
seq_num=1)
filler('start', run_bundle.start_doc)
filler('descriptor', desc_bundle.descriptor_doc)
filler('descriptor', desc_bundle_baseline.descriptor_doc)
filler('resource', res_bundle.resource_doc)
filler('datum', datum_doc)
event = copy.deepcopy(raw_event)
assert isinstance(event['data']['image'], str)
filler('event', event)
stop_doc = run_bundle.compose_stop()
filler('stop', stop_doc)
assert event['data']['image'].shape == (5, 5)
assert not filler._closed
filler.close()
assert filler._closed
# Test context manager with Event.
with event_model.Filler(reg) as filler:
filler('start', run_bundle.start_doc)
filler('descriptor', desc_bundle.descriptor_doc)
filler('descriptor', desc_bundle_baseline.descriptor_doc)
filler('resource', res_bundle.resource_doc)
filler('datum', datum_doc)
event = copy.deepcopy(raw_event)
name, doc = filler('event', event)
assert name == 'event'
assert doc is event
filler('stop', stop_doc)
assert not filler._closed
assert event['data']['image'].shape == (5, 5)
assert filler._closed
# Test context manager with EventPage.
with event_model.Filler(reg) as filler:
filler('start', run_bundle.start_doc)
filler('descriptor', desc_bundle.descriptor_doc)
filler('descriptor', desc_bundle_baseline.descriptor_doc)
filler('resource', res_bundle.resource_doc)
filler('datum', datum_doc)
event_page = event_model.pack_event_page(copy.deepcopy(raw_event))
name, doc = filler('event_page', event_page)
assert name == 'event_page'
assert doc is event_page
filler('stop', stop_doc)
assert not filler._closed
assert event_page['data']['image'][0].shape == (5, 5)
assert filler._closed
# Test undefined handler spec
with event_model.Filler({}) as filler:
filler('start', run_bundle.start_doc)
filler('descriptor', desc_bundle.descriptor_doc)
filler('descriptor', desc_bundle_baseline.descriptor_doc)
filler('resource', res_bundle.resource_doc)
filler('datum', datum_doc)
event = copy.deepcopy(raw_event)
assert isinstance(event['data']['image'], str)
with pytest.raises(event_model.UndefinedAssetSpecification):
filler('event', event)
# Test exclude and include.
with pytest.raises(ValueError):
event_model.Filler({}, include=[], exclude=[])
with event_model.Filler(reg, exclude=['image']) as filler:
filler('start', run_bundle.start_doc)
filler('descriptor', desc_bundle.descriptor_doc)
filler('descriptor', desc_bundle_baseline.descriptor_doc)
filler('resource', res_bundle.resource_doc)
filler('datum', datum_doc)
event = copy.deepcopy(raw_event)
assert isinstance(event['data']['image'], str)
filler('event', event)
filler('stop', stop_doc)
with event_model.Filler(reg, include=['image']) as filler:
filler('start', run_bundle.start_doc)
filler('descriptor', desc_bundle.descriptor_doc)
filler('descriptor', desc_bundle_baseline.descriptor_doc)
filler('resource', res_bundle.resource_doc)
filler('datum', datum_doc)
event = copy.deepcopy(raw_event)
filler('event', event)
filler('stop', stop_doc)
assert not filler._closed
assert event['data']['image'].shape == (5, 5)
with event_model.Filler(reg, include=['image', 'EXTRA THING']) as filler:
filler('start', run_bundle.start_doc)
filler('descriptor', desc_bundle.descriptor_doc)
filler('descriptor', desc_bundle_baseline.descriptor_doc)
filler('resource', res_bundle.resource_doc)
filler('datum', datum_doc)
event = copy.deepcopy(raw_event)
filler('event', event)
filler('stop', stop_doc)
assert not filler._closed
assert event['data']['image'].shape == (5, 5)
class DummyHandlerRootMapTest:
def __init__(self, resource_path, a, b):
assert a == 1
assert b == 2
assert resource_path == str(tmp_path / "moved" / "stack.tiff")
def __call__(self, c, d):
assert c == 3
assert d == 4
return numpy.ones((5, 5))
with event_model.Filler({'DUMMY': DummyHandlerRootMapTest},
root_map={path_root:
str(tmp_path / "moved")}) as filler:
filler('start', run_bundle.start_doc)
filler('descriptor', desc_bundle.descriptor_doc)
filler('descriptor', desc_bundle_baseline.descriptor_doc)
filler('resource', res_bundle.resource_doc)
filler('datum', datum_doc)
event = copy.deepcopy(raw_event)
filler('event', event)
filler('stop', stop_doc)
assert not filler._closed
assert event['data']['image'].shape == (5, 5)
# Test verify_filled.
with pytest.raises(event_model.UnfilledData):
event_model.verify_filled(event_model.pack_event_page(raw_event))
event_model.verify_filled(event_model.pack_event_page(event))
def test_filler(tmp_path):
class DummyHandler:
def __init__(self, resource_path, a, b):
assert a == 1
assert b == 2
assert resource_path == str(tmp_path / "stack.tiff")
def __call__(self, c, d):
assert c == 3
assert d == 4
return numpy.ones((5, 5))
path_root = str(tmp_path)
reg = {'DUMMY': DummyHandler}
filler = event_model.Filler(reg, inplace=True)
run_bundle = event_model.compose_run()
desc_bundle = run_bundle.compose_descriptor(data_keys={
'motor': {
'shape': [],
'dtype': 'number',
'source': '...'
},
'image': {
'shape': [512, 512],
'dtype': 'number',
'source': '...',
'external': 'FILESTORE:'
}
},
name='primary')
desc_bundle_baseline = run_bundle.compose_descriptor(
data_keys={'motor': {
'shape': [],
'dtype': 'number',
'source': '...'
}},
name='baseline')
res_bundle = run_bundle.compose_resource(spec='DUMMY',
root=path_root,
resource_path='stack.tiff',
resource_kwargs={
'a': 1,
'b': 2
})
datum_doc = res_bundle.compose_datum(datum_kwargs={'c': 3, 'd': 4})
raw_event = desc_bundle.compose_event(data={
'motor': 0,
'image': datum_doc['datum_id']
},
timestamps={
'motor': 0,
'image': 0
},
filled={'image': False},
seq_num=1)
filler('start', run_bundle.start_doc)
filler('descriptor', desc_bundle.descriptor_doc)
filler('descriptor', desc_bundle_baseline.descriptor_doc)
filler('resource', res_bundle.resource_doc)
filler('datum', datum_doc)
event = copy.deepcopy(raw_event)
assert isinstance(event['data']['image'], str)
filler('event', event)
stop_doc = run_bundle.compose_stop()
filler('stop', stop_doc)
assert event['data']['image'].shape == (5, 5)
assert not filler._closed
# Test NoFiller.
filler = event_model.NoFiller(reg)
filler('start', run_bundle.start_doc)
filler('descriptor', desc_bundle.descriptor_doc)
filler('descriptor', desc_bundle_baseline.descriptor_doc)
filler('resource', res_bundle.resource_doc)
filler('datum', datum_doc)
event = copy.deepcopy(raw_event)
assert isinstance(event['data']['image'], str)
filler('event', event)
# Check that it *hasn't* been filled.
assert isinstance(event['data']['image'], str)
filler('stop', stop_doc)
# Test get_handler() method.
handler = filler.get_handler(res_bundle.resource_doc)
# The method does not expose the internal cache of handlers, so it should
# not return the same instance when called repeatedly.
assert filler.get_handler(res_bundle.resource_doc) is not handler
# Test closing.
filler.close()
with pytest.raises(event_model.EventModelRuntimeError):
filler.get_handler(res_bundle.resource_doc)
with pytest.raises(event_model.EventModelRuntimeError):
filler('stop', stop_doc)
# Test context manager with Event.
with event_model.Filler(reg, inplace=True) as filler:
filler('start', run_bundle.start_doc)
filler('descriptor', desc_bundle.descriptor_doc)
filler('descriptor', desc_bundle_baseline.descriptor_doc)
filler('resource', res_bundle.resource_doc)
filler('datum', datum_doc)
event = copy.deepcopy(raw_event)
name, doc = filler('event', event)
assert name == 'event'
assert doc is event
filler('stop', stop_doc)
assert not filler._closed
assert event['data']['image'].shape == (5, 5)
assert filler._closed
# Test context manager with EventPage.
with event_model.Filler(reg, inplace=True) as filler:
filler('start', run_bundle.start_doc)
filler('descriptor', desc_bundle.descriptor_doc)
filler('descriptor', desc_bundle_baseline.descriptor_doc)
filler('resource', res_bundle.resource_doc)
filler('datum', datum_doc)
event_page = event_model.pack_event_page(copy.deepcopy(raw_event))
name, doc = filler('event_page', event_page)
assert name == 'event_page'
assert doc is event_page
filler('stop', stop_doc)
assert not filler._closed
assert event_page['data']['image'][0].shape == (5, 5)
assert filler._closed
# Test undefined handler spec
with event_model.Filler({}, inplace=True) as filler:
filler('start', run_bundle.start_doc)
filler('descriptor', desc_bundle.descriptor_doc)
filler('descriptor', desc_bundle_baseline.descriptor_doc)
filler('resource', res_bundle.resource_doc)
filler('datum', datum_doc)
event = copy.deepcopy(raw_event)
assert isinstance(event['data']['image'], str)
with pytest.raises(event_model.UndefinedAssetSpecification):
filler('event', event)
# Test exclude and include.
with pytest.raises(ValueError):
event_model.Filler({}, include=[], exclude=[], inplace=True)
with pytest.warns(DeprecationWarning):
with event_model.Filler(reg, exclude=['image'],
inplace=True) as filler:
filler('start', run_bundle.start_doc)
filler('descriptor', desc_bundle.descriptor_doc)
filler('descriptor', desc_bundle_baseline.descriptor_doc)
filler('resource', res_bundle.resource_doc)
filler('datum', datum_doc)
event = copy.deepcopy(raw_event)
assert isinstance(event['data']['image'], str)
filler('event', event)
filler('stop', stop_doc)
with pytest.warns(DeprecationWarning):
with event_model.Filler(reg, include=['image'],
inplace=True) as filler:
filler('start', run_bundle.start_doc)
filler('descriptor', desc_bundle.descriptor_doc)
filler('descriptor', desc_bundle_baseline.descriptor_doc)
filler('resource', res_bundle.resource_doc)
filler('datum', datum_doc)
event = copy.deepcopy(raw_event)
filler('event', event)
filler('stop', stop_doc)
assert not filler._closed
assert event['data']['image'].shape == (5, 5)
with pytest.warns(DeprecationWarning):
with event_model.Filler(reg,
include=['image', 'EXTRA THING'],
inplace=True) as filler:
filler('start', run_bundle.start_doc)
filler('descriptor', desc_bundle.descriptor_doc)
filler('descriptor', desc_bundle_baseline.descriptor_doc)
filler('resource', res_bundle.resource_doc)
filler('datum', datum_doc)
event = copy.deepcopy(raw_event)
filler('event', event)
filler('stop', stop_doc)
assert not filler._closed
assert event['data']['image'].shape == (5, 5)
class DummyHandlerRootMapTest:
def __init__(self, resource_path, a, b):
assert a == 1
assert b == 2
assert resource_path == str(tmp_path / "moved" / "stack.tiff")
def __call__(self, c, d):
assert c == 3
assert d == 4
return numpy.ones((5, 5))
with event_model.Filler({'DUMMY': DummyHandlerRootMapTest},
root_map={path_root: str(tmp_path / "moved")},
inplace=True) as filler:
filler('start', run_bundle.start_doc)
filler('descriptor', desc_bundle.descriptor_doc)
filler('descriptor', desc_bundle_baseline.descriptor_doc)
filler('resource', res_bundle.resource_doc)
filler('datum', datum_doc)
event = copy.deepcopy(raw_event)
filler('event', event)
filler('stop', stop_doc)
assert not filler._closed
assert event['data']['image'].shape == (5, 5)
# Test verify_filled.
with pytest.raises(event_model.UnfilledData):
event_model.verify_filled(event_model.pack_event_page(raw_event))
event_model.verify_filled(event_model.pack_event_page(event))
# Test inplace.
with event_model.Filler(reg, inplace=True) as filler:
filler('start', run_bundle.start_doc)
filler('descriptor', desc_bundle.descriptor_doc)
filler('descriptor', desc_bundle_baseline.descriptor_doc)
filler('resource', res_bundle.resource_doc)
filler('datum', datum_doc)
# Test event()
event = copy.deepcopy(raw_event)
name, filled_event = filler('event', event)
assert filled_event is event
event = copy.deepcopy(raw_event)
# Test fill_event()
filled_event = filler.fill_event(event)
assert filled_event is event
# Test event_page()
event_page = event_model.pack_event_page(copy.deepcopy(raw_event))
_, filled_event_page = filler('event_page', event_page)
assert filled_event_page is event_page
# Test fill_event_page()
event_page = event_model.pack_event_page(copy.deepcopy(raw_event))
filled_event_page = filler.fill_event_page(event_page)
assert filled_event_page is event_page
# Test fill_event and fill_event_page again with inplace=False.
# Test fill_event()
filled_event = filler.fill_event(event, inplace=False)
assert filled_event is not event
# Test fill_event_page()
event_page = event_model.pack_event_page(copy.deepcopy(raw_event))
filled_event_page = filler.fill_event_page(event_page, inplace=False)
assert filled_event_page is not event_page
with event_model.Filler(reg, inplace=False) as filler:
filler('start', run_bundle.start_doc)
filler('descriptor', desc_bundle.descriptor_doc)
filler('descriptor', desc_bundle_baseline.descriptor_doc)
filler('resource', res_bundle.resource_doc)
filler('datum', datum_doc)
event = copy.deepcopy(raw_event)
name, filled_event = filler('event', event)
assert filled_event is not event
assert isinstance(event['data']['image'], str)
event = copy.deepcopy(raw_event)
# Test fill_event()
filled_event = filler.fill_event(event)
assert filled_event is not event
# Test event_page()
event_page = event_model.pack_event_page(copy.deepcopy(raw_event))
_, filled_event_page = filler('event_page', event_page)
assert filled_event_page is not event_page
# Test fill_event_page()
event_page = event_model.pack_event_page(copy.deepcopy(raw_event))
filled_event_page = filler.fill_event_page(event_page)
assert filled_event_page is not event_page
# Test fill_event and fill_event_page again with inplace=True.
# Test fill_event()
filled_event = filler.fill_event(event, inplace=True)
assert filled_event is event
# Test fill_event_page()
event_page = event_model.pack_event_page(copy.deepcopy(raw_event))
filled_event_page = filler.fill_event_page(event_page, inplace=True)
assert filled_event_page is event_page
with pytest.warns(UserWarning):
filler = event_model.Filler(reg)
class OtherDummyHandler:
"Same as DummyHandler, but a different object to test mutating reg"
def __init__(self, resource_path, a, b):
assert a == 1
assert b == 2
assert resource_path == str(tmp_path / "stack.tiff")
def __call__(self, c, d):
assert c == 3
assert d == 4
return numpy.ones((5, 5))
with event_model.Filler(reg, inplace=False) as filler:
with pytest.raises(event_model.EventModelTypeError):
# Updating an existing key fails.
filler.handler_registry['DUMMY'] = OtherDummyHandler
with pytest.raises(event_model.EventModelTypeError):
# Setting a new key fails.
filler.handler_registry['SOMETHING_ELSE'] = OtherDummyHandler
with pytest.raises(event_model.EventModelTypeError):
# Deleting a item fails.
del filler.handler_registry['DUMMY']
filler('start', run_bundle.start_doc)
filler('descriptor', desc_bundle.descriptor_doc)
filler('descriptor', desc_bundle_baseline.descriptor_doc)
filler('resource', res_bundle.resource_doc)
filler('datum', datum_doc)
event = copy.deepcopy(raw_event)
name, filled_event = filler('event', event)
assert filled_event is not event
assert isinstance(event['data']['image'], str)
# Now there should be a handler instance in the cache.
assert filler._handler_cache # implementation detail
with pytest.raises(event_model.DuplicateHandler):
filler.register_handler('DUMMY', OtherDummyHandler)
filler.register_handler('DUMMY', OtherDummyHandler, overwrite=True)
assert filler.handler_registry['DUMMY'] is OtherDummyHandler
# Replacing the handler for a given spec should clear the cache.
assert not filler._handler_cache # implementation detail
# Filling should work the same....
filler('start', run_bundle.start_doc)
filler('descriptor', desc_bundle.descriptor_doc)
filler('descriptor', desc_bundle_baseline.descriptor_doc)
filler('resource', res_bundle.resource_doc)
filler('datum', datum_doc)
event = copy.deepcopy(raw_event)
name, filled_event = filler('event', event)
assert filled_event is not event
assert isinstance(event['data']['image'], str)
filler.deregister_handler('DUMMY')
assert not filler.handler_registry
assert not filler._handler_cache # implementation detail
def export_run(
run,
directory,
*,
external=None,
dry_run=False,
handler_registry=None,
root_map=None,
serializer_class=None,
):
"""
Export one Run.
Parameters
----------
run: BlueskyRun
directory: Union[Str, Manager]
Where files containing documents will be written, or a Manager for
writing to non-file buffers.
external: {None, 'fill', 'ignore')
If None, return the paths to external files.
If 'fill', fill the external data into the Documents.
If 'ignore', do not locate external files.
dry_run: Bool, optional
If True, do not write any files. False by default.
handler_registry: Union[Dict, None]
If None, automatic handler discovery is used.
serializer_class: Serializer, optional
Expected to be a lossless serializer that encodes a format for which
there is a corresponding databroker intake driver. Default (None) is
currently ``suitcase.msgpack.Serializer``, but this may change in the
future. If you want ``suitcase.msgpack.Serializer`` specifically, pass
it in explicitly.
Returns
-------
files: Dict[Str, Set[Str]]
Maps each "root" to a set of absolute file paths.
"""
if serializer_class is None:
import suitcase.msgpack
serializer_class = suitcase.msgpack.Serializer
root_map = root_map or {}
resources = []
files = collections.defaultdict(set)
if handler_registry is None:
handler_registry = databroker.core.discover_handlers()
with event_model.Filler(handler_registry, inplace=False,
root_map=root_map) as filler:
with serializer_class(directory) as serializer:
with tqdm(position=0) as progress:
for name, doc in run.canonical(fill="no"):
if name == "resource":
resources.append(doc)
if external == "fill":
name, doc = filler(name, doc)
if not dry_run:
serializer(name, doc)
progress.update()
if external is None:
for resource in resources:
root = root_map.get(resource["root"], resource["root"])
files[root].update(run.get_file_list(resource))
return serializer.artifacts, dict(files)
def export_run(
run,
directory,
root_hash_func,
*,
external=None,
no_documents=False,
handler_registry=None,
root_map=None,
serializer_class=None,
):
"""
Export one Run.
Parameters
----------
run: BlueskyRun
directory: Union[Str, Manager]
Where files containing documents will be written, or a Manager for
writing to non-file buffers.
external: {None, 'fill', 'ignore')
If None, return the paths to external files.
If 'fill', fill the external data into the Documents.
If 'ignore', do not locate external files.
no_documents: Bool, optional
If True, do not serialize documents. False by default.
handler_registry: Union[Dict, None]
If None, automatic handler discovery is used.
serializer_class: Serializer, optional
Expected to be a lossless serializer that encodes a format for which
there is a corresponding databroker intake driver. Default (None) is
currently ``suitcase.msgpack.Serializer``, but this may change in the
future. If you want ``suitcase.msgpack.Serializer`` specifically, pass
it in explicitly.
Returns
-------
artifacts, files
Notes
-----
* ``artifacts`` maps a human-readable string (typically just ``'all'`` in
this case) to a list of buffers or filepaths where the documents were
serialized.
* ``files`` is the set of filepaths of all external files referenced by
Resource documents, keyed on ``(root_in_document, root, unique_id)``.
"""
EXTERNAL_RELATED_DOCS = ("resource", "datum", "datum_page")
if serializer_class is None:
import suitcase.msgpack
serializer_class = suitcase.msgpack.Serializer
root_map = root_map or {}
files = collections.defaultdict(set)
if handler_registry is None:
handler_registry = databroker.core.discover_handlers()
with event_model.Filler(
handler_registry, inplace=False, root_map=root_map
) as filler:
with serializer_class(
directory, file_prefix="documents/{start[uid]}"
) as serializer:
with tqdm(position=0, unit="documents") as progress:
for name, doc in run.canonical(fill="no"):
if external == "fill":
name, doc = filler(name, doc)
# Omit Resource and Datum[Page] because the data was
# filled in place.
if name in EXTERNAL_RELATED_DOCS:
progress.update()
continue
elif name == "resource":
root = root_map.get(doc["root"], doc["root"])
unique_id = root_hash_func(doc["root"])
if external is None:
if no_documents:
root_in_document = doc["root"]
else:
root_in_document = root
# - root_in_document is the 'root' actually in the
# resource_document
# - root may be different depending on the
# source_catalog configuration, which can map the
# recorded 'root' in the document to some other
# location. This is where we should go looking for
# the data if we plan to copy it.
# - unique_id is unique to this (root, salt)
# combination and used to place the data in a
# unique location.
key = (root_in_document, root, unique_id)
files[key].update(run.get_file_list(doc))
if not no_documents:
# Replace root with a unique ID before serialization.
# We are overriding the local variable name doc here
# (yuck!) so that serializer(name, doc) below works on
# all document types.
doc = doc.copy()
doc["root"] = unique_id
if not no_documents:
serializer(name, doc)
progress.update()
return serializer.artifacts, dict(files)
