def test_canonical_unfilled(bundle): run = bundle.cat['xyz']()[bundle.uid] run.canonical_unfilled() def sorted_actual(): for name_ in ('start', 'descriptor', 'resource', 'datum', 'event_page', 'event', 'stop'): for name, doc in bundle.docs: if name == name_ and name in ('start', 'descriptor', 'event', 'event_page', 'stop'): yield name, doc raw_run = [(name, doc) for name, doc in list(run.canonical_unfilled()) if name not in ('resource', 'datum', 'datum_page')] for actual, expected in itertools.zip_longest(raw_run, sorted_actual()): actual_name, actual_doc = actual expected_name, expected_doc = expected print(actual_name, expected_name) try: assert actual_name == expected_name except ValueError: assert numpy.array_equal(actual_doc, expected_doc) # Passing the run through the filler to check resource and datum are # received before corresponding event. filler = event_model.Filler({'NPY_SEQ': ophyd.sim.NumpySeqHandler}, inplace=False) for name, doc in run.canonical_unfilled(): filler(name, doc)
def __init__(self, handler_registry=None, query=None, **kwargs): """ This Catalog is backed by Python collections in memory. Subclasses should define a ``_load`` method (same as any intake Catalog) that calls this class's ``upsert`` method (which is particular to this class). Parameters ---------- handler_registry : dict, optional Maps each asset spec to a handler class or a string specifying the module name and class name, as in (for example) ``{'SOME_SPEC': 'module.submodule.class_name'}``. query : dict, optional Mongo query that filters entries' RunStart documents **kwargs : Additional keyword arguments are passed through to the base class, Catalog. """ self._query = query or {} if handler_registry is None: handler_registry = {} parsed_handler_registry = parse_handler_registry(handler_registry) self.filler = event_model.Filler(parsed_handler_registry, inplace=True) self._uid_to_run_start_doc = {} super().__init__(**kwargs)
def test_canonical(bundle): run = bundle.cat['xyz']()[bundle.uid] # Smoke test for back-compat alias with pytest.warns(UserWarning): next(run.read_canonical()) filler = event_model.Filler({'NPY_SEQ': ophyd.sim.NumpySeqHandler}, inplace=False) def sorted_actual(): for name_ in ('start', 'descriptor', 'resource', 'datum', 'event_page', 'event', 'stop'): for name, doc in bundle.docs: # Fill external data. _, filled_doc = filler(name, doc) if name == name_ and name in ('start', 'descriptor', 'event', 'event_page', 'stop'): yield name, filled_doc for actual, expected in itertools.zip_longest(run.canonical(), sorted_actual()): actual_name, actual_doc = actual expected_name, expected_doc = expected print(actual_name, expected_name) try: assert actual_name == expected_name except ValueError: assert numpy.array_equal(actual_doc, expected_doc)
def test_root_map(): new_path = pathlib.Path("/another_placeholder", "moved") class DummyHandlerRootMapTest: def __init__(self, resource_path, a, b): assert a == 1 assert b == 2 assert resource_path == str(new_path / "stack.tiff") def __call__(self, c, d): assert c == 3 assert d == 4 return numpy.ones((5, 5)) with event_model.Filler({'DUMMY': DummyHandlerRootMapTest}, root_map={str(path_root): str(new_path)}, inplace=True) as filler: filler('start', run_bundle.start_doc) filler('descriptor', desc_bundle.descriptor_doc) filler('descriptor', desc_bundle_baseline.descriptor_doc) filler('resource', res_bundle.resource_doc) filler('datum', datum_doc) event = copy.deepcopy(raw_event) filler('event', event) filler('stop', stop_doc) assert not filler.closed assert event['data']['image'].shape == (5, 5)
def __init__(self, file_list, handler, *, query=None, **kwargs): """ This Catalog is backed by a newline-delimited JSON (jsonl) file. Each line of the file is expected to be a JSON list with two elements, the document name (type) and the document itself. The documents are expected to be in chronological order. Parameters ---------- file_list : list list of filepaths handler_registry : dict, optional Maps each asset spec to a handler class or a string specifying the module name and class name, as in (for example) ``{'SOME_SPEC': 'module.submodule.class_name'}``. **kwargs : Additional keyword arguments are passed through to the base class, Catalog. """ name = 'bluesky-files-catalog' # noqa self._runs = {} # This maps run_start_uids to file paths. self._run_starts = {} # This maps run_start_uids to run_start_docs. self._query = query or {} self.handler = handler self.filler = event_model.Filler({handler.name: handler}) self._update_index(file_list) super().__init__(**kwargs)
def test_include_and_exclude(): with pytest.raises(ValueError): event_model.Filler({}, include=[], exclude=[], inplace=True) with pytest.warns(DeprecationWarning): with event_model.Filler(reg, exclude=['image'], inplace=True) as filler: filler('start', run_bundle.start_doc) filler('descriptor', desc_bundle.descriptor_doc) filler('descriptor', desc_bundle_baseline.descriptor_doc) filler('resource', res_bundle.resource_doc) filler('datum', datum_doc) event = copy.deepcopy(raw_event) assert isinstance(event['data']['image'], str) filler('event', event) filler('stop', stop_doc) with pytest.warns(DeprecationWarning): with event_model.Filler(reg, include=['image'], inplace=True) as filler: filler('start', run_bundle.start_doc) filler('descriptor', desc_bundle.descriptor_doc) filler('descriptor', desc_bundle_baseline.descriptor_doc) filler('resource', res_bundle.resource_doc) filler('datum', datum_doc) event = copy.deepcopy(raw_event) filler('event', event) filler('stop', stop_doc) assert not filler.closed assert event['data']['image'].shape == (5, 5) with pytest.warns(DeprecationWarning): with event_model.Filler(reg, include=['image', 'EXTRA THING'], inplace=True) as filler: filler('start', run_bundle.start_doc) filler('descriptor', desc_bundle.descriptor_doc) filler('descriptor', desc_bundle_baseline.descriptor_doc) filler('resource', res_bundle.resource_doc) filler('datum', datum_doc) event = copy.deepcopy(raw_event) filler('event', event) filler('stop', stop_doc) assert not filler.closed assert event['data']['image'].shape == (5, 5)
def test_handler_registry_access(): "Test the handler_registery can be viewed but is immutable." class OtherDummyHandler: "Same as DummyHandler, but a different object to test mutating reg" def __init__(self, resource_path, a, b): assert a == 1 assert b == 2 assert resource_path == str(path_root / "stack.tiff") def __call__(self, c, d): assert c == 3 assert d == 4 return numpy.ones((5, 5)) with event_model.Filler(reg, inplace=False) as filler: with pytest.raises(event_model.EventModelTypeError): # Updating an existing key fails. filler.handler_registry['DUMMY'] = OtherDummyHandler with pytest.raises(event_model.EventModelTypeError): # Setting a new key fails. filler.handler_registry['SOMETHING_ELSE'] = OtherDummyHandler with pytest.raises(event_model.EventModelTypeError): # Deleting a item fails. del filler.handler_registry['DUMMY'] filler('start', run_bundle.start_doc) filler('descriptor', desc_bundle.descriptor_doc) filler('descriptor', desc_bundle_baseline.descriptor_doc) filler('resource', res_bundle.resource_doc) filler('datum', datum_doc) event = copy.deepcopy(raw_event) name, filled_event = filler('event', event) assert filled_event is not event assert isinstance(event['data']['image'], str) # Now there should be a handler instance in the cache. assert filler._handler_cache # implementation detail with pytest.raises(event_model.DuplicateHandler): filler.register_handler('DUMMY', OtherDummyHandler) filler.register_handler('DUMMY', OtherDummyHandler, overwrite=True) assert filler.handler_registry['DUMMY'] is OtherDummyHandler # Replacing the handler for a given spec should clear the cache. assert not filler._handler_cache # implementation detail # Filling should work the same.... filler('start', run_bundle.start_doc) filler('descriptor', desc_bundle.descriptor_doc) filler('descriptor', desc_bundle_baseline.descriptor_doc) filler('resource', res_bundle.resource_doc) filler('datum', datum_doc) event = copy.deepcopy(raw_event) name, filled_event = filler('event', event) assert filled_event is not event assert isinstance(event['data']['image'], str) handler = filler.deregister_handler('DUMMY') assert handler is OtherDummyHandler assert not filler.handler_registry assert not filler._handler_cache # implementation detail
def test_fill(data_files): import event_model with event_model.Filler({TIFSeriesPlugin.name: TIFSeriesPlugin}) as filler: doc_stream = TIFSeriesPlugin.ingest(data_files) for name, doc in doc_stream: name, doc = filler(name, doc) doc['filled'] = {'image': False} if name == 'event': assert isinstance(doc['data']['image'], np.ndarray)
def test_undefined_handler_spec(): "Check failure path when an unknown spec is found in a resource." with event_model.Filler({}, inplace=True) as filler: filler('start', run_bundle.start_doc) filler('descriptor', desc_bundle.descriptor_doc) filler('descriptor', desc_bundle_baseline.descriptor_doc) filler('resource', res_bundle.resource_doc) filler('datum', datum_doc) event = copy.deepcopy(raw_event) assert isinstance(event['data']['image'], str) with pytest.raises(event_model.UndefinedAssetSpecification): filler('event', event)
def test_canonical(bundle): run = bundle.cat['xyz']()[bundle.uid] filler = event_model.Filler({'NPY_SEQ': ophyd.sim.NumpySeqHandler}, inplace=False) # Smoke test for back-compat alias with pytest.warns(UserWarning): next(run.read_canonical()) compare(run.canonical(fill='yes'), (filler(name, doc) for name, doc in bundle.docs))
def test_canonical_delayed(bundle): run = bundle.cat['xyz']()[bundle.uid] filler = event_model.Filler({'NPY_SEQ': ophyd.sim.NumpySeqHandler}, inplace=False) if bundle.remote: with pytest.raises(NotImplementedError): next(run.canonical(fill='delayed')) else: compare(run.canonical(fill='delayed'), (filler(name, doc) for name, doc in bundle.docs))
def test_no_descriptors(): run_bundle = event_model.compose_run() start_doc = run_bundle.start_doc stop_doc = run_bundle.compose_stop() documents_to_xarray(start_doc=start_doc, stop_doc=stop_doc, descriptor_docs=[], get_event_pages=no_event_pages, filler=event_model.Filler({}, inplace=True), get_resource=None, lookup_resource_for_datum=None, get_datum_pages=None)
def test_canonical_unfilled(bundle): run = bundle.cat['xyz']()[bundle.uid] run.canonical(fill='no') compare(run.canonical(fill='no'), bundle.docs) # Passing the run through the filler to check resource and datum are # received before corresponding event. filler = event_model.Filler({'NPY_SEQ': ophyd.sim.NumpySeqHandler}, inplace=False) for name, doc in run.canonical(fill='no'): filler(name, doc)
def __init__(self, gen_func, gen_args, gen_kwargs, filler=None, **kwargs): if filler is None: filler = event_model.Filler({}, inplace=True) document_cache = DocumentCache() for item in gen_func(*gen_args, **gen_kwargs): document_cache(*item) assert document_cache.start_doc is not None def get_run_start(): return document_cache.start_doc def get_run_stop(): return document_cache.stop_doc def get_event_descriptors(): return document_cache.descriptors.values() def get_event_pages(descriptor_uid, skip=0, limit=None): if skip != 0 and limit is not None: raise NotImplementedError return document_cache.event_pages[descriptor_uid] def get_event_count(descriptor_uid): return sum(len(page['seq_num']) for page in (document_cache.event_pages[descriptor_uid])) def get_resource(uid): return document_cache.resources[uid] def lookup_resource_for_datum(datum_id): return document_cache.resource_uid_by_datum_id[datum_id] def get_datum_pages(resource_uid, skip=0, limit=None): if skip != 0 and limit is not None: raise NotImplementedError return document_cache.datum_pages_by_resource[resource_uid] super().__init__( get_run_start=get_run_start, get_run_stop=get_run_stop, get_event_descriptors=get_event_descriptors, get_event_pages=get_event_pages, get_event_count=get_event_count, get_resource=get_resource, lookup_resource_for_datum=lookup_resource_for_datum, get_datum_pages=get_datum_pages, filler=filler, **kwargs)
def test_canonical(bundle): "This is a deprecated synonym for documents()." run = bundle.cat['xyz']()[bundle.uid] filler = event_model.Filler({'NPY_SEQ': ophyd.sim.NumpySeqHandler}, inplace=False) # Smoke test for back-compat alias with pytest.warns(UserWarning): next(run.canonical(fill="yes")) compare(run.documents(fill='yes'), (filler(name, doc) for name, doc in bundle.docs))
def filler(): filler = event_model.Filler(reg, inplace=True) filler('start', run_bundle.start_doc) filler('descriptor', desc_bundle.descriptor_doc) filler('descriptor', desc_bundle_baseline.descriptor_doc) filler('resource', res_bundle.resource_doc) filler('datum', datum_doc) event = copy.deepcopy(raw_event) assert isinstance(event['data']['image'], str) filler('event', event) assert event['data']['image'].shape == (5, 5) filler('stop', stop_doc) assert not filler.closed return filler
def test_context_manager_with_event_page(): with event_model.Filler(reg, inplace=True) as filler: filler('start', run_bundle.start_doc) filler('descriptor', desc_bundle.descriptor_doc) filler('descriptor', desc_bundle_baseline.descriptor_doc) filler('resource', res_bundle.resource_doc) filler('datum', datum_doc) event_page = event_model.pack_event_page(copy.deepcopy(raw_event)) name, doc = filler('event_page', event_page) assert name == 'event_page' assert doc is event_page filler('stop', stop_doc) assert not filler.closed assert event_page['data']['image'][0].shape == (5, 5) assert filler.closed
def test_context_manager(): "Construct Filler as a context manager." with event_model.Filler(reg, inplace=True) as filler: filler('start', run_bundle.start_doc) filler('descriptor', desc_bundle.descriptor_doc) filler('descriptor', desc_bundle_baseline.descriptor_doc) filler('resource', res_bundle.resource_doc) filler('datum', datum_doc) event = copy.deepcopy(raw_event) name, doc = filler('event', event) assert name == 'event' assert doc is event filler('stop', stop_doc) assert not filler.closed assert event['data']['image'].shape == (5, 5) assert filler.closed
def test_no_events(): run_bundle = event_model.compose_run() start_doc = run_bundle.start_doc desc_bundle = run_bundle.compose_descriptor( data_keys={'x': {'source': '...', 'shape': [], 'dtype': 'number'}}, name='primary') descriptor_doc = desc_bundle.descriptor_doc stop_doc = run_bundle.compose_stop() documents_to_xarray( start_doc=start_doc, stop_doc=stop_doc, descriptor_docs=[descriptor_doc], get_event_pages=no_event_pages, filler=event_model.Filler({}, inplace=True), get_resource=None, lookup_resource_for_datum=None, get_datum_pages=None)
def __init__(self, datastore_db, *, handler_registry=None, query=None, **kwargs): """ This Catalog is backed by a MongoDB with an embedded data model. This embedded data model has three collections: header, event, datum. The header collection includes start, stop, descriptor, and resource documents. The event_pages are stored in the event colleciton, and datum_pages are stored in the datum collection. Parameters ---------- datastore_db : pymongo.database.Database or string Must be a Database or a URI string that includes a database name. handler_registry : dict, optional Maps each asset spec to a handler class or a string specifying the module name and class name, as in (for example) ``{'SOME_SPEC': 'module.submodule.class_name'}``. query : dict, optional MongoDB query. Used internally by the ``search()`` method. **kwargs : Additional keyword arguments are passed through to the base class, Catalog. """ name = 'bluesky-mongo-embedded-catalog' # noqa if isinstance(datastore_db, str): self._db = _get_database(datastore_db) else: self._db = datastore_db self._query = query or {} if handler_registry is None: handler_registry = {} parsed_handler_registry = parse_handler_registry(handler_registry) self.filler = event_model.Filler(parsed_handler_registry, inplace=True) super().__init__(**kwargs)
def test_mismatched_data_keys(): "Test that we raise specifically when data keys do not match between event and descriptor." with pytest.raises(event_model.MismatchedDataKeys): with event_model.NoFiller(reg) as filler: filler('start', run_bundle.start_doc) filler('descriptor', desc_bundle.descriptor_doc) filler('descriptor', desc_bundle_baseline.descriptor_doc) filler('resource', res_bundle.resource_doc) filler('datum', datum_doc) event = copy.deepcopy(raw_event) del event['data']['image'] filler('event', event) with pytest.raises(event_model.MismatchedDataKeys): with event_model.Filler(reg, inplace=False) as filler: filler('start', run_bundle.start_doc) filler('descriptor', desc_bundle.descriptor_doc) filler('descriptor', desc_bundle_baseline.descriptor_doc) filler('resource', res_bundle.resource_doc) filler('datum', datum_doc) event = copy.deepcopy(raw_event) del event['data']['image'] filler('event', event)
def test_inplace(): "Test the behavior of the 'inplace' parameter." with event_model.Filler(reg, inplace=True) as filler: filler('start', run_bundle.start_doc) filler('descriptor', desc_bundle.descriptor_doc) filler('descriptor', desc_bundle_baseline.descriptor_doc) filler('resource', res_bundle.resource_doc) filler('datum', datum_doc) # Test event() event = copy.deepcopy(raw_event) name, filled_event = filler('event', event) assert filled_event is event event = copy.deepcopy(raw_event) # Test fill_event() filled_event = filler.fill_event(event) assert filled_event is event # Test event_page() event_page = event_model.pack_event_page(copy.deepcopy(raw_event)) _, filled_event_page = filler('event_page', event_page) assert filled_event_page is event_page # Test fill_event_page() event_page = event_model.pack_event_page(copy.deepcopy(raw_event)) filled_event_page = filler.fill_event_page(event_page) assert filled_event_page is event_page # Test fill_event and fill_event_page again with inplace=False. # Test fill_event() filled_event = filler.fill_event(event, inplace=False) assert filled_event is not event # Test fill_event_page() event_page = event_model.pack_event_page(copy.deepcopy(raw_event)) filled_event_page = filler.fill_event_page(event_page, inplace=False) assert filled_event_page is not event_page with event_model.Filler(reg, inplace=False) as filler: filler('start', run_bundle.start_doc) filler('descriptor', desc_bundle.descriptor_doc) filler('descriptor', desc_bundle_baseline.descriptor_doc) filler('resource', res_bundle.resource_doc) filler('datum', datum_doc) event = copy.deepcopy(raw_event) name, filled_event = filler('event', event) assert filled_event is not event assert isinstance(event['data']['image'], str) event = copy.deepcopy(raw_event) # Test fill_event() filled_event = filler.fill_event(event) assert filled_event is not event # Test event_page() event_page = event_model.pack_event_page(copy.deepcopy(raw_event)) _, filled_event_page = filler('event_page', event_page) assert filled_event_page is not event_page # Test fill_event_page() event_page = event_model.pack_event_page(copy.deepcopy(raw_event)) filled_event_page = filler.fill_event_page(event_page) assert filled_event_page is not event_page # Test fill_event and fill_event_page again with inplace=True. # Test fill_event() filled_event = filler.fill_event(event, inplace=True) assert filled_event is event # Test fill_event_page() event_page = event_model.pack_event_page(copy.deepcopy(raw_event)) filled_event_page = filler.fill_event_page(event_page, inplace=True) assert filled_event_page is event_page with pytest.warns(UserWarning): # warnings because inplace is not specified filler = event_model.Filler(reg)
def test_pickle_filler(): filler = event_model.Filler({}, inplace=False) serialized = pickle.dumps(filler) deserialized = pickle.loads(serialized) assert filler == deserialized
def test_retry_intervals_input_normalization(retry_intervals): filler = event_model.Filler({}, retry_intervals=retry_intervals, inplace=False) assert isinstance(filler.retry_intervals, list)
def test_filler(tmp_path): class DummyHandler: def __init__(self, resource_path, a, b): assert a == 1 assert b == 2 assert resource_path == str(tmp_path / "stack.tiff") def __call__(self, c, d): assert c == 3 assert d == 4 return numpy.ones((5, 5)) path_root = str(tmp_path) reg = {'DUMMY': DummyHandler} filler = event_model.Filler(reg) run_bundle = event_model.compose_run() desc_bundle = run_bundle.compose_descriptor(data_keys={ 'motor': { 'shape': [], 'dtype': 'number', 'source': '...' }, 'image': { 'shape': [512, 512], 'dtype': 'number', 'source': '...', 'external': 'FILESTORE:' } }, name='primary') desc_bundle_baseline = run_bundle.compose_descriptor( data_keys={'motor': { 'shape': [], 'dtype': 'number', 'source': '...' }}, name='baseline') res_bundle = run_bundle.compose_resource(spec='DUMMY', root=path_root, resource_path='stack.tiff', resource_kwargs={ 'a': 1, 'b': 2 }) datum_doc = res_bundle.compose_datum(datum_kwargs={'c': 3, 'd': 4}) raw_event = desc_bundle.compose_event(data={ 'motor': 0, 'image': datum_doc['datum_id'] }, timestamps={ 'motor': 0, 'image': 0 }, filled={'image': False}, seq_num=1) filler('start', run_bundle.start_doc) filler('descriptor', desc_bundle.descriptor_doc) filler('descriptor', desc_bundle_baseline.descriptor_doc) filler('resource', res_bundle.resource_doc) filler('datum', datum_doc) event = copy.deepcopy(raw_event) assert isinstance(event['data']['image'], str) filler('event', event) stop_doc = run_bundle.compose_stop() filler('stop', stop_doc) assert event['data']['image'].shape == (5, 5) assert not filler._closed filler.close() assert filler._closed # Test context manager with Event. with event_model.Filler(reg) as filler: filler('start', run_bundle.start_doc) filler('descriptor', desc_bundle.descriptor_doc) filler('descriptor', desc_bundle_baseline.descriptor_doc) filler('resource', res_bundle.resource_doc) filler('datum', datum_doc) event = copy.deepcopy(raw_event) name, doc = filler('event', event) assert name == 'event' assert doc is event filler('stop', stop_doc) assert not filler._closed assert event['data']['image'].shape == (5, 5) assert filler._closed # Test context manager with EventPage. with event_model.Filler(reg) as filler: filler('start', run_bundle.start_doc) filler('descriptor', desc_bundle.descriptor_doc) filler('descriptor', desc_bundle_baseline.descriptor_doc) filler('resource', res_bundle.resource_doc) filler('datum', datum_doc) event_page = event_model.pack_event_page(copy.deepcopy(raw_event)) name, doc = filler('event_page', event_page) assert name == 'event_page' assert doc is event_page filler('stop', stop_doc) assert not filler._closed assert event_page['data']['image'][0].shape == (5, 5) assert filler._closed # Test undefined handler spec with event_model.Filler({}) as filler: filler('start', run_bundle.start_doc) filler('descriptor', desc_bundle.descriptor_doc) filler('descriptor', desc_bundle_baseline.descriptor_doc) filler('resource', res_bundle.resource_doc) filler('datum', datum_doc) event = copy.deepcopy(raw_event) assert isinstance(event['data']['image'], str) with pytest.raises(event_model.UndefinedAssetSpecification): filler('event', event) # Test exclude and include. with pytest.raises(ValueError): event_model.Filler({}, include=[], exclude=[]) with event_model.Filler(reg, exclude=['image']) as filler: filler('start', run_bundle.start_doc) filler('descriptor', desc_bundle.descriptor_doc) filler('descriptor', desc_bundle_baseline.descriptor_doc) filler('resource', res_bundle.resource_doc) filler('datum', datum_doc) event = copy.deepcopy(raw_event) assert isinstance(event['data']['image'], str) filler('event', event) filler('stop', stop_doc) with event_model.Filler(reg, include=['image']) as filler: filler('start', run_bundle.start_doc) filler('descriptor', desc_bundle.descriptor_doc) filler('descriptor', desc_bundle_baseline.descriptor_doc) filler('resource', res_bundle.resource_doc) filler('datum', datum_doc) event = copy.deepcopy(raw_event) filler('event', event) filler('stop', stop_doc) assert not filler._closed assert event['data']['image'].shape == (5, 5) with event_model.Filler(reg, include=['image', 'EXTRA THING']) as filler: filler('start', run_bundle.start_doc) filler('descriptor', desc_bundle.descriptor_doc) filler('descriptor', desc_bundle_baseline.descriptor_doc) filler('resource', res_bundle.resource_doc) filler('datum', datum_doc) event = copy.deepcopy(raw_event) filler('event', event) filler('stop', stop_doc) assert not filler._closed assert event['data']['image'].shape == (5, 5) class DummyHandlerRootMapTest: def __init__(self, resource_path, a, b): assert a == 1 assert b == 2 assert resource_path == str(tmp_path / "moved" / "stack.tiff") def __call__(self, c, d): assert c == 3 assert d == 4 return numpy.ones((5, 5)) with event_model.Filler({'DUMMY': DummyHandlerRootMapTest}, root_map={path_root: str(tmp_path / "moved")}) as filler: filler('start', run_bundle.start_doc) filler('descriptor', desc_bundle.descriptor_doc) filler('descriptor', desc_bundle_baseline.descriptor_doc) filler('resource', res_bundle.resource_doc) filler('datum', datum_doc) event = copy.deepcopy(raw_event) filler('event', event) filler('stop', stop_doc) assert not filler._closed assert event['data']['image'].shape == (5, 5) # Test verify_filled. with pytest.raises(event_model.UnfilledData): event_model.verify_filled(event_model.pack_event_page(raw_event)) event_model.verify_filled(event_model.pack_event_page(event))
def test_filler(tmp_path): class DummyHandler: def __init__(self, resource_path, a, b): assert a == 1 assert b == 2 assert resource_path == str(tmp_path / "stack.tiff") def __call__(self, c, d): assert c == 3 assert d == 4 return numpy.ones((5, 5)) path_root = str(tmp_path) reg = {'DUMMY': DummyHandler} filler = event_model.Filler(reg, inplace=True) run_bundle = event_model.compose_run() desc_bundle = run_bundle.compose_descriptor(data_keys={ 'motor': { 'shape': [], 'dtype': 'number', 'source': '...' }, 'image': { 'shape': [512, 512], 'dtype': 'number', 'source': '...', 'external': 'FILESTORE:' } }, name='primary') desc_bundle_baseline = run_bundle.compose_descriptor( data_keys={'motor': { 'shape': [], 'dtype': 'number', 'source': '...' }}, name='baseline') res_bundle = run_bundle.compose_resource(spec='DUMMY', root=path_root, resource_path='stack.tiff', resource_kwargs={ 'a': 1, 'b': 2 }) datum_doc = res_bundle.compose_datum(datum_kwargs={'c': 3, 'd': 4}) raw_event = desc_bundle.compose_event(data={ 'motor': 0, 'image': datum_doc['datum_id'] }, timestamps={ 'motor': 0, 'image': 0 }, filled={'image': False}, seq_num=1) filler('start', run_bundle.start_doc) filler('descriptor', desc_bundle.descriptor_doc) filler('descriptor', desc_bundle_baseline.descriptor_doc) filler('resource', res_bundle.resource_doc) filler('datum', datum_doc) event = copy.deepcopy(raw_event) assert isinstance(event['data']['image'], str) filler('event', event) stop_doc = run_bundle.compose_stop() filler('stop', stop_doc) assert event['data']['image'].shape == (5, 5) assert not filler._closed # Test NoFiller. filler = event_model.NoFiller(reg) filler('start', run_bundle.start_doc) filler('descriptor', desc_bundle.descriptor_doc) filler('descriptor', desc_bundle_baseline.descriptor_doc) filler('resource', res_bundle.resource_doc) filler('datum', datum_doc) event = copy.deepcopy(raw_event) assert isinstance(event['data']['image'], str) filler('event', event) # Check that it *hasn't* been filled. assert isinstance(event['data']['image'], str) filler('stop', stop_doc) # Test get_handler() method. handler = filler.get_handler(res_bundle.resource_doc) # The method does not expose the internal cache of handlers, so it should # not return the same instance when called repeatedly. assert filler.get_handler(res_bundle.resource_doc) is not handler # Test closing. filler.close() with pytest.raises(event_model.EventModelRuntimeError): filler.get_handler(res_bundle.resource_doc) with pytest.raises(event_model.EventModelRuntimeError): filler('stop', stop_doc) # Test context manager with Event. with event_model.Filler(reg, inplace=True) as filler: filler('start', run_bundle.start_doc) filler('descriptor', desc_bundle.descriptor_doc) filler('descriptor', desc_bundle_baseline.descriptor_doc) filler('resource', res_bundle.resource_doc) filler('datum', datum_doc) event = copy.deepcopy(raw_event) name, doc = filler('event', event) assert name == 'event' assert doc is event filler('stop', stop_doc) assert not filler._closed assert event['data']['image'].shape == (5, 5) assert filler._closed # Test context manager with EventPage. with event_model.Filler(reg, inplace=True) as filler: filler('start', run_bundle.start_doc) filler('descriptor', desc_bundle.descriptor_doc) filler('descriptor', desc_bundle_baseline.descriptor_doc) filler('resource', res_bundle.resource_doc) filler('datum', datum_doc) event_page = event_model.pack_event_page(copy.deepcopy(raw_event)) name, doc = filler('event_page', event_page) assert name == 'event_page' assert doc is event_page filler('stop', stop_doc) assert not filler._closed assert event_page['data']['image'][0].shape == (5, 5) assert filler._closed # Test undefined handler spec with event_model.Filler({}, inplace=True) as filler: filler('start', run_bundle.start_doc) filler('descriptor', desc_bundle.descriptor_doc) filler('descriptor', desc_bundle_baseline.descriptor_doc) filler('resource', res_bundle.resource_doc) filler('datum', datum_doc) event = copy.deepcopy(raw_event) assert isinstance(event['data']['image'], str) with pytest.raises(event_model.UndefinedAssetSpecification): filler('event', event) # Test exclude and include. with pytest.raises(ValueError): event_model.Filler({}, include=[], exclude=[], inplace=True) with pytest.warns(DeprecationWarning): with event_model.Filler(reg, exclude=['image'], inplace=True) as filler: filler('start', run_bundle.start_doc) filler('descriptor', desc_bundle.descriptor_doc) filler('descriptor', desc_bundle_baseline.descriptor_doc) filler('resource', res_bundle.resource_doc) filler('datum', datum_doc) event = copy.deepcopy(raw_event) assert isinstance(event['data']['image'], str) filler('event', event) filler('stop', stop_doc) with pytest.warns(DeprecationWarning): with event_model.Filler(reg, include=['image'], inplace=True) as filler: filler('start', run_bundle.start_doc) filler('descriptor', desc_bundle.descriptor_doc) filler('descriptor', desc_bundle_baseline.descriptor_doc) filler('resource', res_bundle.resource_doc) filler('datum', datum_doc) event = copy.deepcopy(raw_event) filler('event', event) filler('stop', stop_doc) assert not filler._closed assert event['data']['image'].shape == (5, 5) with pytest.warns(DeprecationWarning): with event_model.Filler(reg, include=['image', 'EXTRA THING'], inplace=True) as filler: filler('start', run_bundle.start_doc) filler('descriptor', desc_bundle.descriptor_doc) filler('descriptor', desc_bundle_baseline.descriptor_doc) filler('resource', res_bundle.resource_doc) filler('datum', datum_doc) event = copy.deepcopy(raw_event) filler('event', event) filler('stop', stop_doc) assert not filler._closed assert event['data']['image'].shape == (5, 5) class DummyHandlerRootMapTest: def __init__(self, resource_path, a, b): assert a == 1 assert b == 2 assert resource_path == str(tmp_path / "moved" / "stack.tiff") def __call__(self, c, d): assert c == 3 assert d == 4 return numpy.ones((5, 5)) with event_model.Filler({'DUMMY': DummyHandlerRootMapTest}, root_map={path_root: str(tmp_path / "moved")}, inplace=True) as filler: filler('start', run_bundle.start_doc) filler('descriptor', desc_bundle.descriptor_doc) filler('descriptor', desc_bundle_baseline.descriptor_doc) filler('resource', res_bundle.resource_doc) filler('datum', datum_doc) event = copy.deepcopy(raw_event) filler('event', event) filler('stop', stop_doc) assert not filler._closed assert event['data']['image'].shape == (5, 5) # Test verify_filled. with pytest.raises(event_model.UnfilledData): event_model.verify_filled(event_model.pack_event_page(raw_event)) event_model.verify_filled(event_model.pack_event_page(event)) # Test inplace. with event_model.Filler(reg, inplace=True) as filler: filler('start', run_bundle.start_doc) filler('descriptor', desc_bundle.descriptor_doc) filler('descriptor', desc_bundle_baseline.descriptor_doc) filler('resource', res_bundle.resource_doc) filler('datum', datum_doc) # Test event() event = copy.deepcopy(raw_event) name, filled_event = filler('event', event) assert filled_event is event event = copy.deepcopy(raw_event) # Test fill_event() filled_event = filler.fill_event(event) assert filled_event is event # Test event_page() event_page = event_model.pack_event_page(copy.deepcopy(raw_event)) _, filled_event_page = filler('event_page', event_page) assert filled_event_page is event_page # Test fill_event_page() event_page = event_model.pack_event_page(copy.deepcopy(raw_event)) filled_event_page = filler.fill_event_page(event_page) assert filled_event_page is event_page # Test fill_event and fill_event_page again with inplace=False. # Test fill_event() filled_event = filler.fill_event(event, inplace=False) assert filled_event is not event # Test fill_event_page() event_page = event_model.pack_event_page(copy.deepcopy(raw_event)) filled_event_page = filler.fill_event_page(event_page, inplace=False) assert filled_event_page is not event_page with event_model.Filler(reg, inplace=False) as filler: filler('start', run_bundle.start_doc) filler('descriptor', desc_bundle.descriptor_doc) filler('descriptor', desc_bundle_baseline.descriptor_doc) filler('resource', res_bundle.resource_doc) filler('datum', datum_doc) event = copy.deepcopy(raw_event) name, filled_event = filler('event', event) assert filled_event is not event assert isinstance(event['data']['image'], str) event = copy.deepcopy(raw_event) # Test fill_event() filled_event = filler.fill_event(event) assert filled_event is not event # Test event_page() event_page = event_model.pack_event_page(copy.deepcopy(raw_event)) _, filled_event_page = filler('event_page', event_page) assert filled_event_page is not event_page # Test fill_event_page() event_page = event_model.pack_event_page(copy.deepcopy(raw_event)) filled_event_page = filler.fill_event_page(event_page) assert filled_event_page is not event_page # Test fill_event and fill_event_page again with inplace=True. # Test fill_event() filled_event = filler.fill_event(event, inplace=True) assert filled_event is event # Test fill_event_page() event_page = event_model.pack_event_page(copy.deepcopy(raw_event)) filled_event_page = filler.fill_event_page(event_page, inplace=True) assert filled_event_page is event_page with pytest.warns(UserWarning): filler = event_model.Filler(reg) class OtherDummyHandler: "Same as DummyHandler, but a different object to test mutating reg" def __init__(self, resource_path, a, b): assert a == 1 assert b == 2 assert resource_path == str(tmp_path / "stack.tiff") def __call__(self, c, d): assert c == 3 assert d == 4 return numpy.ones((5, 5)) with event_model.Filler(reg, inplace=False) as filler: with pytest.raises(event_model.EventModelTypeError): # Updating an existing key fails. filler.handler_registry['DUMMY'] = OtherDummyHandler with pytest.raises(event_model.EventModelTypeError): # Setting a new key fails. filler.handler_registry['SOMETHING_ELSE'] = OtherDummyHandler with pytest.raises(event_model.EventModelTypeError): # Deleting a item fails. del filler.handler_registry['DUMMY'] filler('start', run_bundle.start_doc) filler('descriptor', desc_bundle.descriptor_doc) filler('descriptor', desc_bundle_baseline.descriptor_doc) filler('resource', res_bundle.resource_doc) filler('datum', datum_doc) event = copy.deepcopy(raw_event) name, filled_event = filler('event', event) assert filled_event is not event assert isinstance(event['data']['image'], str) # Now there should be a handler instance in the cache. assert filler._handler_cache # implementation detail with pytest.raises(event_model.DuplicateHandler): filler.register_handler('DUMMY', OtherDummyHandler) filler.register_handler('DUMMY', OtherDummyHandler, overwrite=True) assert filler.handler_registry['DUMMY'] is OtherDummyHandler # Replacing the handler for a given spec should clear the cache. assert not filler._handler_cache # implementation detail # Filling should work the same.... filler('start', run_bundle.start_doc) filler('descriptor', desc_bundle.descriptor_doc) filler('descriptor', desc_bundle_baseline.descriptor_doc) filler('resource', res_bundle.resource_doc) filler('datum', datum_doc) event = copy.deepcopy(raw_event) name, filled_event = filler('event', event) assert filled_event is not event assert isinstance(event['data']['image'], str) filler.deregister_handler('DUMMY') assert not filler.handler_registry assert not filler._handler_cache # implementation detail
def export_run( run, directory, *, external=None, dry_run=False, handler_registry=None, root_map=None, serializer_class=None, ): """ Export one Run. Parameters ---------- run: BlueskyRun directory: Union[Str, Manager] Where files containing documents will be written, or a Manager for writing to non-file buffers. external: {None, 'fill', 'ignore') If None, return the paths to external files. If 'fill', fill the external data into the Documents. If 'ignore', do not locate external files. dry_run: Bool, optional If True, do not write any files. False by default. handler_registry: Union[Dict, None] If None, automatic handler discovery is used. serializer_class: Serializer, optional Expected to be a lossless serializer that encodes a format for which there is a corresponding databroker intake driver. Default (None) is currently ``suitcase.msgpack.Serializer``, but this may change in the future. If you want ``suitcase.msgpack.Serializer`` specifically, pass it in explicitly. Returns ------- files: Dict[Str, Set[Str]] Maps each "root" to a set of absolute file paths. """ if serializer_class is None: import suitcase.msgpack serializer_class = suitcase.msgpack.Serializer root_map = root_map or {} resources = [] files = collections.defaultdict(set) if handler_registry is None: handler_registry = databroker.core.discover_handlers() with event_model.Filler(handler_registry, inplace=False, root_map=root_map) as filler: with serializer_class(directory) as serializer: with tqdm(position=0) as progress: for name, doc in run.canonical(fill="no"): if name == "resource": resources.append(doc) if external == "fill": name, doc = filler(name, doc) if not dry_run: serializer(name, doc) progress.update() if external is None: for resource in resources: root = root_map.get(resource["root"], resource["root"]) files[root].update(run.get_file_list(resource)) return serializer.artifacts, dict(files)
def export_run( run, directory, root_hash_func, *, external=None, no_documents=False, handler_registry=None, root_map=None, serializer_class=None, ): """ Export one Run. Parameters ---------- run: BlueskyRun directory: Union[Str, Manager] Where files containing documents will be written, or a Manager for writing to non-file buffers. external: {None, 'fill', 'ignore') If None, return the paths to external files. If 'fill', fill the external data into the Documents. If 'ignore', do not locate external files. no_documents: Bool, optional If True, do not serialize documents. False by default. handler_registry: Union[Dict, None] If None, automatic handler discovery is used. serializer_class: Serializer, optional Expected to be a lossless serializer that encodes a format for which there is a corresponding databroker intake driver. Default (None) is currently ``suitcase.msgpack.Serializer``, but this may change in the future. If you want ``suitcase.msgpack.Serializer`` specifically, pass it in explicitly. Returns ------- artifacts, files Notes ----- * ``artifacts`` maps a human-readable string (typically just ``'all'`` in this case) to a list of buffers or filepaths where the documents were serialized. * ``files`` is the set of filepaths of all external files referenced by Resource documents, keyed on ``(root_in_document, root, unique_id)``. """ EXTERNAL_RELATED_DOCS = ("resource", "datum", "datum_page") if serializer_class is None: import suitcase.msgpack serializer_class = suitcase.msgpack.Serializer root_map = root_map or {} files = collections.defaultdict(set) if handler_registry is None: handler_registry = databroker.core.discover_handlers() with event_model.Filler( handler_registry, inplace=False, root_map=root_map ) as filler: with serializer_class( directory, file_prefix="documents/{start[uid]}" ) as serializer: with tqdm(position=0, unit="documents") as progress: for name, doc in run.canonical(fill="no"): if external == "fill": name, doc = filler(name, doc) # Omit Resource and Datum[Page] because the data was # filled in place. if name in EXTERNAL_RELATED_DOCS: progress.update() continue elif name == "resource": root = root_map.get(doc["root"], doc["root"]) unique_id = root_hash_func(doc["root"]) if external is None: if no_documents: root_in_document = doc["root"] else: root_in_document = root # - root_in_document is the 'root' actually in the # resource_document # - root may be different depending on the # source_catalog configuration, which can map the # recorded 'root' in the document to some other # location. This is where we should go looking for # the data if we plan to copy it. # - unique_id is unique to this (root, salt) # combination and used to place the data in a # unique location. key = (root_in_document, root, unique_id) files[key].update(run.get_file_list(doc)) if not no_documents: # Replace root with a unique ID before serialization. # We are overriding the local variable name doc here # (yuck!) so that serializer(name, doc) below works on # all document types. doc = doc.copy() doc["root"] = unique_id if not no_documents: serializer(name, doc) progress.update() return serializer.artifacts, dict(files)