def build_runs_from_documents(document_generator): """ Build BlueskyRuns from a stream of documents --- (name, doc) pairs. This will work for any number of runs, including 0 (empty generator). Parameters ---------- document_generator: Iterable[Tuple[String, Dict]] Iterable of ``(name, doc)`` pairs. Returns ------- List[BlueskyRun] """ runs = [] def factory(name, doc): dc = DocumentCache() def add_run_to_list(event): run = BlueskyRun(dc) runs.append(run) dc.events.started.connect(add_run_to_list) return [dc], [] rr = event_model.RunRouter([factory]) for item in document_generator: rr(*item) return runs
def test_subfactory_callback_exception(): """ Test that RunRouter._start_to_descriptors and RunRouter._descriptor_to_start are updated in the case that a subfactory callback raises an Exception. """ def exception_callback(name, doc): """ a callback that always raises Exception """ raise Exception() def exception_callback_subfactory(descriptor_doc_name, descriptor_doc): """ a subfactory that always returns one exception_callback """ return [exception_callback] def exception_callback_factory(start_doc_name, start_doc): """ a factory that return 0 callbacks and one exception_callback_subfactory """ return ([], [exception_callback_subfactory]) rr = event_model.RunRouter([exception_callback_factory]) start_doc = {'time': 0, 'uid': 'abcdef'} rr('start', start_doc) descriptor_doc = {'run_start': 'abcdef', 'uid': 'ghijkl'} with pytest.raises(Exception): rr('descriptor', descriptor_doc) assert rr._start_to_descriptors['abcdef'] == ['ghijkl'] assert rr._descriptor_to_start['ghijkl'] == 'abcdef' event_document = {'descriptor': 'ghijkl', 'uid': 'mnopqr'} rr.event(event_document)
def stream_documents_into_runs(add_run): """ Convert a flat stream of documents to "live" BlueskyRuns. Parameters ---------- add_run : callable This will be called as ``add_run(run: BlueskyRun)`` each time a 'start' document is received. Returns ------- callback : callable This should be subscribed to a callback registry that calls it like ``callback(name, doc)``. Examples -------- This is used for connecting something that emits a flat stream of documents to something that wants to receive BlueskyRuns. Append to a plain list. >>> from bluesky import RunEngine >>> RE = RunEngine() >>> runs = [] >>> RE.subscribe(stream_documents_into_runs(runs.append)) Or, more usefully to an observable list. >>> from bluesky_widgets.models.utils import RunList >>> runs = RunList() >>> RE.subscribe(stream_documents_into_runs(runs.append)) Add runs to a model with an ``add_run`` method. For example, it might be a model that generates figures. >>> from bluesky_widgets.models.plot_builders import AutoLines >>> model = AutoLines() >>> RE.subscribe(stream_documents_into_runs(model.add_run)) """ def factory(name, doc): dc = DocumentCache() def build_and_add_run(event): run = BlueskyRun(dc) add_run(run) dc.events.started.connect(build_and_add_run) return [dc], [] rr = event_model.RunRouter([factory], handler_registry={"NPY_SEQ": NumpySeqHandler}) return rr
def test_subfactory(): # this test targeted the bug described in issue #170 factory_documents = defaultdict(list) subfactory_documents = defaultdict(list) def factory(name, start_doc): def collect_factory_documents(name, doc): factory_documents[name].append(doc) def collect_subfactory_documents(name, doc): subfactory_documents[name].append(doc) def subfactory(name, descriptor_doc): return [collect_subfactory_documents] return [collect_factory_documents], [subfactory] rr = event_model.RunRouter([factory]) run_bundle = event_model.compose_run() rr("start", run_bundle.start_doc) assert len(factory_documents) == 1 assert len(factory_documents["start"]) == 1 assert factory_documents["start"] == [run_bundle.start_doc] assert len(subfactory_documents) == 0 descriptor_bundle = run_bundle.compose_descriptor( data_keys={"motor": { "shape": [], "dtype": "number", "source": "..." }}, name="primary", ) rr("descriptor", descriptor_bundle.descriptor_doc) assert len(factory_documents) == 2 assert len(factory_documents["start"]) == 1 assert factory_documents["start"] == [run_bundle.start_doc] assert len(factory_documents["descriptor"]) == 1 assert factory_documents["descriptor"] == [ descriptor_bundle.descriptor_doc ] assert len(subfactory_documents) == 2 assert len(subfactory_documents["start"]) == 1 assert subfactory_documents["start"] == [run_bundle.start_doc] assert len(subfactory_documents["descriptor"]) == 1 assert subfactory_documents["descriptor"] == [ descriptor_bundle.descriptor_doc ] stop_doc = run_bundle.compose_stop() rr("stop", stop_doc) assert len(rr._start_to_start_doc) == 0
def test_same_start_doc_twice(): "If the user sends us the same uid twice, raise helpfully." rr = event_model.RunRouter([]) doc = {'time': 0, 'uid': 'stuff'} rr('start', doc) with pytest.raises(ValueError): rr('start', doc) # exact same object with pytest.raises(ValueError): rr('start', doc.copy()) # same content doc2 = {'time': 1, 'uid': 'stuff'} with pytest.raises(ValueError): rr('start', doc2) # same uid, different content
def save_example_data(): """ Run this from repo root to re-generate data. python -c "import bluesky_tutorial_utils; bluesky_tutorial_utils.save_example_data()" """ import suitcase.jsonl def factory(name, doc): return [suitcase.jsonl.Serializer(str(directory))], [] rr = event_model.RunRouter([factory]) generate_example_data(rr)
def test_with_run_router(tmp_path, md): # use a directory that does not exist to test that it will be created output_dir_path = tmp_path / Path("doesnotexist") def factory(name, doc): serializer = nxsas.Serializer(file_prefix="doesnotexist/", directory=output_dir_path) return [serializer], [] rr = event_model.RunRouter([factory]) start_doc_md = {} start_doc_md.update(rsoxs_start_doc) start_doc_md.update(md) # compose_run will raise an exception if "time" and "uid" are in the metadata start_doc_md.pop("time") start_doc_md.pop("uid") ( start_doc, compose_descriptor, compose_resource, compose_stop, ) = event_model.compose_run(metadata=start_doc_md) rr("start", start_doc) descriptor_doc_md = dict() descriptor_doc_md.update(rsoxs_descriptor_en_doc) # compose_descriptor will raise an exception if "run_start" is in the metadata descriptor_doc_md.pop("run_start") descriptor_doc, compose_event, compose_event_page = compose_descriptor( **descriptor_doc_md) rr("descriptor", descriptor_doc) event_md = dict() event_md.update(rsoxs_event_page_en_doc) # event_md["seq_num"] = [1] # the descriptor uid will interfere with compose_event event_md.pop("descriptor") event_doc = compose_event(**event_md) rr("event", event_doc) stop_doc = compose_stop() rr("stop", stop_doc) print(os.listdir(path=output_dir_path)) assert len(os.listdir(path=output_dir_path)) == 1
def start_worker(export_dir, kafka_bootstrap_servers, kafka_topics): def worker_factory(name, start_doc, export_dir): example_worker = ExampleWorker() return [example_worker], [] dispatcher = bluesky_kafka.RemoteDispatcher( topics=kafka_topics, group_id="iss-example-worker", bootstrap_servers=kafka_bootstrap_servers, #deserializer=msgpack.loads, ) rr = event_model.RunRouter( [partial(worker_factory, export_dir="export_dir")], handler_registry={ "AD_TIFF": databroker.assets.handlers.AreaDetectorTiffHandler, "NPY_SEQ": ophyd.sim.NumpySeqHandler, }, ) dispatcher.subscribe(rr) dispatcher.start()
def test_run_router(tmp_path): bundle = event_model.compose_run() docs = [] start_doc, compose_descriptor, compose_resource, compose_stop = bundle docs.append(('start', start_doc)) bundle = compose_descriptor(data_keys={ 'motor': { 'shape': [], 'dtype': 'number', 'source': '...' }, 'image': { 'shape': [512, 512], 'dtype': 'number', 'source': '...', 'external': 'FILESTORE:' } }, name='primary') primary_descriptor_doc, compose_primary_event, compose_event_page = bundle docs.append(('descriptor', primary_descriptor_doc)) bundle = compose_descriptor( data_keys={'motor': { 'shape': [], 'dtype': 'number', 'source': '...' }}, name='baseline') baseline_descriptor_doc, compose_baseline_event, compose_event_page = bundle docs.append(('descriptor', baseline_descriptor_doc)) bundle = compose_resource(spec='TIFF', root=str(tmp_path), resource_path='stack.tiff', resource_kwargs={}) resource_doc, compose_datum, compose_datum_page = bundle docs.append(('resource', resource_doc)) datum_doc = compose_datum(datum_kwargs={'slice': 5}) docs.append(('datum', datum_doc)) primary_event_doc = compose_primary_event(data={ 'motor': 0, 'image': datum_doc['datum_id'] }, timestamps={ 'motor': 0, 'image': 0 }, filled={'image': False}) docs.append(('event', primary_event_doc)) baseline_event_doc = compose_baseline_event(data={'motor': 0}, timestamps={'motor': 0}) docs.append(('event', baseline_event_doc)) stop_doc = compose_stop() docs.append(('stop', stop_doc)) # Empty list of factories. Just make sure nothing blows up. rr = event_model.RunRouter([]) for name, doc in docs: rr(name, doc) # A factory that rejects all runs. def null_factory(name, doc): return [], [] rr = event_model.RunRouter([null_factory]) for name, doc in docs: rr(name, doc) # A factory that accepts all runs. collected = [] def collector(name, doc): if name == 'event_page': name = 'event' doc, = event_model.unpack_event_page(doc) elif name == 'datum_page': name = 'datum' doc, = event_model.unpack_datum_page(doc) collected.append((name, doc)) def all_factory(name, doc): return [collector], [] rr = event_model.RunRouter([all_factory]) for name, doc in docs: rr(name, doc) assert collected == docs collected.clear() # A factory that returns a subfactory interested in 'baseline' only. def subfactory(name, doc): if doc.get('name') == 'baseline': return [collector] return [] def factory_with_subfactory_only(name, doc): return [], [subfactory] rr = event_model.RunRouter([factory_with_subfactory_only]) for name, doc in docs: rr(name, doc) expected_item = ('event', baseline_event_doc) unexpected_item = ('event', primary_event_doc) assert expected_item in collected assert unexpected_item not in collected collected.clear() # Test factory that expects old (pre-1.14.0) RunRouter behavior. collected_header_docs = {} class LocalException3(Exception): ... def header_collector(name, doc): if name in ('start', 'stop', 'descriptor'): key = (name, doc['uid']) if key in collected_header_docs: raise LocalException3 collected_header_docs[key] = doc def all_factory(name, doc): header_collector(name, doc) return [header_collector], [] rr = event_model.RunRouter([all_factory]) with pytest.warns(UserWarning, match='1.14.0'), pytest.raises(LocalException3): for name, doc in docs: rr(name, doc) collected_header_docs.clear() # Test subfactory that expects old (pre-1.14.0) RunRouter behavior. def factory_with_subfactory_only(name, doc): header_collector(name, doc) def subfactory(name, doc): if doc.get('name') == 'baseline': header_collector(name, doc) return [header_collector] return [] return [], [subfactory] rr = event_model.RunRouter([factory_with_subfactory_only]) with pytest.warns(UserWarning, match='1.14.0'), pytest.raises(LocalException3): for name, doc in docs: rr(name, doc) collected_header_docs.clear() # Test RunRouter with handler_registry. class FakeTiffHandler: def __init__(self, resource_path): assert resource_path == str(tmp_path / "stack.tiff") def __call__(self, slice): return numpy.ones((5, 5)) reg = {'TIFF': FakeTiffHandler} def check_filled(name, doc): if name == 'event_page': for is_filled in doc['filled'].values(): assert all(is_filled) elif name == 'event': for is_filled in doc['filled'].values(): assert is_filled def check_not_filled(name, doc): if name == 'event_page': for is_filled in doc['filled'].values(): assert not any(is_filled) elif name == 'event': for is_filled in doc['filled'].values(): assert not is_filled def check_filled_factory(name, doc): return [check_filled], [] def check_not_filled_factory(name, doc): return [check_not_filled], [] # If reg is missing our spec (or just not given) docs pass through # unfilled. rr = event_model.RunRouter([check_not_filled_factory]) for name, doc in docs: rr(name, doc) # If fill_or_fail is set to True and reg is missing our spec (or just not # given) we raise. rr = event_model.RunRouter([check_not_filled_factory], fill_or_fail=True) with pytest.raises(event_model.UndefinedAssetSpecification): for name, doc in docs: rr(name, doc) # If spec is provided, docs are filled, regardless of fill_or_fail. rr = event_model.RunRouter([check_filled_factory], reg) for name, doc in docs: rr(name, doc) rr = event_model.RunRouter([check_filled_factory], reg, fill_or_fail=True) for name, doc in docs: rr(name, doc)
def test_run_router(): bundle = event_model.compose_run() docs = [] start_doc, compose_descriptor, compose_resource, compose_stop = bundle docs.append(('start', start_doc)) bundle = compose_descriptor(data_keys={ 'motor': { 'shape': [], 'dtype': 'number', 'source': '...' }, 'image': { 'shape': [512, 512], 'dtype': 'number', 'source': '...', 'external': 'FILESTORE:' } }, name='primary') primary_descriptor_doc, compose_primary_event, compose_event_page = bundle docs.append(('descriptor', primary_descriptor_doc)) bundle = compose_descriptor( data_keys={'motor': { 'shape': [], 'dtype': 'number', 'source': '...' }}, name='baseline') baseline_descriptor_doc, compose_baseline_event, compose_event_page = bundle docs.append(('descriptor', baseline_descriptor_doc)) bundle = compose_resource(spec='TIFF', root='/tmp', resource_path='stack.tiff', resource_kwargs={}) resource_doc, compose_datum, compose_datum_page = bundle docs.append(('resource', resource_doc)) datum_doc = compose_datum(datum_kwargs={'slice': 5}) docs.append(('datum', datum_doc)) primary_event_doc = compose_primary_event(data={ 'motor': 0, 'image': datum_doc['datum_id'] }, timestamps={ 'motor': 0, 'image': 0 }, filled={'image': False}) docs.append(('event', primary_event_doc)) baseline_event_doc = compose_baseline_event(data={'motor': 0}, timestamps={'motor': 0}) docs.append(('event', baseline_event_doc)) stop_doc = compose_stop() docs.append(('stop', stop_doc)) # Empty list of factories. Just make sure nothing blows up. rr = event_model.RunRouter([]) for name, doc in docs: rr(name, doc) # A factory that rejects all runs. def null_factory(name, doc): return [], [] rr = event_model.RunRouter([null_factory]) for name, doc in docs: rr(name, doc) # A factory that accepts all runs. collected = [] def collector(name, doc): if name == 'event_page': name = 'event' doc, = event_model.unpack_event_page(doc) elif name == 'datum_page': name = 'datum' doc, = event_model.unpack_datum_page(doc) collected.append((name, doc)) def all_factory(name, doc): collector(name, doc) return [collector], [] rr = event_model.RunRouter([all_factory]) for name, doc in docs: rr(name, doc) assert collected == docs collected.clear() # A factory that returns a subfactory interested in 'baseline' only. def subfactory(name, doc): if doc.get('name') == 'baseline': return [collector] return [] def factory_with_subfactory_only(name, doc): return [], [subfactory] rr = event_model.RunRouter([factory_with_subfactory_only]) for name, doc in docs: rr(name, doc) expected_item = ('event', baseline_event_doc) unexpected_item = ('event', primary_event_doc) assert expected_item in collected assert unexpected_item not in collected collected.clear()