def test_partial_uid_lookup2(db): key_parts = ['a' * 6, 'b' * 6] run_bundle_A = event_model.compose_run(uid=''.join(key_parts)) db.insert('start', run_bundle_A.start_doc) db.insert('stop', run_bundle_A.compose_stop()) run_bundle_B = event_model.compose_run(uid=''.join(key_parts[::-1])) db.insert('start', run_bundle_B.start_doc) db.insert('stop', run_bundle_B.compose_stop()) hA = db[key_parts[0]] assert dict(hA.start) == run_bundle_A.start_doc hB = db[key_parts[1]] assert dict(hB.start) == run_bundle_B.start_doc
def test_compose_run(): # Compose each kind of document type. These calls will trigger # jsonschema.validate and ensure that the document-generation code composes # valid documents. bundle = event_model.compose_run() start_doc, compose_descriptor, compose_resource, compose_stop = bundle assert bundle.start_doc is start_doc assert bundle.compose_descriptor is compose_descriptor assert bundle.compose_resource is compose_resource assert bundle.compose_stop is compose_stop bundle = compose_descriptor( data_keys={'motor': {'shape': [], 'dtype': 'number', 'source': '...'}, 'image': {'shape': [512, 512], 'dtype': 'number', 'source': '...', 'external': 'FILESTORE:'}}, name='primary') descriptor_doc, compose_event, compose_event_page = bundle assert bundle.descriptor_doc is descriptor_doc assert bundle.compose_event is compose_event assert bundle.compose_event_page is compose_event_page bundle = compose_resource( spec='TIFF', root='/tmp', resource_path='stack.tiff', resource_kwargs={}) resource_doc, compose_datum, compose_datum_page = bundle assert bundle.resource_doc is resource_doc assert bundle.compose_datum is compose_datum assert bundle.compose_datum_page is compose_datum_page datum_doc = compose_datum(datum_kwargs={'slice': 5}) event_doc = compose_event( data={'motor': 0, 'image': datum_doc['datum_id']}, timestamps={'motor': 0, 'image': 0}, filled={'image': False}, seq_num=1) assert 'descriptor' in event_doc compose_stop()
def test_broken_table(): start_doc, descriptor_factory, *_ = compose_run() desc, compose_event, _ = descriptor_factory( name="primary", data_keys={ "x": { "dtype": "integer", "source": "", "shape": [] }, "y": { "dtype": "number", "source": "", "shape": [] }, }, ) ev1 = compose_event(data={ "x": 1, "y": 2.0 }, timestamps={k: time.time() for k in ("x", "y")}) ev2 = compose_event(data={ "x": 1, "y": 2 }, timestamps={k: time.time() for k in ("x", "y")}) ev3 = compose_event(data={ "x": 1.0, "y": 2.0 }, timestamps={k: time.time() for k in ("x", "y")}) ev4 = compose_event( data={ "x": 1.0, "y": "aardvark" }, timestamps={k: time.time() for k in ("x", "y")}, ) sio = StringIO() LT = LiveTable(["x", "y"], out=lambda s: sio.write(s + "\n")) LT("start", start_doc) LT("descriptor", desc) LT("event", ev1) LT("event", ev2) LT("event", ev3) LT("event", ev4) sio.seek(0) lines = sio.readlines() assert len(lines) == 7 for ln in lines[-2:]: assert ln.strip() == "failed to format row"
def test_descriptor_nexus_metadata(tmp_path): documents = [] ( start_doc, compose_descriptor, compose_resource, compose_stop, ) = event_model.compose_run( # 'run start' document metadata=start_doc_md ) print(start_doc) documents.append(("start", start_doc)) descriptor_doc, _, _ = compose_descriptor(**descriptor_doc_md) documents.append(("descriptor", descriptor_doc)) stop_doc = compose_stop() print(stop_doc) documents.append(("stop", stop_doc)) artifacts = nxsas.export(documents, tmp_path) assert len(artifacts["stream_data"]) == 1 print(artifacts) output_filepath = artifacts["stream_data"][0] assert output_filepath.exists() with h5py.File(output_filepath, "r") as h5f: assert "bluesky" in h5f print(list(h5f["bluesky"])) assert "primary" in h5f["bluesky"]["descriptors"] print(list(h5f["bluesky"]["descriptors"]["primary"]))
def test_sanitize_doc(): run_bundle = event_model.compose_run() desc_bundle = run_bundle.compose_descriptor( data_keys={'motor': {'shape': [], 'dtype': 'number', 'source': '...'}, 'image': {'shape': [512, 512], 'dtype': 'number', 'source': '...', 'external': 'FILESTORE:'}}, name='primary') desc_bundle_baseline = run_bundle.compose_descriptor( data_keys={'motor': {'shape': [], 'dtype': 'number', 'source': '...'}}, name='baseline') event1 = desc_bundle.compose_event( data={'motor': 0, 'image': numpy.ones((512, 512))}, timestamps={'motor': 0, 'image': 0}, filled={'image': True}, seq_num=1) event2 = desc_bundle.compose_event( data={'motor': 0, 'image': numpy.ones((512, 512))}, timestamps={'motor': 0, 'image': 0}, filled={'image': True}, seq_num=2) event3 = desc_bundle_baseline.compose_event( data={'motor': 0}, timestamps={'motor': 0}, seq_num=1) event_page = event_model.pack_event_page(event1, event2) bulk_events = {'primary': [event1, event2], 'baseline': [event3]} json.dumps(event_model.sanitize_doc(event_page)) json.dumps(event_model.sanitize_doc(bulk_events)) json.dumps(event_model.sanitize_doc(event1))
def test_bulk_events_to_event_page(): run_bundle = event_model.compose_run() desc_bundle = run_bundle.compose_descriptor( data_keys={'motor': {'shape': [], 'dtype': 'number', 'source': '...'}, 'image': {'shape': [512, 512], 'dtype': 'number', 'source': '...', 'external': 'FILESTORE:'}}, name='primary') desc_bundle_baseline = run_bundle.compose_descriptor( data_keys={'motor': {'shape': [], 'dtype': 'number', 'source': '...'}}, name='baseline') res_bundle = run_bundle.compose_resource( spec='TIFF', root='/tmp', resource_path='stack.tiff', resource_kwargs={}) datum_doc1 = res_bundle.compose_datum(datum_kwargs={'slice': 5}) datum_doc2 = res_bundle.compose_datum(datum_kwargs={'slice': 10}) event1 = desc_bundle.compose_event( data={'motor': 0, 'image': datum_doc1['datum_id']}, timestamps={'motor': 0, 'image': 0}, filled={'image': False}, seq_num=1) event2 = desc_bundle.compose_event( data={'motor': 0, 'image': datum_doc2['datum_id']}, timestamps={'motor': 0, 'image': 0}, filled={'image': False}, seq_num=2) event3 = desc_bundle_baseline.compose_event( data={'motor': 0}, timestamps={'motor': 0}, seq_num=1) primary_event_page = event_model.pack_event_page(event1, event2) baseline_event_page = event_model.pack_event_page(event3) bulk_events = {'primary': [event1, event2], 'baseline': [event3]} pages = event_model.bulk_events_to_event_pages(bulk_events) assert tuple(pages) == (primary_event_page, baseline_event_page)
def doc_gen(stream_names): # Compose run start run_bundle = event_model.compose_run() # type: event_model.ComposeRunBundle start_doc = run_bundle.start_doc yield "start", start_doc for stream_name in stream_names: data = np.random.random(data_shape) # Compose descriptor source = "NCEM" frame_data_keys = { "raw": { "source": source, "dtype": "array", "shape": data.shape, "dims": ("x", "y"), } } if not with_dims: del frame_data_keys['raw']['dims'] frame_stream_bundle = run_bundle.compose_descriptor( data_keys=frame_data_keys, name=stream_name, ) yield "descriptor", frame_stream_bundle.descriptor_doc yield "event", frame_stream_bundle.compose_event( data={"raw": data}, timestamps={"raw": time.time()} ) yield "stop", run_bundle.compose_stop()
def edf_ingestor(paths): # TODO -- metadata? # TODO -- use Datum and Resources timestamp = time.time() # TODO -- use start_doc's time (if it has one)? run_bundle = event_model.compose_run() yield "start", run_bundle.start_doc with fabio.open(paths[0]) as first_frame: field = "pilatus2M" source = "Beamline 7.3.3" shape = list(first_frame.data.shape) frame_data_keys = { field: { "source": source, "dtype": "number", "shape": shape } } frame_stream_name = "primary" frame_stream_bundle = run_bundle.compose_descriptor( data_keys=frame_data_keys, name=frame_stream_name) yield "descriptor", frame_stream_bundle.descriptor_doc for path in paths: with fabio.open(path) as frame: yield "event", frame_stream_bundle.compose_event( data={field: frame.data}, timestamps={field: timestamp}) yield "stop", run_bundle.compose_stop()
def start(self, doc): metadata = { "raw_uid": doc["uid"], "raw_scan_id": doc["scan_id"], "processor_parameters": doc["param"] } self.compose_run_bundle = compose_run(metadata=metadata) return self.compose_run_bundle.start_doc
def __init__(self, metadata=None, uid=None, time=None): self._cache = DocumentCache() self._run_bundle = event_model.compose_run(uid=uid, time=time, metadata=metadata) self._cache.start(self._run_bundle.start_doc) # maps stream name to bundle returned by compose_descriptor self._streams = {}
def start(self, doc): metadata = { 'raw_uid': doc['uid'], 'raw_scan_id': doc['scan_id'], 'processor_parameters': doc['param'] } self.compose_run_bundle = compose_run(metadata=metadata) return self.compose_run_bundle.start_doc
def start_doc(self, x): global _GLOBAL_SCAN_ID _GLOBAL_SCAN_ID += 1 self.md.update(scan_id=_GLOBAL_SCAN_ID) bundle = compose_run(metadata=self.md, validate=False) new_start_doc, self.desc_fac, self.resc_fac, self.stop_factory = bundle self.start_uid = new_start_doc["uid"] return new_start_doc
def start(self, doc): self._source_uid = doc["original_start_uid"] self._sample_name = doc.get("sample_name", None) self.start_bundle = compose_run( metadata=dict(raw_uid=self._source_uid, integrated_uid=doc["uid"], batch_count=doc.get("batch_count", None))) self._pub("start", self.start_bundle.start_doc)
def send_documents(topic, bootstrap_servers): print("send documents to kafka broker") kafka_publisher = Publisher(topic=topic, bootstrap_servers=bootstrap_servers, key="testing") run_start_doc, compose_desc, compose_resource, compose_stop = compose_run() kafka_publisher("start", run_start_doc) # copied from run 588bed89-b8e9-4882-86b2-c9471612914e at SRX event_descriptor_doc, compose_event, compose_event_page = compose_desc( data_keys={ "ROI_01": { "source": "PV:XF:05IDD-ES{Xsp:1}:C1_ROI1:Value_RBV", "dtype": "number", "shape": [], "precision": 4, "units": "", "lower_ctrl_limit": 0.0, "upper_ctrl_limit": 0.0, } }, configuration={ "ROI_01": { "data": { "ROI_01": 6201.48337647908 }, "timestamps": { "ROI_01": 1572730676.801648 }, "data_keys": { "ROI_01": { "source": "PV:XF:05IDD-ES{Xsp:1}:C1_ROI1:Value_RBV", "dtype": "number", "shape": [], "precision": 4, "units": "", "lower_ctrl_limit": 0.0, "upper_ctrl_limit": 0.0, } }, } }, name="ROI_01_monitor", object_keys={"ROI_01": ["ROI_01"]}, ) kafka_publisher("descriptor", event_descriptor_doc) for _ in range(10): event_doc = compose_event( data={'ROI_01': 5.0}, timestamps={"ROI_01": 1572730676.801648}, ) kafka_publisher("event", event_doc) run_stop_doc = compose_stop() kafka_publisher("stop", run_stop_doc)
def test_data_session(): run_bundle = event_model.compose_run(uid="42", metadata={ "data_groups": ["a", "b"], "data_session": "magrathia_visit_42" }) start_doc = run_bundle.start_doc assert start_doc["data_groups"] == ['a', 'b'] with pytest.raises(ValidationError): event_model.compose_run( uid="42", metadata={"data_session": ["shoud", "not", "be", "a", "list"]}) with pytest.raises(ValidationError): event_model.compose_run( uid="42", metadata={"data_groups": "this should be an array"})
def start(self, doc): metadata = { 'raw_uid': doc['uid'], 'processor_version': self.version, 'processor_parameters': { 'factor': self.factor } } self.compose_run_bundle = compose_run(metadata=metadata) return self.compose_run_bundle.start_doc
def getStartDoc(self, paths: typing.List[str]): metadata = self.handler(paths[0]).metadata() descriptor_keys = getattr(self, 'descriptor_keys', []) metadata = dict([(key, metadata.get(key)) for key in descriptor_keys]) metadata = self._setTitle(metadata, paths) run_bundle = event_model.compose_run(metadata=metadata) return run_bundle
def start(self, doc): doc = super().start(doc) self.start_doc = doc metadata = { 'raw_uid': doc['uid'], 'processor_version': self.version, 'processor_parameters': {} } self.compose_run_bundle = compose_run(metadata=metadata) return self.compose_run_bundle.start_doc
def test_subfactory(): # this test targeted the bug described in issue #170 factory_documents = defaultdict(list) subfactory_documents = defaultdict(list) def factory(name, start_doc): def collect_factory_documents(name, doc): factory_documents[name].append(doc) def collect_subfactory_documents(name, doc): subfactory_documents[name].append(doc) def subfactory(name, descriptor_doc): return [collect_subfactory_documents] return [collect_factory_documents], [subfactory] rr = event_model.RunRouter([factory]) run_bundle = event_model.compose_run() rr("start", run_bundle.start_doc) assert len(factory_documents) == 1 assert len(factory_documents["start"]) == 1 assert factory_documents["start"] == [run_bundle.start_doc] assert len(subfactory_documents) == 0 descriptor_bundle = run_bundle.compose_descriptor( data_keys={"motor": { "shape": [], "dtype": "number", "source": "..." }}, name="primary", ) rr("descriptor", descriptor_bundle.descriptor_doc) assert len(factory_documents) == 2 assert len(factory_documents["start"]) == 1 assert factory_documents["start"] == [run_bundle.start_doc] assert len(factory_documents["descriptor"]) == 1 assert factory_documents["descriptor"] == [ descriptor_bundle.descriptor_doc ] assert len(subfactory_documents) == 2 assert len(subfactory_documents["start"]) == 1 assert subfactory_documents["start"] == [run_bundle.start_doc] assert len(subfactory_documents["descriptor"]) == 1 assert subfactory_documents["descriptor"] == [ descriptor_bundle.descriptor_doc ] stop_doc = run_bundle.compose_stop() rr("stop", stop_doc) assert len(rr._start_to_start_doc) == 0
def test_event_page_nexus_metadata(tmp_path): documents = [] ( start_doc, compose_descriptor, compose_resource, compose_stop, ) = event_model.compose_run( # 'run start' document metadata=start_doc_md ) print(start_doc) documents.append(("start", start_doc)) descriptor_doc, compose_event, compose_event_page = compose_descriptor( **descriptor_doc_md ) documents.append(("descriptor", descriptor_doc)) event_page = compose_event_page(**event_page_md) documents.append(("event_page", event_page)) stop_doc = compose_stop() print(stop_doc) documents.append(("stop", stop_doc)) artifacts = nxsas.export(documents, tmp_path) assert len(artifacts["stream_data"]) == 1 print(artifacts) output_filepath = artifacts["stream_data"][0] assert output_filepath.exists() with h5py.File(output_filepath, "r") as h5f: assert "bluesky" in h5f print(list(h5f["bluesky"])) assert "primary" in h5f["bluesky"]["events"] assert "random_walk:dt" in h5f["bluesky"]["events"]["primary"]["data"] assert h5f["bluesky"]["events"]["primary"]["data"]["random_walk:dt"].shape == ( 1, ) assert h5f["bluesky"]["events"]["primary"]["data"]["random_walk:dt"][ () ] == np.array([-1.0]) assert "random_walk:x" in h5f["bluesky"]["events"]["primary"]["data"] assert h5f["bluesky"]["events"]["primary"]["data"]["random_walk:x"].shape == ( 1, ) assert h5f["bluesky"]["events"]["primary"]["data"]["random_walk:x"][ () ] == np.array([1.9221013521832928])
def test_no_descriptors(): run_bundle = event_model.compose_run() start_doc = run_bundle.start_doc stop_doc = run_bundle.compose_stop() documents_to_xarray(start_doc=start_doc, stop_doc=stop_doc, descriptor_docs=[], get_event_pages=no_event_pages, filler=event_model.Filler({}, inplace=True), get_resource=None, lookup_resource_for_datum=None, get_datum_pages=None)
def ingest_NCEM_EMD_VELOX(paths): assert len(paths) == 1 path = paths[0] emd_handle = emdVelox.fileEMDVelox(path) # Compose run start run_bundle = event_model.compose_run() # type: event_model.ComposeRunBundle start_doc = run_bundle.start_doc start_doc["sample_name"] = Path(paths[0]).resolve().stem metadata = _metadata_velox(path) metadata.update(start_doc) start_doc = metadata yield 'start', start_doc num_t = _num_t_velox(emd_handle) first_frame = _get_slice_velox(emd_handle, 0) shape = first_frame.shape dtype = first_frame.dtype delayed_get_slice = dask.delayed(_get_slice_velox) dask_data = da.stack([da.from_delayed(delayed_get_slice(emd_handle, t), shape=shape, dtype=dtype) for t in range(num_t)]) # Compose descriptor source = 'NCEM' frame_data_keys = {'raw': {'source': source, 'dtype': 'number', 'shape': (num_t, *shape)}} frame_stream_name = 'primary' frame_stream_bundle = run_bundle.compose_descriptor(data_keys=frame_data_keys, name=frame_stream_name, # configuration=_metadata(path) ) yield 'descriptor', frame_stream_bundle.descriptor_doc # NOTE: Resource document may be meaningful in the future. For transient access it is not useful # # Compose resource # resource = run_bundle.compose_resource(root=Path(path).root, resource_path=path, spec='NCEM_DM', resource_kwargs={}) # yield 'resource', resource.resource_doc # Compose datum_page # z_indices, t_indices = zip(*itertools.product(z_indices, t_indices)) # datum_page_doc = resource.compose_datum_page(datum_kwargs={'index_z': list(z_indices), 'index_t': list(t_indices)}) # datum_ids = datum_page_doc['datum_id'] # yield 'datum_page', datum_page_doc yield 'event', frame_stream_bundle.compose_event(data={'raw': dask_data}, timestamps={'raw': time.time()}) yield 'stop', run_bundle.compose_stop()
def ingest_result_set(workflow: Workflow, result_set): timestamp = time.time() projections = [] descriptors = [] events = [] run_bundle = event_model.compose_run() end_ops = workflow._end_operations() for end_op, result in zip(end_ops, result_set): frame_data_keys = {} for name, value in result.items(): frame_data_keys[name] = {"source": 'Xi-cam Workflow', "dtype": "array", # TODO: map dtype and shape "shape": getattr(value, 'shape', [])} operation_id = f'{end_op.name}:{workflow.operations.index(end_op)}' frame_stream_bundle = run_bundle.compose_descriptor(data_keys=frame_data_keys, name=operation_id) descriptors.append(("descriptor", frame_stream_bundle.descriptor_doc)) descriptors.append(("event", frame_stream_bundle.compose_event(data=result, timestamps={name: timestamp for name in result}))) for intent_type, name, output_map, args, kwargs in end_op.intent_blueprints: projections.append({'name': 'intent', 'version': '0.1.0', 'projection': {f'intent_type': {'type': 'static', 'value': intent_type}, f'name': {'type': 'static', 'value': name}, f'output_map': {'type': 'static', 'value': output_map}, f'args': {'type': 'static', 'value': args}, f'kwargs': {'type': 'static', 'value': kwargs}, f'operation_id': {'type': 'static', 'value': operation_id}}}) start_doc = run_bundle.start_doc start_doc['projections'] = projections start_doc['sample_name'] = 'Workflow Execution' yield "start", start_doc yield from descriptors yield from events yield "stop", run_bundle.compose_stop()
def emit_start(self, x): # if we have seen this start document already do nothing, we have # multiple parents so we may get a start doc multiple times name, doc = x if doc["uid"] is self.incoming_start_uid: return else: self.incoming_start_uid = doc["uid"] # Prime stop document self.incoming_stop_uid = None # Emergency stop if we get a new start document and no stop has been # issued if self.state != "stopped": self.emit_stop(x) # This is a bit of jank to make certain we don't override the # user metadata with pipeline metadata old_md = dict(self.md) self.md.update( dict( parent_uids=list( set( [ v.start_uid for k, v in self.translation_nodes.items() if v.start_uid is not None ] ) ), parent_node_map={ v.uid: v.start_uid for k, v in self.translation_nodes.items() if v.start_uid is not None }, # keep track of this so we know which node we're sending # data from (see merkle hash in DBFriendly) outbound_node=self.uid, ) ) global _GLOBAL_SCAN_ID _GLOBAL_SCAN_ID += 1 self.md.update(scan_id=_GLOBAL_SCAN_ID) bundle = compose_run(metadata=self.md, validate=False) start, self.desc_fac, self.resc_fac, self.stop_factory = bundle self.start_uid = start["uid"] self.md = old_md # emit starts to subs first in case we create an event from the start [s.emit_start(x) for s in self.subs] self.emit(("start", start)) self.state = "started"
def test_array_like(): "Accept any __array__-like as an array." dask_array = pytest.importorskip("dask.array") bundle = event_model.compose_run() desc_bundle = bundle.compose_descriptor( data_keys={"a": { "shape": (3, ), "dtype": "array", "source": "" }}, name="primary") desc_bundle.compose_event_page(data={"a": dask_array.ones((5, 3))}, timestamps={"a": [1, 2, 3]}, seq_num=[1, 2, 3])
def test_bulk_datum_to_datum_page(): run_bundle = event_model.compose_run() res_bundle = run_bundle.compose_resource( spec='TIFF', root='/tmp', resource_path='stack.tiff', resource_kwargs={}) datum1 = res_bundle.compose_datum(datum_kwargs={'slice': 5}) datum2 = res_bundle.compose_datum(datum_kwargs={'slice': 10}) actual = event_model.pack_datum_page(datum1, datum2) bulk_datum = {'resource': res_bundle.resource_doc['uid'], 'datum_kwarg_list': [datum1['datum_kwargs'], datum2['datum_kwargs']], 'datum_ids': [datum1['datum_id'], datum2['datum_id']]} expected = event_model.bulk_datum_to_datum_page(bulk_datum) assert actual == expected
def gen_stream( data_lst: tp.List[dict], metadata: dict, uid: str = None) -> tp.Generator[tp.Tuple[str, dict], None, None]: """Generate a fake doc stream from data and metadata.""" crb = compose_run(metadata=metadata, uid=uid if uid else str(uuid.uuid4())) yield "start", crb.start_doc if len(data_lst) == 0: yield "stop", crb.compose_stop() cdb: ComposeDescriptorBundle = crb.compose_descriptor( name="primary", data_keys=compose_data_keys(data_lst[0])) yield "descriptor", cdb.descriptor_doc for data in data_lst: yield "event", cdb.compose_event(data=data, timestamps=compose_timestamps(data)) yield "stop", crb.compose_stop()
def test_with_run_router(tmp_path, md): # use a directory that does not exist to test that it will be created output_dir_path = tmp_path / Path("doesnotexist") def factory(name, doc): serializer = nxsas.Serializer(file_prefix="doesnotexist/", directory=output_dir_path) return [serializer], [] rr = event_model.RunRouter([factory]) start_doc_md = {} start_doc_md.update(rsoxs_start_doc) start_doc_md.update(md) # compose_run will raise an exception if "time" and "uid" are in the metadata start_doc_md.pop("time") start_doc_md.pop("uid") ( start_doc, compose_descriptor, compose_resource, compose_stop, ) = event_model.compose_run(metadata=start_doc_md) rr("start", start_doc) descriptor_doc_md = dict() descriptor_doc_md.update(rsoxs_descriptor_en_doc) # compose_descriptor will raise an exception if "run_start" is in the metadata descriptor_doc_md.pop("run_start") descriptor_doc, compose_event, compose_event_page = compose_descriptor( **descriptor_doc_md) rr("descriptor", descriptor_doc) event_md = dict() event_md.update(rsoxs_event_page_en_doc) # event_md["seq_num"] = [1] # the descriptor uid will interfere with compose_event event_md.pop("descriptor") event_doc = compose_event(**event_md) rr("event", event_doc) stop_doc = compose_stop() rr("stop", stop_doc) print(os.listdir(path=output_dir_path)) assert len(os.listdir(path=output_dir_path)) == 1
def ingest(paths): """Ingest a generic image file. Can be used to load any formats that FabIO can read. Parameters ---------- paths: List List of file paths to ingest. Note that only a single path is currently supported. """ assert len(paths) == 1 path = paths[0] image = fabio.open(path) # type: fabio.fabioimage.FabioImage # Create a BlueskyRun run_bundle = event_model.compose_run() # Create the start document start_doc = run_bundle.start_doc start_doc["sample_name"] = image.filename yield "start", start_doc # Create the descriptor document - defines the data keys source = image.header.get("source", "Local") frame_data_keys = { "raw": {"source": source, "dtype": "number", "shape": (image.nframes, *image.shape)} } frame_stream_name = "primary" frame_stream_bundle = run_bundle.compose_descriptor( data_keys=frame_data_keys, name=frame_stream_name ) yield "descriptor", frame_stream_bundle.descriptor_doc # Create the event document - contains the data # delayed_get_slice = dask.delayed(_get_slice) # dask_data = da.stack( # [da.from_delayed(delayed_get_slice(image, frame), shape=image.shape, dtype=image.dtype) # for frame in range(image.nframes)] # ) dask_data = da.stack([frame.data for frame in image.frames()]) timestamp = image.header.get("time", time.time()) yield "event", frame_stream_bundle.compose_event( data={"raw": dask_data}, timestamps={"raw": timestamp} ) # Create the stop document yield "stop", run_bundle.compose_stop()
def test_no_events(): run_bundle = event_model.compose_run() start_doc = run_bundle.start_doc desc_bundle = run_bundle.compose_descriptor( data_keys={'x': {'source': '...', 'shape': [], 'dtype': 'number'}}, name='primary') descriptor_doc = desc_bundle.descriptor_doc stop_doc = run_bundle.compose_stop() documents_to_xarray( start_doc=start_doc, stop_doc=stop_doc, descriptor_docs=[descriptor_doc], get_event_pages=no_event_pages, filler=event_model.Filler({}, inplace=True), get_resource=None, lookup_resource_for_datum=None, get_datum_pages=None)
def start(self, doc): metadata = {'raw_uid': doc['uid'], 'raw_scan_id': doc['scan_id'], 'processor_parameters': doc['param']} self.compose_run_bundle = compose_run(metadata=metadata) return self.compose_run_bundle.start_doc