def test_partial_uid_lookup2(db):
key_parts = ['a' * 6, 'b' * 6]
run_bundle_A = event_model.compose_run(uid=''.join(key_parts))
db.insert('start', run_bundle_A.start_doc)
db.insert('stop', run_bundle_A.compose_stop())
run_bundle_B = event_model.compose_run(uid=''.join(key_parts[::-1]))
db.insert('start', run_bundle_B.start_doc)
db.insert('stop', run_bundle_B.compose_stop())
hA = db[key_parts[0]]
assert dict(hA.start) == run_bundle_A.start_doc
hB = db[key_parts[1]]
assert dict(hB.start) == run_bundle_B.start_doc
def test_compose_run():
# Compose each kind of document type. These calls will trigger
# jsonschema.validate and ensure that the document-generation code composes
# valid documents.
bundle = event_model.compose_run()
start_doc, compose_descriptor, compose_resource, compose_stop = bundle
assert bundle.start_doc is start_doc
assert bundle.compose_descriptor is compose_descriptor
assert bundle.compose_resource is compose_resource
assert bundle.compose_stop is compose_stop
bundle = compose_descriptor(
data_keys={'motor': {'shape': [], 'dtype': 'number', 'source': '...'},
'image': {'shape': [512, 512], 'dtype': 'number',
'source': '...', 'external': 'FILESTORE:'}},
name='primary')
descriptor_doc, compose_event, compose_event_page = bundle
assert bundle.descriptor_doc is descriptor_doc
assert bundle.compose_event is compose_event
assert bundle.compose_event_page is compose_event_page
bundle = compose_resource(
spec='TIFF', root='/tmp', resource_path='stack.tiff',
resource_kwargs={})
resource_doc, compose_datum, compose_datum_page = bundle
assert bundle.resource_doc is resource_doc
assert bundle.compose_datum is compose_datum
assert bundle.compose_datum_page is compose_datum_page
datum_doc = compose_datum(datum_kwargs={'slice': 5})
event_doc = compose_event(
data={'motor': 0, 'image': datum_doc['datum_id']},
timestamps={'motor': 0, 'image': 0}, filled={'image': False},
seq_num=1)
assert 'descriptor' in event_doc
compose_stop()
def test_broken_table():
start_doc, descriptor_factory, *_ = compose_run()
desc, compose_event, _ = descriptor_factory(
name="primary",
data_keys={
"x": {
"dtype": "integer",
"source": "",
"shape": []
},
"y": {
"dtype": "number",
"source": "",
"shape": []
},
},
)
ev1 = compose_event(data={
"x": 1,
"y": 2.0
},
timestamps={k: time.time()
for k in ("x", "y")})
ev2 = compose_event(data={
"x": 1,
"y": 2
},
timestamps={k: time.time()
for k in ("x", "y")})
ev3 = compose_event(data={
"x": 1.0,
"y": 2.0
},
timestamps={k: time.time()
for k in ("x", "y")})
ev4 = compose_event(
data={
"x": 1.0,
"y": "aardvark"
},
timestamps={k: time.time()
for k in ("x", "y")},
)
sio = StringIO()
LT = LiveTable(["x", "y"], out=lambda s: sio.write(s + "\n"))
LT("start", start_doc)
LT("descriptor", desc)
LT("event", ev1)
LT("event", ev2)
LT("event", ev3)
LT("event", ev4)
sio.seek(0)
lines = sio.readlines()
assert len(lines) == 7
for ln in lines[-2:]:
assert ln.strip() == "failed to format row"
def test_descriptor_nexus_metadata(tmp_path):
documents = []
(
start_doc,
compose_descriptor,
compose_resource,
compose_stop,
) = event_model.compose_run(
# 'run start' document
metadata=start_doc_md
)
print(start_doc)
documents.append(("start", start_doc))
descriptor_doc, _, _ = compose_descriptor(**descriptor_doc_md)
documents.append(("descriptor", descriptor_doc))
stop_doc = compose_stop()
print(stop_doc)
documents.append(("stop", stop_doc))
artifacts = nxsas.export(documents, tmp_path)
assert len(artifacts["stream_data"]) == 1
print(artifacts)
output_filepath = artifacts["stream_data"][0]
assert output_filepath.exists()
with h5py.File(output_filepath, "r") as h5f:
assert "bluesky" in h5f
print(list(h5f["bluesky"]))
assert "primary" in h5f["bluesky"]["descriptors"]
print(list(h5f["bluesky"]["descriptors"]["primary"]))
def test_sanitize_doc():
run_bundle = event_model.compose_run()
desc_bundle = run_bundle.compose_descriptor(
data_keys={'motor': {'shape': [], 'dtype': 'number', 'source': '...'},
'image': {'shape': [512, 512], 'dtype': 'number',
'source': '...', 'external': 'FILESTORE:'}},
name='primary')
desc_bundle_baseline = run_bundle.compose_descriptor(
data_keys={'motor': {'shape': [], 'dtype': 'number', 'source': '...'}},
name='baseline')
event1 = desc_bundle.compose_event(
data={'motor': 0, 'image': numpy.ones((512, 512))},
timestamps={'motor': 0, 'image': 0}, filled={'image': True},
seq_num=1)
event2 = desc_bundle.compose_event(
data={'motor': 0, 'image': numpy.ones((512, 512))},
timestamps={'motor': 0, 'image': 0}, filled={'image': True},
seq_num=2)
event3 = desc_bundle_baseline.compose_event(
data={'motor': 0},
timestamps={'motor': 0},
seq_num=1)
event_page = event_model.pack_event_page(event1, event2)
bulk_events = {'primary': [event1, event2], 'baseline': [event3]}
json.dumps(event_model.sanitize_doc(event_page))
json.dumps(event_model.sanitize_doc(bulk_events))
json.dumps(event_model.sanitize_doc(event1))
def test_bulk_events_to_event_page():
run_bundle = event_model.compose_run()
desc_bundle = run_bundle.compose_descriptor(
data_keys={'motor': {'shape': [], 'dtype': 'number', 'source': '...'},
'image': {'shape': [512, 512], 'dtype': 'number',
'source': '...', 'external': 'FILESTORE:'}},
name='primary')
desc_bundle_baseline = run_bundle.compose_descriptor(
data_keys={'motor': {'shape': [], 'dtype': 'number', 'source': '...'}},
name='baseline')
res_bundle = run_bundle.compose_resource(
spec='TIFF', root='/tmp', resource_path='stack.tiff',
resource_kwargs={})
datum_doc1 = res_bundle.compose_datum(datum_kwargs={'slice': 5})
datum_doc2 = res_bundle.compose_datum(datum_kwargs={'slice': 10})
event1 = desc_bundle.compose_event(
data={'motor': 0, 'image': datum_doc1['datum_id']},
timestamps={'motor': 0, 'image': 0}, filled={'image': False},
seq_num=1)
event2 = desc_bundle.compose_event(
data={'motor': 0, 'image': datum_doc2['datum_id']},
timestamps={'motor': 0, 'image': 0}, filled={'image': False},
seq_num=2)
event3 = desc_bundle_baseline.compose_event(
data={'motor': 0},
timestamps={'motor': 0},
seq_num=1)
primary_event_page = event_model.pack_event_page(event1, event2)
baseline_event_page = event_model.pack_event_page(event3)
bulk_events = {'primary': [event1, event2], 'baseline': [event3]}
pages = event_model.bulk_events_to_event_pages(bulk_events)
assert tuple(pages) == (primary_event_page, baseline_event_page)
def doc_gen(stream_names):
# Compose run start
run_bundle = event_model.compose_run() # type: event_model.ComposeRunBundle
start_doc = run_bundle.start_doc
yield "start", start_doc
for stream_name in stream_names:
data = np.random.random(data_shape)
# Compose descriptor
source = "NCEM"
frame_data_keys = {
"raw": {
"source": source,
"dtype": "array",
"shape": data.shape,
"dims": ("x", "y"),
}
}
if not with_dims:
del frame_data_keys['raw']['dims']
frame_stream_bundle = run_bundle.compose_descriptor(
data_keys=frame_data_keys, name=stream_name,
)
yield "descriptor", frame_stream_bundle.descriptor_doc
yield "event", frame_stream_bundle.compose_event(
data={"raw": data}, timestamps={"raw": time.time()}
)
yield "stop", run_bundle.compose_stop()
def edf_ingestor(paths):
# TODO -- metadata?
# TODO -- use Datum and Resources
timestamp = time.time() # TODO -- use start_doc's time (if it has one)?
run_bundle = event_model.compose_run()
yield "start", run_bundle.start_doc
with fabio.open(paths[0]) as first_frame:
field = "pilatus2M"
source = "Beamline 7.3.3"
shape = list(first_frame.data.shape)
frame_data_keys = {
field: {
"source": source,
"dtype": "number",
"shape": shape
}
}
frame_stream_name = "primary"
frame_stream_bundle = run_bundle.compose_descriptor(
data_keys=frame_data_keys, name=frame_stream_name)
yield "descriptor", frame_stream_bundle.descriptor_doc
for path in paths:
with fabio.open(path) as frame:
yield "event", frame_stream_bundle.compose_event(
data={field: frame.data}, timestamps={field: timestamp})
yield "stop", run_bundle.compose_stop()
def start(self, doc):
metadata = {
"raw_uid": doc["uid"],
"raw_scan_id": doc["scan_id"],
"processor_parameters": doc["param"]
}
self.compose_run_bundle = compose_run(metadata=metadata)
return self.compose_run_bundle.start_doc
def __init__(self, metadata=None, uid=None, time=None):
self._cache = DocumentCache()
self._run_bundle = event_model.compose_run(uid=uid,
time=time,
metadata=metadata)
self._cache.start(self._run_bundle.start_doc)
# maps stream name to bundle returned by compose_descriptor
self._streams = {}
def start(self, doc):
metadata = {
'raw_uid': doc['uid'],
'raw_scan_id': doc['scan_id'],
'processor_parameters': doc['param']
}
self.compose_run_bundle = compose_run(metadata=metadata)
return self.compose_run_bundle.start_doc
def start_doc(self, x):
global _GLOBAL_SCAN_ID
_GLOBAL_SCAN_ID += 1
self.md.update(scan_id=_GLOBAL_SCAN_ID)
bundle = compose_run(metadata=self.md, validate=False)
new_start_doc, self.desc_fac, self.resc_fac, self.stop_factory = bundle
self.start_uid = new_start_doc["uid"]
return new_start_doc
def start(self, doc):
self._source_uid = doc["original_start_uid"]
self._sample_name = doc.get("sample_name", None)
self.start_bundle = compose_run(
metadata=dict(raw_uid=self._source_uid,
integrated_uid=doc["uid"],
batch_count=doc.get("batch_count", None)))
self._pub("start", self.start_bundle.start_doc)
def send_documents(topic, bootstrap_servers):
print("send documents to kafka broker")
kafka_publisher = Publisher(topic=topic,
bootstrap_servers=bootstrap_servers,
key="testing")
run_start_doc, compose_desc, compose_resource, compose_stop = compose_run()
kafka_publisher("start", run_start_doc)
# copied from run 588bed89-b8e9-4882-86b2-c9471612914e at SRX
event_descriptor_doc, compose_event, compose_event_page = compose_desc(
data_keys={
"ROI_01": {
"source": "PV:XF:05IDD-ES{Xsp:1}:C1_ROI1:Value_RBV",
"dtype": "number",
"shape": [],
"precision": 4,
"units": "",
"lower_ctrl_limit": 0.0,
"upper_ctrl_limit": 0.0,
}
},
configuration={
"ROI_01": {
"data": {
"ROI_01": 6201.48337647908
},
"timestamps": {
"ROI_01": 1572730676.801648
},
"data_keys": {
"ROI_01": {
"source": "PV:XF:05IDD-ES{Xsp:1}:C1_ROI1:Value_RBV",
"dtype": "number",
"shape": [],
"precision": 4,
"units": "",
"lower_ctrl_limit": 0.0,
"upper_ctrl_limit": 0.0,
}
},
}
},
name="ROI_01_monitor",
object_keys={"ROI_01": ["ROI_01"]},
)
kafka_publisher("descriptor", event_descriptor_doc)
for _ in range(10):
event_doc = compose_event(
data={'ROI_01': 5.0},
timestamps={"ROI_01": 1572730676.801648},
)
kafka_publisher("event", event_doc)
run_stop_doc = compose_stop()
kafka_publisher("stop", run_stop_doc)
def test_data_session():
run_bundle = event_model.compose_run(uid="42",
metadata={
"data_groups": ["a", "b"],
"data_session":
"magrathia_visit_42"
})
start_doc = run_bundle.start_doc
assert start_doc["data_groups"] == ['a', 'b']
with pytest.raises(ValidationError):
event_model.compose_run(
uid="42",
metadata={"data_session": ["shoud", "not", "be", "a", "list"]})
with pytest.raises(ValidationError):
event_model.compose_run(
uid="42", metadata={"data_groups": "this should be an array"})
def start(self, doc):
metadata = {
'raw_uid': doc['uid'],
'processor_version': self.version,
'processor_parameters': {
'factor': self.factor
}
}
self.compose_run_bundle = compose_run(metadata=metadata)
return self.compose_run_bundle.start_doc
def getStartDoc(self, paths: typing.List[str]):
metadata = self.handler(paths[0]).metadata()
descriptor_keys = getattr(self, 'descriptor_keys', [])
metadata = dict([(key, metadata.get(key)) for key in descriptor_keys])
metadata = self._setTitle(metadata, paths)
run_bundle = event_model.compose_run(metadata=metadata)
return run_bundle
def start(self, doc):
doc = super().start(doc)
self.start_doc = doc
metadata = {
'raw_uid': doc['uid'],
'processor_version': self.version,
'processor_parameters': {}
}
self.compose_run_bundle = compose_run(metadata=metadata)
return self.compose_run_bundle.start_doc
def test_subfactory():
# this test targeted the bug described in issue #170
factory_documents = defaultdict(list)
subfactory_documents = defaultdict(list)
def factory(name, start_doc):
def collect_factory_documents(name, doc):
factory_documents[name].append(doc)
def collect_subfactory_documents(name, doc):
subfactory_documents[name].append(doc)
def subfactory(name, descriptor_doc):
return [collect_subfactory_documents]
return [collect_factory_documents], [subfactory]
rr = event_model.RunRouter([factory])
run_bundle = event_model.compose_run()
rr("start", run_bundle.start_doc)
assert len(factory_documents) == 1
assert len(factory_documents["start"]) == 1
assert factory_documents["start"] == [run_bundle.start_doc]
assert len(subfactory_documents) == 0
descriptor_bundle = run_bundle.compose_descriptor(
data_keys={"motor": {
"shape": [],
"dtype": "number",
"source": "..."
}},
name="primary",
)
rr("descriptor", descriptor_bundle.descriptor_doc)
assert len(factory_documents) == 2
assert len(factory_documents["start"]) == 1
assert factory_documents["start"] == [run_bundle.start_doc]
assert len(factory_documents["descriptor"]) == 1
assert factory_documents["descriptor"] == [
descriptor_bundle.descriptor_doc
]
assert len(subfactory_documents) == 2
assert len(subfactory_documents["start"]) == 1
assert subfactory_documents["start"] == [run_bundle.start_doc]
assert len(subfactory_documents["descriptor"]) == 1
assert subfactory_documents["descriptor"] == [
descriptor_bundle.descriptor_doc
]
stop_doc = run_bundle.compose_stop()
rr("stop", stop_doc)
assert len(rr._start_to_start_doc) == 0
def test_event_page_nexus_metadata(tmp_path):
documents = []
(
start_doc,
compose_descriptor,
compose_resource,
compose_stop,
) = event_model.compose_run(
# 'run start' document
metadata=start_doc_md
)
print(start_doc)
documents.append(("start", start_doc))
descriptor_doc, compose_event, compose_event_page = compose_descriptor(
**descriptor_doc_md
)
documents.append(("descriptor", descriptor_doc))
event_page = compose_event_page(**event_page_md)
documents.append(("event_page", event_page))
stop_doc = compose_stop()
print(stop_doc)
documents.append(("stop", stop_doc))
artifacts = nxsas.export(documents, tmp_path)
assert len(artifacts["stream_data"]) == 1
print(artifacts)
output_filepath = artifacts["stream_data"][0]
assert output_filepath.exists()
with h5py.File(output_filepath, "r") as h5f:
assert "bluesky" in h5f
print(list(h5f["bluesky"]))
assert "primary" in h5f["bluesky"]["events"]
assert "random_walk:dt" in h5f["bluesky"]["events"]["primary"]["data"]
assert h5f["bluesky"]["events"]["primary"]["data"]["random_walk:dt"].shape == (
1,
)
assert h5f["bluesky"]["events"]["primary"]["data"]["random_walk:dt"][
()
] == np.array([-1.0])
assert "random_walk:x" in h5f["bluesky"]["events"]["primary"]["data"]
assert h5f["bluesky"]["events"]["primary"]["data"]["random_walk:x"].shape == (
1,
)
assert h5f["bluesky"]["events"]["primary"]["data"]["random_walk:x"][
()
] == np.array([1.9221013521832928])
def test_no_descriptors():
run_bundle = event_model.compose_run()
start_doc = run_bundle.start_doc
stop_doc = run_bundle.compose_stop()
documents_to_xarray(start_doc=start_doc,
stop_doc=stop_doc,
descriptor_docs=[],
get_event_pages=no_event_pages,
filler=event_model.Filler({}, inplace=True),
get_resource=None,
lookup_resource_for_datum=None,
get_datum_pages=None)
def ingest_NCEM_EMD_VELOX(paths):
assert len(paths) == 1
path = paths[0]
emd_handle = emdVelox.fileEMDVelox(path)
# Compose run start
run_bundle = event_model.compose_run() # type: event_model.ComposeRunBundle
start_doc = run_bundle.start_doc
start_doc["sample_name"] = Path(paths[0]).resolve().stem
metadata = _metadata_velox(path)
metadata.update(start_doc)
start_doc = metadata
yield 'start', start_doc
num_t = _num_t_velox(emd_handle)
first_frame = _get_slice_velox(emd_handle, 0)
shape = first_frame.shape
dtype = first_frame.dtype
delayed_get_slice = dask.delayed(_get_slice_velox)
dask_data = da.stack([da.from_delayed(delayed_get_slice(emd_handle, t), shape=shape, dtype=dtype)
for t in range(num_t)])
# Compose descriptor
source = 'NCEM'
frame_data_keys = {'raw': {'source': source,
'dtype': 'number',
'shape': (num_t, *shape)}}
frame_stream_name = 'primary'
frame_stream_bundle = run_bundle.compose_descriptor(data_keys=frame_data_keys,
name=frame_stream_name,
# configuration=_metadata(path)
)
yield 'descriptor', frame_stream_bundle.descriptor_doc
# NOTE: Resource document may be meaningful in the future. For transient access it is not useful
# # Compose resource
# resource = run_bundle.compose_resource(root=Path(path).root, resource_path=path, spec='NCEM_DM', resource_kwargs={})
# yield 'resource', resource.resource_doc
# Compose datum_page
# z_indices, t_indices = zip(*itertools.product(z_indices, t_indices))
# datum_page_doc = resource.compose_datum_page(datum_kwargs={'index_z': list(z_indices), 'index_t': list(t_indices)})
# datum_ids = datum_page_doc['datum_id']
# yield 'datum_page', datum_page_doc
yield 'event', frame_stream_bundle.compose_event(data={'raw': dask_data},
timestamps={'raw': time.time()})
yield 'stop', run_bundle.compose_stop()
def ingest_result_set(workflow: Workflow, result_set):
timestamp = time.time()
projections = []
descriptors = []
events = []
run_bundle = event_model.compose_run()
end_ops = workflow._end_operations()
for end_op, result in zip(end_ops, result_set):
frame_data_keys = {}
for name, value in result.items():
frame_data_keys[name] = {"source": 'Xi-cam Workflow',
"dtype": "array", # TODO: map dtype and shape
"shape": getattr(value, 'shape', [])}
operation_id = f'{end_op.name}:{workflow.operations.index(end_op)}'
frame_stream_bundle = run_bundle.compose_descriptor(data_keys=frame_data_keys, name=operation_id)
descriptors.append(("descriptor", frame_stream_bundle.descriptor_doc))
descriptors.append(("event", frame_stream_bundle.compose_event(data=result,
timestamps={name: timestamp for name in
result})))
for intent_type, name, output_map, args, kwargs in end_op.intent_blueprints:
projections.append({'name': 'intent',
'version': '0.1.0',
'projection':
{f'intent_type': {'type': 'static',
'value': intent_type},
f'name': {'type': 'static',
'value': name},
f'output_map': {'type': 'static',
'value': output_map},
f'args': {'type': 'static',
'value': args},
f'kwargs': {'type': 'static',
'value': kwargs},
f'operation_id': {'type': 'static',
'value': operation_id}}})
start_doc = run_bundle.start_doc
start_doc['projections'] = projections
start_doc['sample_name'] = 'Workflow Execution'
yield "start", start_doc
yield from descriptors
yield from events
yield "stop", run_bundle.compose_stop()
def emit_start(self, x):
# if we have seen this start document already do nothing, we have
# multiple parents so we may get a start doc multiple times
name, doc = x
if doc["uid"] is self.incoming_start_uid:
return
else:
self.incoming_start_uid = doc["uid"]
# Prime stop document
self.incoming_stop_uid = None
# Emergency stop if we get a new start document and no stop has been
# issued
if self.state != "stopped":
self.emit_stop(x)
# This is a bit of jank to make certain we don't override the
# user metadata with pipeline metadata
old_md = dict(self.md)
self.md.update(
dict(
parent_uids=list(
set(
[
v.start_uid
for k, v in self.translation_nodes.items()
if v.start_uid is not None
]
)
),
parent_node_map={
v.uid: v.start_uid
for k, v in self.translation_nodes.items()
if v.start_uid is not None
},
# keep track of this so we know which node we're sending
# data from (see merkle hash in DBFriendly)
outbound_node=self.uid,
)
)
global _GLOBAL_SCAN_ID
_GLOBAL_SCAN_ID += 1
self.md.update(scan_id=_GLOBAL_SCAN_ID)
bundle = compose_run(metadata=self.md, validate=False)
start, self.desc_fac, self.resc_fac, self.stop_factory = bundle
self.start_uid = start["uid"]
self.md = old_md
# emit starts to subs first in case we create an event from the start
[s.emit_start(x) for s in self.subs]
self.emit(("start", start))
self.state = "started"
def test_array_like():
"Accept any __array__-like as an array."
dask_array = pytest.importorskip("dask.array")
bundle = event_model.compose_run()
desc_bundle = bundle.compose_descriptor(
data_keys={"a": {
"shape": (3, ),
"dtype": "array",
"source": ""
}},
name="primary")
desc_bundle.compose_event_page(data={"a": dask_array.ones((5, 3))},
timestamps={"a": [1, 2, 3]},
seq_num=[1, 2, 3])
def test_bulk_datum_to_datum_page():
run_bundle = event_model.compose_run()
res_bundle = run_bundle.compose_resource(
spec='TIFF', root='/tmp', resource_path='stack.tiff',
resource_kwargs={})
datum1 = res_bundle.compose_datum(datum_kwargs={'slice': 5})
datum2 = res_bundle.compose_datum(datum_kwargs={'slice': 10})
actual = event_model.pack_datum_page(datum1, datum2)
bulk_datum = {'resource': res_bundle.resource_doc['uid'],
'datum_kwarg_list': [datum1['datum_kwargs'],
datum2['datum_kwargs']],
'datum_ids': [datum1['datum_id'], datum2['datum_id']]}
expected = event_model.bulk_datum_to_datum_page(bulk_datum)
assert actual == expected
def gen_stream(
data_lst: tp.List[dict],
metadata: dict,
uid: str = None) -> tp.Generator[tp.Tuple[str, dict], None, None]:
"""Generate a fake doc stream from data and metadata."""
crb = compose_run(metadata=metadata, uid=uid if uid else str(uuid.uuid4()))
yield "start", crb.start_doc
if len(data_lst) == 0:
yield "stop", crb.compose_stop()
cdb: ComposeDescriptorBundle = crb.compose_descriptor(
name="primary", data_keys=compose_data_keys(data_lst[0]))
yield "descriptor", cdb.descriptor_doc
for data in data_lst:
yield "event", cdb.compose_event(data=data,
timestamps=compose_timestamps(data))
yield "stop", crb.compose_stop()
def test_with_run_router(tmp_path, md):
# use a directory that does not exist to test that it will be created
output_dir_path = tmp_path / Path("doesnotexist")
def factory(name, doc):
serializer = nxsas.Serializer(file_prefix="doesnotexist/",
directory=output_dir_path)
return [serializer], []
rr = event_model.RunRouter([factory])
start_doc_md = {}
start_doc_md.update(rsoxs_start_doc)
start_doc_md.update(md)
# compose_run will raise an exception if "time" and "uid" are in the metadata
start_doc_md.pop("time")
start_doc_md.pop("uid")
(
start_doc,
compose_descriptor,
compose_resource,
compose_stop,
) = event_model.compose_run(metadata=start_doc_md)
rr("start", start_doc)
descriptor_doc_md = dict()
descriptor_doc_md.update(rsoxs_descriptor_en_doc)
# compose_descriptor will raise an exception if "run_start" is in the metadata
descriptor_doc_md.pop("run_start")
descriptor_doc, compose_event, compose_event_page = compose_descriptor(
**descriptor_doc_md)
rr("descriptor", descriptor_doc)
event_md = dict()
event_md.update(rsoxs_event_page_en_doc)
# event_md["seq_num"] = [1]
# the descriptor uid will interfere with compose_event
event_md.pop("descriptor")
event_doc = compose_event(**event_md)
rr("event", event_doc)
stop_doc = compose_stop()
rr("stop", stop_doc)
print(os.listdir(path=output_dir_path))
assert len(os.listdir(path=output_dir_path)) == 1
def ingest(paths):
"""Ingest a generic image file.
Can be used to load any formats that FabIO can read.
Parameters
----------
paths: List
List of file paths to ingest. Note that only a single path is currently supported.
"""
assert len(paths) == 1
path = paths[0]
image = fabio.open(path) # type: fabio.fabioimage.FabioImage
# Create a BlueskyRun
run_bundle = event_model.compose_run()
# Create the start document
start_doc = run_bundle.start_doc
start_doc["sample_name"] = image.filename
yield "start", start_doc
# Create the descriptor document - defines the data keys
source = image.header.get("source", "Local")
frame_data_keys = {
"raw": {"source": source, "dtype": "number", "shape": (image.nframes, *image.shape)}
}
frame_stream_name = "primary"
frame_stream_bundle = run_bundle.compose_descriptor(
data_keys=frame_data_keys, name=frame_stream_name
)
yield "descriptor", frame_stream_bundle.descriptor_doc
# Create the event document - contains the data
# delayed_get_slice = dask.delayed(_get_slice)
# dask_data = da.stack(
# [da.from_delayed(delayed_get_slice(image, frame), shape=image.shape, dtype=image.dtype)
# for frame in range(image.nframes)]
# )
dask_data = da.stack([frame.data for frame in image.frames()])
timestamp = image.header.get("time", time.time())
yield "event", frame_stream_bundle.compose_event(
data={"raw": dask_data}, timestamps={"raw": timestamp}
)
# Create the stop document
yield "stop", run_bundle.compose_stop()
def test_no_events():
run_bundle = event_model.compose_run()
start_doc = run_bundle.start_doc
desc_bundle = run_bundle.compose_descriptor(
data_keys={'x': {'source': '...', 'shape': [], 'dtype': 'number'}},
name='primary')
descriptor_doc = desc_bundle.descriptor_doc
stop_doc = run_bundle.compose_stop()
documents_to_xarray(
start_doc=start_doc,
stop_doc=stop_doc,
descriptor_docs=[descriptor_doc],
get_event_pages=no_event_pages,
filler=event_model.Filler({}, inplace=True),
get_resource=None,
lookup_resource_for_datum=None,
get_datum_pages=None)
def start(self, doc):
metadata = {'raw_uid': doc['uid'],
'raw_scan_id': doc['scan_id'],
'processor_parameters': doc['param']}
self.compose_run_bundle = compose_run(metadata=metadata)
return self.compose_run_bundle.start_doc
