def run_list_to_dict(embedded_run_list):
"""
Converts a run from the mongo database to a dictionary.
"""
run_dict = {
'start': {},
'stop': {},
'descriptor': [],
'resource': [],
'event': [],
'datum': []
}
header = embedded_run_list[0][1]
run_dict['start'] = header['start'][0]
run_dict['stop'] = header['stop'][0]
run_dict['descriptor'] = header.get('descriptors', [])
run_dict['resource'] = header.get('resources', [])
for name, doc in embedded_run_list[1:]:
if name == 'event':
run_dict['event'] += list(event_model.unpack_event_page(doc))
elif name == 'datum':
run_dict['datum'] += list(event_model.unpack_datum_page(doc))
return run_dict
def collector(name, doc):
if name == 'event_page':
name = 'event'
doc, = event_model.unpack_event_page(doc)
elif name == 'datum_page':
name = 'datum'
doc, = event_model.unpack_datum_page(doc)
collected.append((name, doc))
def datum_page(self, doc):
# Unpack an DatumPage into Datum and do the actual insert inside
# the `datum` method. (This is the oppose what DocumentRouter does by
# default.)
datum_method = self.datum # Avoid attribute lookup in hot loop.
filled_datums = []
for datum_doc in event_model.unpack_datum_page(doc):
filled_datums.append(datum_method(datum_doc))
def test_round_trip_datum_page_with_empty_data():
datum_page = {
'datum_id': ['a', 'b', 'c'],
'resource': 'd',
'datum_kwargs': {}
}
datums = list(event_model.unpack_datum_page(datum_page))
assert len(datums) == 3
page_again = event_model.pack_datum_page(*datums)
assert page_again == datum_page
def lookup_resource_for_datum(datum_id):
""" This method is likely very slow. """
resources = [resource['uid']
for resource in get_header_field('resources')]
datum_page = self.catalog._db.datum.find_one(
{'$and': [{'resource': {'$in': resources}},
{'datum_id': datum_id}]},
{'_id': False})
for datum in event_model.unpack_datum_page(datum_page):
if datum['datum_id'] == datum_id:
return datum['resource']
raise ValueError(f"Could not find Datum with datum_id={datum_id}")
def normalize(gen):
"""
Converted any pages to singles.
"""
for name, doc in gen:
if name == 'event_page':
for event in event_model.unpack_event_page(doc):
yield 'event', event
elif name == 'datum_page':
for datum in event_model.unpack_datum_page(doc):
yield 'datum', datum
else:
yield name, doc
def normalize(gen):
"""
Converted any pages to singles.
"""
for name, doc in gen:
if name == "event_page":
for event in event_model.unpack_event_page(doc):
yield "event", event
elif name == "datum_page":
for datum in event_model.unpack_datum_page(doc):
yield "datum", datum
else:
yield name, doc
def page_to_list(name, page):
"""
Converts event/datum_page to event/datum lists.
"""
doc_list = []
if name == 'event_page':
doc_list.extend([['event', event]
for event in event_model.unpack_event_page(page)])
if name == 'datum_page':
doc_list.extend([['datum', datum]
for datum in event_model.unpack_datum_page(page)])
return doc_list
def pages_to_list(pages):
"""
Converts event/datum_page to event/datum lists.
"""
doc_list = []
for page in pages:
if page[0] == 'event_page':
doc_list.extend(
[['event', event]
for event in event_model.unpack_event_page(page[1])])
if page[0] == 'datum_page':
doc_list.extend(
[['datum', datum]
for datum in event_model.unpack_datum_page(page[1])])
return doc_list
zmq_publisher = Publisher(
f"{args.zmq_host}:{args.zmq_publish_port}", prefix=args.zmq_publish_prefix.encode()
)
extra = count()
for name, doc in db[args.run_id].documents():
print(f"trying to emit {name}")
doc = dict(doc)
if name == "descriptor":
doc["data_keys"]["extra"] = {
"dtype": "number",
"source": "computed",
"units": "arb",
"shape": [],
}
zmq_publisher("descriptor", doc)
if name == "event_page":
for ev in unpack_event_page(doc):
for j in range(5):
new_seq = next(extra)
ev["seq_num"] = new_seq
ev["uid"] = str(uuid.uuid4())
ev["data"]["extra"] = 5
ev["timestamps"]["extra"] = 0
zmq_publisher("event", ev)
elif name == "datum_page":
for datum in unpack_datum_page(doc):
zmq_publisher("datum", datum)
else:
zmq_publisher(name, doc)
def test_round_trip_pagination():
run_bundle = event_model.compose_run()
desc_bundle = run_bundle.compose_descriptor(data_keys={
'motor': {
'shape': [],
'dtype': 'number',
'source': '...'
},
'image': {
'shape': [512, 512],
'dtype': 'number',
'source': '...',
'external': 'FILESTORE:'
}
},
name='primary')
res_bundle = run_bundle.compose_resource(spec='TIFF',
root='/tmp',
resource_path='stack.tiff',
resource_kwargs={})
datum_doc1 = res_bundle.compose_datum(datum_kwargs={'slice': 5})
datum_doc2 = res_bundle.compose_datum(datum_kwargs={'slice': 10})
datum_doc3 = res_bundle.compose_datum(datum_kwargs={'slice': 15})
event_doc1 = desc_bundle.compose_event(data={
'motor': 0,
'image': datum_doc1['datum_id']
},
timestamps={
'motor': 0,
'image': 0
},
filled={'image': False},
seq_num=1)
event_doc2 = desc_bundle.compose_event(data={
'motor': 1,
'image': datum_doc2['datum_id']
},
timestamps={
'motor': 0,
'image': 0
},
filled={'image': False},
seq_num=1)
event_doc3 = desc_bundle.compose_event(data={
'motor': 2,
'image': datum_doc3['datum_id']
},
timestamps={
'motor': 0,
'image': 0
},
filled={'image': False},
seq_num=1)
# Round trip single event -> event_page -> event.
expected = event_doc1
actual, = event_model.unpack_event_page(
event_model.pack_event_page(expected))
assert actual == expected
# Round trip two events -> event_page -> events.
expected = [event_doc1, event_doc2]
actual = list(
event_model.unpack_event_page(event_model.pack_event_page(*expected)))
assert actual == expected
# Round trip three events -> event_page -> events.
expected = [event_doc1, event_doc2, event_doc3]
actual = list(
event_model.unpack_event_page(event_model.pack_event_page(*expected)))
assert actual == expected
# Round trip on docs that don't have a filled key
unfilled_doc1 = event_doc1
unfilled_doc1.pop('filled')
unfilled_doc2 = event_doc2
unfilled_doc2.pop('filled')
unfilled_doc3 = event_doc3
unfilled_doc3.pop('filled')
expected = [unfilled_doc1, unfilled_doc2, unfilled_doc3]
actual = list(
event_model.unpack_event_page(event_model.pack_event_page(*expected)))
for doc in actual:
doc.pop('filled')
assert actual == expected
# Round trip one datum -> datum_page -> datum.
expected = datum_doc1
actual, = event_model.unpack_datum_page(
event_model.pack_datum_page(expected))
assert actual == expected
# Round trip two datum -> datum_page -> datum.
expected = [datum_doc1, datum_doc2]
actual = list(
event_model.unpack_datum_page(event_model.pack_datum_page(*expected)))
assert actual == expected
# Round trip three datum -> datum_page -> datum.
expected = [datum_doc1, datum_doc2, datum_doc3]
actual = list(
event_model.unpack_datum_page(event_model.pack_datum_page(*expected)))
assert actual == expected
# Check edge case where datum_kwargs are empty.
datum_doc1 = res_bundle.compose_datum(datum_kwargs={})
datum_doc2 = res_bundle.compose_datum(datum_kwargs={})
datum_doc3 = res_bundle.compose_datum(datum_kwargs={})
# Round trip one datum -> datum_page -> datum.
expected = datum_doc1
actual, = event_model.unpack_datum_page(
event_model.pack_datum_page(expected))
assert actual == expected
# Round trip two datum -> datum_page -> datum.
expected = [datum_doc1, datum_doc2]
actual = list(
event_model.unpack_datum_page(event_model.pack_datum_page(*expected)))
assert actual == expected
# Round trip three datum -> datum_page -> datum.
expected = [datum_doc1, datum_doc2, datum_doc3]
actual = list(
event_model.unpack_datum_page(event_model.pack_datum_page(*expected)))
assert actual == expected
def datum_kwarg_gen():
for page in run._get_datum_pages(resource['uid']):
for datum in event_model.unpack_datum_page(page):
yield datum['datum_kwargs']
