def run_list_to_dict(embedded_run_list): """ Converts a run from the mongo database to a dictionary. """ run_dict = { 'start': {}, 'stop': {}, 'descriptor': [], 'resource': [], 'event': [], 'datum': [] } header = embedded_run_list[0][1] run_dict['start'] = header['start'][0] run_dict['stop'] = header['stop'][0] run_dict['descriptor'] = header.get('descriptors', []) run_dict['resource'] = header.get('resources', []) for name, doc in embedded_run_list[1:]: if name == 'event': run_dict['event'] += list(event_model.unpack_event_page(doc)) elif name == 'datum': run_dict['datum'] += list(event_model.unpack_datum_page(doc)) return run_dict
def collector(name, doc): if name == 'event_page': name = 'event' doc, = event_model.unpack_event_page(doc) elif name == 'datum_page': name = 'datum' doc, = event_model.unpack_datum_page(doc) collected.append((name, doc))
def datum_page(self, doc): # Unpack an DatumPage into Datum and do the actual insert inside # the `datum` method. (This is the oppose what DocumentRouter does by # default.) datum_method = self.datum # Avoid attribute lookup in hot loop. filled_datums = [] for datum_doc in event_model.unpack_datum_page(doc): filled_datums.append(datum_method(datum_doc))
def test_round_trip_datum_page_with_empty_data(): datum_page = { 'datum_id': ['a', 'b', 'c'], 'resource': 'd', 'datum_kwargs': {} } datums = list(event_model.unpack_datum_page(datum_page)) assert len(datums) == 3 page_again = event_model.pack_datum_page(*datums) assert page_again == datum_page
def lookup_resource_for_datum(datum_id): """ This method is likely very slow. """ resources = [resource['uid'] for resource in get_header_field('resources')] datum_page = self.catalog._db.datum.find_one( {'$and': [{'resource': {'$in': resources}}, {'datum_id': datum_id}]}, {'_id': False}) for datum in event_model.unpack_datum_page(datum_page): if datum['datum_id'] == datum_id: return datum['resource'] raise ValueError(f"Could not find Datum with datum_id={datum_id}")
def normalize(gen): """ Converted any pages to singles. """ for name, doc in gen: if name == 'event_page': for event in event_model.unpack_event_page(doc): yield 'event', event elif name == 'datum_page': for datum in event_model.unpack_datum_page(doc): yield 'datum', datum else: yield name, doc
def normalize(gen): """ Converted any pages to singles. """ for name, doc in gen: if name == "event_page": for event in event_model.unpack_event_page(doc): yield "event", event elif name == "datum_page": for datum in event_model.unpack_datum_page(doc): yield "datum", datum else: yield name, doc
def page_to_list(name, page): """ Converts event/datum_page to event/datum lists. """ doc_list = [] if name == 'event_page': doc_list.extend([['event', event] for event in event_model.unpack_event_page(page)]) if name == 'datum_page': doc_list.extend([['datum', datum] for datum in event_model.unpack_datum_page(page)]) return doc_list
def pages_to_list(pages): """ Converts event/datum_page to event/datum lists. """ doc_list = [] for page in pages: if page[0] == 'event_page': doc_list.extend( [['event', event] for event in event_model.unpack_event_page(page[1])]) if page[0] == 'datum_page': doc_list.extend( [['datum', datum] for datum in event_model.unpack_datum_page(page[1])]) return doc_list
zmq_publisher = Publisher( f"{args.zmq_host}:{args.zmq_publish_port}", prefix=args.zmq_publish_prefix.encode() ) extra = count() for name, doc in db[args.run_id].documents(): print(f"trying to emit {name}") doc = dict(doc) if name == "descriptor": doc["data_keys"]["extra"] = { "dtype": "number", "source": "computed", "units": "arb", "shape": [], } zmq_publisher("descriptor", doc) if name == "event_page": for ev in unpack_event_page(doc): for j in range(5): new_seq = next(extra) ev["seq_num"] = new_seq ev["uid"] = str(uuid.uuid4()) ev["data"]["extra"] = 5 ev["timestamps"]["extra"] = 0 zmq_publisher("event", ev) elif name == "datum_page": for datum in unpack_datum_page(doc): zmq_publisher("datum", datum) else: zmq_publisher(name, doc)
def test_round_trip_pagination(): run_bundle = event_model.compose_run() desc_bundle = run_bundle.compose_descriptor(data_keys={ 'motor': { 'shape': [], 'dtype': 'number', 'source': '...' }, 'image': { 'shape': [512, 512], 'dtype': 'number', 'source': '...', 'external': 'FILESTORE:' } }, name='primary') res_bundle = run_bundle.compose_resource(spec='TIFF', root='/tmp', resource_path='stack.tiff', resource_kwargs={}) datum_doc1 = res_bundle.compose_datum(datum_kwargs={'slice': 5}) datum_doc2 = res_bundle.compose_datum(datum_kwargs={'slice': 10}) datum_doc3 = res_bundle.compose_datum(datum_kwargs={'slice': 15}) event_doc1 = desc_bundle.compose_event(data={ 'motor': 0, 'image': datum_doc1['datum_id'] }, timestamps={ 'motor': 0, 'image': 0 }, filled={'image': False}, seq_num=1) event_doc2 = desc_bundle.compose_event(data={ 'motor': 1, 'image': datum_doc2['datum_id'] }, timestamps={ 'motor': 0, 'image': 0 }, filled={'image': False}, seq_num=1) event_doc3 = desc_bundle.compose_event(data={ 'motor': 2, 'image': datum_doc3['datum_id'] }, timestamps={ 'motor': 0, 'image': 0 }, filled={'image': False}, seq_num=1) # Round trip single event -> event_page -> event. expected = event_doc1 actual, = event_model.unpack_event_page( event_model.pack_event_page(expected)) assert actual == expected # Round trip two events -> event_page -> events. expected = [event_doc1, event_doc2] actual = list( event_model.unpack_event_page(event_model.pack_event_page(*expected))) assert actual == expected # Round trip three events -> event_page -> events. expected = [event_doc1, event_doc2, event_doc3] actual = list( event_model.unpack_event_page(event_model.pack_event_page(*expected))) assert actual == expected # Round trip on docs that don't have a filled key unfilled_doc1 = event_doc1 unfilled_doc1.pop('filled') unfilled_doc2 = event_doc2 unfilled_doc2.pop('filled') unfilled_doc3 = event_doc3 unfilled_doc3.pop('filled') expected = [unfilled_doc1, unfilled_doc2, unfilled_doc3] actual = list( event_model.unpack_event_page(event_model.pack_event_page(*expected))) for doc in actual: doc.pop('filled') assert actual == expected # Round trip one datum -> datum_page -> datum. expected = datum_doc1 actual, = event_model.unpack_datum_page( event_model.pack_datum_page(expected)) assert actual == expected # Round trip two datum -> datum_page -> datum. expected = [datum_doc1, datum_doc2] actual = list( event_model.unpack_datum_page(event_model.pack_datum_page(*expected))) assert actual == expected # Round trip three datum -> datum_page -> datum. expected = [datum_doc1, datum_doc2, datum_doc3] actual = list( event_model.unpack_datum_page(event_model.pack_datum_page(*expected))) assert actual == expected # Check edge case where datum_kwargs are empty. datum_doc1 = res_bundle.compose_datum(datum_kwargs={}) datum_doc2 = res_bundle.compose_datum(datum_kwargs={}) datum_doc3 = res_bundle.compose_datum(datum_kwargs={}) # Round trip one datum -> datum_page -> datum. expected = datum_doc1 actual, = event_model.unpack_datum_page( event_model.pack_datum_page(expected)) assert actual == expected # Round trip two datum -> datum_page -> datum. expected = [datum_doc1, datum_doc2] actual = list( event_model.unpack_datum_page(event_model.pack_datum_page(*expected))) assert actual == expected # Round trip three datum -> datum_page -> datum. expected = [datum_doc1, datum_doc2, datum_doc3] actual = list( event_model.unpack_datum_page(event_model.pack_datum_page(*expected))) assert actual == expected
def datum_kwarg_gen(): for page in run._get_datum_pages(resource['uid']): for datum in event_model.unpack_datum_page(page): yield datum['datum_kwargs']