def _cache_run_start(run_start, run_start_cache):
"""De-reference and cache a RunStart document
The de-referenced Document is cached against the
ObjectId and the uid -> ObjectID mapping is stored.
Parameters
----------
run_start : dict
raw pymongo dictionary. This is expected to have
an entry `_id` with the ObjectId used by mongo.
Returns
-------
run_start : doc.Document
Document instance for this RunStart document.
The ObjectId has been stripped.
"""
run_start = dict(run_start)
# TODO actually do this de-reference for documents that have it
# There is no known actually usage of this document and it is not being
# created going forward
run_start.pop('beamline_config_id', None)
# get the mongo ObjectID
oid = run_start.pop('_id', None)
# convert the remaining document to a Document object
run_start = doc.Document('RunStart', run_start)
run_start_cache[run_start['uid']] = run_start
run_start_cache[oid] = run_start
return run_start
def find_events(start_col, start_cache,
descriptor_col, descriptor_cache,
event_col, tz, descriptor=None, **kwargs):
"""Given search criteria, locate Event Documents.
Parameters
-----------
start_time : time-like, optional
time-like representation of the earliest time that an Event
was created. Valid options are:
- timestamps --> time.time()
- '2015'
- '2015-01'
- '2015-01-30'
- '2015-03-30 03:00:00'
- datetime.datetime.now()
stop_time : time-like, optional
timestamp of the latest time that an Event was created. See
docs for `start_time` for examples.
descriptor : doc.Document or str, optional
Find events for a given EventDescriptor
uid : str, optional
Globally unique id string provided to metadatastore
Returns
-------
events : iterable of doc.Document objects
"""
# Some user-friendly error messages for an easy mistake to make
if 'event_descriptor' in kwargs:
raise ValueError("Use 'descriptor', not 'event_descriptor'.")
if descriptor:
descriptor_uid = doc_or_uid_to_uid(descriptor)
kwargs['descriptor'] = descriptor_uid
_format_time(kwargs, tz)
col = event_col
events = col.find(kwargs,
sort=[('descriptor', pymongo.DESCENDING),
('time', pymongo.ASCENDING)],
no_cursor_timeout=True)
try:
for ev in events:
ev.pop('_id', None)
# pop the descriptor oid
desc_uid = ev.pop('descriptor')
# replace it with the defererenced descriptor
ev['descriptor'] = descriptor_given_uid(desc_uid, descriptor_col,
descriptor_cache,
start_col, start_cache)
# wrap it our fancy dict
ev = doc.Document('Event', ev)
yield ev
finally:
events.close()
def wrap_in_doct(name, doc):
"""
Put document contents into a doct.Document object.
A ``doct.Document`` is a subclass of dict that:
* is immutable
* provides human-readable :meth:`__repr__` and :meth:`__str__`
* supports dot access (:meth:`__getattr__`) as a synonym for item access
(:meth:`__getitem__`) whenever possible
"""
return doct.Document(DOCT_NAMES[name], doc)
def get_events_generator(descriptor, event_col, descriptor_col,
descriptor_cache, run_start_col,
run_start_cache, convert_arrays=True):
"""A generator which yields all events from the event stream
Parameters
----------
descriptor : doc.Document or dict or str
The EventDescriptor to get the Events for. Can be either
a Document/dict with a 'uid' key or a uid string
convert_arrays: boolean, optional
convert 'array' type to numpy.ndarray; True by default
Yields
------
event : doc.Document
All events for the given EventDescriptor from oldest to
newest
"""
descriptor_uid = doc_or_uid_to_uid(descriptor)
descriptor = descriptor_given_uid(descriptor_uid, descriptor_col,
descriptor_cache, run_start_col,
run_start_cache)
col = event_col
ev_cur = col.find({'descriptor': descriptor_uid},
sort=[('descriptor', pymongo.DESCENDING),
('time', pymongo.ASCENDING)])
data_keys = descriptor['data_keys']
external_keys = [k for k in data_keys if 'external' in data_keys[k]]
for ev in ev_cur:
# ditch the ObjectID
del ev['_id']
# replace descriptor with the defererenced descriptor
ev['descriptor'] = descriptor
for k, v in ev['data'].items():
_dk = data_keys[k]
# convert any arrays stored directly in mds into ndarray
if convert_arrays:
if _dk['dtype'] == 'array' and not _dk.get('external', False):
ev['data'][k] = np.asarray(ev['data'][k])
# note which keys refer to dereferences (external) data
ev['filled'] = {k: False for k in external_keys}
# wrap it in our fancy dict
ev = doc.Document('Event', ev)
yield ev
def compare(o, n):
try:
assert o['uid'] == n['uid']
if 'reason' in o and o['reason'] == '':
d_o = dict(o)
del d_o['reason']
o = doct.Document('RunStop', d_o)
assert o == n
except AssertionError:
if o['run_start']['beamline_id'] in ['CSX', 'xf23id', 'CSX-1']:
pass
else:
print(o)
print(n)
raise
def _cache_descriptor(descriptor, descritor_cache,
run_start_col, run_start_cache):
"""De-reference and cache a RunStop document
The de-referenced Document is cached against the
ObjectId and the uid -> ObjectID mapping is stored.
Parameters
----------
descriptor : dict
raw pymongo dictionary. This is expected to have
an entry `_id` with the ObjectId used by mongo.
Returns
-------
descriptor : doc.Document
Document instance for this EventDescriptor document.
The ObjectId has been stripped.
"""
descriptor = dict(descriptor)
# pop the ObjectID
oid = descriptor.pop('_id', None)
try:
run_start_uid = descriptor['run_start']
except KeyError:
run_start_uid = descriptor.pop('run_start_id')
# do the run_start referencing
descriptor['run_start'] = run_start_given_uid(run_start_uid,
run_start_col,
run_start_cache)
# create the Document instance
descriptor = doc.Document('EventDescriptor', descriptor)
descritor_cache[descriptor['uid']] = descriptor
descritor_cache[oid] = descriptor
return descriptor
def _cache_run_stop(run_stop, run_stop_cache,
run_start_col, run_start_cache):
"""De-reference and cache a RunStop document
The de-referenced Document is cached against the
ObjectId and the uid -> ObjectID mapping is stored.
Parameters
----------
run_stop : dict
raw pymongo dictionary. This is expected to have
an entry `_id` with the ObjectId used by mongo.
Returns
-------
run_stop : doc.Document
Document instance for this RunStop document.
The ObjectId has been stripped.
"""
run_stop = dict(run_stop)
# pop off the ObjectId of this document
oid = run_stop.pop('_id', None)
try:
run_start_uid = run_stop['run_start']
except KeyError:
run_start_uid = run_stop.pop('run_start_id')
# do the run-start de-reference
run_stop['run_start'] = run_start_given_uid(run_start_uid,
run_start_col,
run_start_cache)
# create the Document object
run_stop = doc.Document('RunStop', run_stop)
run_stop_cache[run_stop['uid']] = run_stop
run_stop_cache[oid] = run_stop
return run_stop
def start(self, doc):
self.filenames = []
# Convert doc from dict into dottable dict, more convenient
# in Python format strings: doc.key == doc['key']
self._start = doct.Document('start', doc)
super().start(doc)