def stop_by_start(run_start):
"""Given a RunStart return it's RunStop
Raises if no RunStop exists.
Parameters
----------
run_start : utils.Document or dict or str
The RunStart to get the RunStop for. Can be either
a Document/dict with a 'uid' key or a uid string.
Returns
-------
run_stop : utils.Document
The RunStop document
Raises
------
NoRunStop
If no RunStop document exists for the given RunStart.
"""
rstart = doc_or_uid_to_uid(run_start)
try:
run_stop = next(find_run_stops(run_start=rstart))
return utils.Document('RunStop', run_stop)
except StopIteration:
raise NoRunStop("No run stop exists for {!r}".format(run_start))
def descriptors_by_start(run_start):
"""Given a RunStart return a list of it's descriptors
Raises if no EventDescriptors exist.
Parameters
----------
run_start : utils.Document or dict or str
The RunStart to get the EventDescriptors for. Can be either
a Document/dict with a 'uid' key or a uid string
Returns
-------
event_descriptors : list
A list of EventDescriptor documents
Raises
------
NoEventDescriptors
If no EventDescriptor documents exist for the given RunStart
"""
run_start_id = doc_or_uid_to_uid(run_start)
res = find_descriptors(run_start=run_start_id)
e_descs = [utils.Document('EventDescriptor', r) for r in res]
if not e_descs:
raise NoEventDescriptors("No EventDescriptors exists "
"for {!r}".format(run_start))
return e_descs
def stop_by_start(run_start):
"""Given a RunStart return it's RunStop
Raises if no RunStop exists.
Parameters
----------
run_start : doc.Document or dict or str
The RunStart to get the RunStop for. Can be either
a Document/dict with a 'uid' key or a uid string
Returns
-------
run_stop : utils.Document
The RunStop document
Raises
------
NoRunStop
If no RunStop document exists for the given RunStart
"""
# run_start_uid = doc_or_uid_to_uid(run_start)
# rstart = run_start_given_uid(run_start.uid)
# if rstart:
run_stop = next(find_run_stops(run_start=run_start.uid))
# run_stop['run_start'] = run_start
return utils.Document('RunStop', run_stop)
def insert_run_start(time,
scan_id,
config,
beamline_id,
beamline_config={},
uid=None,
owner=None,
group=None,
project=None,
custom=None):
"""Provide a head for a sequence of events. Entry point for an
experiment's run.
Parameters
----------
time : float
The date/time as found at the client side when an event is
created.
scan_id : int
Unique scan identifier visible to the user and data analysis
beamline_id : str
Beamline String identifier. Not unique, just an indicator of
beamline code for multiple beamline systems
beamline_config : metadatastore.documents.Document or str
if Document:
The metadatastore beamline config document
if str:
uid of beamline config corresponding to a given run
uid : str, optional
Globally unique id string provided to metadatastore
owner : str, optional
A username associated with the entry
group : str, optional
A group (e.g., UNIX group) associated with the entry
project : str, optional
Any project name to help users locate the data
custom: dict, optional
Any additional information that data acquisition code/user wants
to append to the Header at the start of the run.
Returns
----------
utils.Document
The run_start document that is successfully saved in mongo
"""
data = locals()
if custom:
data.update(custom)
payload = ujson.dumps(data)
r = requests.post(_server_path + '/run_start', data=payload)
if r.status_code != 200:
raise Exception("Server cannot complete the request", r)
else:
return utils.Document('RunStart', r.json())
def find_descriptors(run_start=None, **kwargs):
"""Given search criteria, locate EventDescriptor Documents.
Parameters
----------
run_start : metadatastore.document.Document or uid, optional
The metadatastore run start document or the metadatastore uid to get
the corresponding run end for
run_start_uid : str
Globally unique id string provided to metadatastore for the
RunStart Document.
start_time : time-like, optional
time-like representation of the earliest time that an EventDescriptor
was created. Valid options are:
- timestamps --> time.time()
- '2015'
- '2015-01'
- '2015-01-30'
- '2015-03-30 03:00:00'
- datetime.datetime.now()
stop_time : time-like, optional
timestamp of the latest time that an EventDescriptor was created. See
docs for `start_time` for examples.
uid : str, optional
Globally unique id string provided to metadatastore
Returns
-------
content : iterable of list of json documents
We need lists to be able to JSON encode multiple dicts. We can return an iterator of
iterator?
"""
_format_time(kwargs)
query = kwargs
rstart = None
if run_start:
query['run_start'] = doc_or_uid_to_uid(run_start)
try:
rstart = next(find_run_starts(uid=query['run_start']))
except StopIteration:
raise NoRunStart()
r = requests.get(_server_path + "/event_descriptor",
params=ujson.dumps(query))
if r.status_code != 200:
return None
else:
content = ujson.loads(r.text)
if not content:
return None
else:
for c in content:
if rstart is None:
rstart = next(find_run_starts(uid=c['run_start']))
c['run_start'] = rstart
yield utils.Document('EventDescriptor', c)
def find_last(num=1):
"""Return the last 'num' many headers"""
query = dict(num=num)
r = requests.get(_server_path + "/run_start", params=ujson.dumps(query))
if r.status_code != 200:
return None
content = ujson.loads(r.text)
if not content:
return None
else:
for c in content:
yield utils.Document('RunStart', c)
def find_events(descriptor=None, **kwargs):
"""
Parameters
-----------
start_time : time-like, optional
time-like representation of the earliest time that an Event
was created. Valid options are:
- timestamps --> time.time()
- '2015'
- '2015-01'
- '2015-01-30'
- '2015-03-30 03:00:00'
- datetime.datetime.now()
stop_time : time-like, optional
timestamp of the latest time that an Event was created. See
docs for `start_time` for examples.
descriptor : doc.Document or str, optional
Find events for a given EventDescriptor
uid : str, optional
Globally unique id string provided to metadatastore
Yields
-------
events : iterable of utils.Document objects
"""
# TODO: Add more tests!!! Make sure descriptor is returned for each event
if 'event_descriptor' in kwargs:
raise ValueError("Use 'descriptor', not 'event_descriptor'.")
query = kwargs
desc = None
if descriptor:
desc_uid = doc_or_uid_to_uid(descriptor)
query['descriptor'] = desc_uid
_format_time(query)
r = requests.get(_server_path + "/event",
params=ujson.dumps(query),
stream=True)
r.raise_for_status()
content = ujson.loads(r.text)
if not content:
return None
else:
if descriptor:
desc = next(find_descriptors(uid=doc_or_uid_to_uid(descriptor)))
for c in content:
if not desc:
# Obvious bottleneck!!!
# Fix using local caching!!!!
desc = next(find_descriptors(uid=c['descriptor']))
c['descriptor'] = desc
yield utils.Document('Event', c)
def run_start_given_uid(uid):
"""Given a uid, return the RunStart document
Parameters
----------
uid : str
The uid
Returns
-------
run_start : utils.Document
The RunStart document.
"""
return utils.Document('RunStart', next(find_run_starts(uid=uid)))
def descriptor_given_uid(uid):
"""Given a uid, return the EventDescriptor document
Parameters
----------
uid : str
The uid
Returns
-------
EventDescriptor : utils.Document
The EventDescriptor document fully de-referenced
"""
return utils.Document('EventDescriptor', next(find_descriptors(uid=uid)))
def find_run_stops(run_start=None, **kwargs):
"""Given search criteria, query for RunStop Documents.
Parameters
----------
run_start : utils.Document or str, optional
The metadatastore run start document or the metadatastore uid to get
the corresponding run end for
start_time : time-like, optional
time-like representation of the earliest time that a RunStop
was created. Valid options are:
- timestamps --> time.time()
- '2015'
- '2015-01'
- '2015-01-30'
- '2015-03-30 03:00:00'
- datetime.datetime.now()
stop_time : time-like, optional
timestamp of the latest time that a RunStop was created. See
docs for `start_time` for examples.
exit_status : {'success', 'fail', 'abort'}, optional
provides information regarding the run success.
reason : str, optional
Long-form description of why the run was terminated.
uid : str, optional
Globally unique id string provided to metadatastore
Yields utils.Document
------
RunStop documents if found in the query
"""
_format_time(kwargs)
query = kwargs
rstart = None
if run_start:
query['run_start'] = doc_or_uid_to_uid(run_start)
rstart = next(find_run_starts(uid=query['run_start']))
r = requests.get(_server_path + "/run_stop", params=ujson.dumps(query))
if r.status_code != 200:
return None
content = ujson.loads(r.text)
if not content:
return None
else:
for c in content:
if rstart is None:
rstart = next(find_run_starts(uid=c['run_start']))
c['run_start'] = rstart
yield utils.Document('RunStop', c)
def insert_run_stop(run_start,
time,
uid=None,
config={},
exit_status='success',
reason=None,
custom=None):
""" Provide an end to a sequence of events. Exit point for an
experiment's run.
Parameters
----------
run_start : metadatastore.documents.Document or str
if Document:
The metadatastore RunStart document
if str:
uid of RunStart object to associate with this record
time : float
The date/time as found at the client side when an event is
created.
uid : str, optional
Globally unique id string provided to metadatastore
exit_status : {'success', 'abort', 'fail'}, optional
indicating reason run stopped, 'success' by default
reason : str, optional
more detailed exit status (stack trace, user remark, etc.)
custom : dict, optional
Any additional information that data acquisition code/user wants
to append to the Header at the end of the run.
Returns
-------
run_stop : mongoengine.Document
Inserted mongoengine object
"""
params = locals()
rs_obj = params.pop('run_start')
try:
params['run_start'] = rs_obj.uid
except AttributeError:
params['run_start'] = rs_obj
payload = ujson.dumps(params)
r = requests.post(_server_path + '/run_stop', data=payload)
if r.status_code != 200:
raise Exception("Server cannot complete the request", r)
else:
return utils.Document('RunStop', r.json())
def runstop_given_uid(uid):
"""Given a uid, return the RunStop document.
Parameters
----------
uid : str
The uid
Returns
-------
run_stop : utils.Document
The RunStop document fully de-referenced
"""
run_stop = dict(next(find_run_stops(uid=uid)))
start_oid = run_stop.pop('run_start_id')
run_stop['run_start'] = _run_start_given_oid(start_oid)
return utils.Document('RunStop', run_stop)
def descriptor_given_uid(uid):
"""Given a uid, return the EventDescriptor document
Parameters
----------
uid : str
The uid
Returns
-------
descriptor : utils.Document
The EventDescriptor document fully de-referenced
"""
descriptor = dict(next(find_descriptors(uid=uid)))
start_oid = descriptor.pop('run_start_id')
descriptor['run_start'] = _run_start_given_oid(start_oid)
return utils.Document('EventDescriptor', descriptor)
def get_events_generator(descriptor):
"""A generator which yields all events from the event stream
Parameters
----------
descriptor : doc.Document or dict or str
The EventDescriptor to get the Events for. Can be either
a Document/dict with a 'uid' key or a uid string
Yields
------
event : utils.Document
All events for the given EventDescriptor from oldest to
newest
"""
descriptor_uid = doc_or_uid_to_uid(descriptor)
ev_cur = find_events(descriptor=descriptor_uid)
desc = next(find_descriptors(uid=descriptor_uid))
for ev in ev_cur:
ev = dict(ev)
ev['descriptor'] = desc
yield utils.Document('Event', ev)
def fetch_events_generator(descriptor):
"""A generator which yields all events from the event stream
Parameters
----------
descriptor : doc.Document or dict or str
The EventDescriptor to get the Events for. Can be either
a Document/dict with a 'uid' key or a uid string
Yields
------
event : utils.Document
All events for the given EventDescriptor from oldest to
newest
"""
desc = descriptor_given_uid(descriptor.uid)
raw_ev_gen = find_events(descriptor_id=str(desc['_id']))
for ev in raw_ev_gen:
ev['descriptor'] = dict(desc)
yield utils.Document('Event', ev)
def _run_start_given_oid(oid):
"""Get RunStart document given an ObjectId
This is an internal function as ObjectIds should not be
leaking out to user code.
When we migrate to using uid as the primary key this function
will be removed.
Parameters
----------
oid : ObjectId
Mongo's unique identifier for the document. This is currently
used to implement foreign keys
Returns
-------
run_start : utils.Document
The RunStart document.
"""
return utils.Document('RunStart', next(find_run_starts(_id=oid)))
def insert_descriptor(run_start, data_keys, time, uid=None, custom=None):
""" Create an event_descriptor in metadatastore server backend
Parameters
----------
run_start : metadatastore.documents.Document or str
if Document:
The metadatastore RunStart document
if str:
uid of RunStart object to associate with this record
data_keys : dict
Provides information about keys of the data dictionary in
an event will contain
time : float
The date/time as found at the client side when an event
descriptor is created.
uid : str, optional
Globally unique id string provided to metadatastore
custom : dict, optional
Any additional information that data acquisition code/user wants
to append to the EventDescriptor.
Returns
-------
ev_desc : EventDescriptor
The document added to the collection.
"""
payload = locals()
tmp = payload['run_start'].uid
payload.pop('run_start')
payload['run_start'] = tmp
r = requests.post(_server_path + '/event_descriptor',
data=ujson.dumps(payload))
if r.status_code != 200:
raise Exception("Server cannot complete the request", r)
else:
return utils.Document('EventDescriptor', dict(r.json()))
def _descriptor_given_oid(oid):
"""Get EventDescriptor document given an ObjectId
This is an internal function as ObjectIds should not be
leaking out to user code.
When we migrate to using uid as the primary key this function
will be removed.
Parameters
----------
oid : ObjectId
Mongo's unique identifier for the document. This is currently
used to implement foreign keys
Returns
-------
descriptor : doc.Document
The EventDescriptor document.
"""
#runstart replaced in server side
descriptor = dict(next(find_descriptors(oid=oid)))
return utils.Document('EventDescriptor', descriptor)
def monitor_run_stop():
r = requests.get(_server_path + '/run_stop_capped')
content = ujson.loads(r.text)
yield utils.Document('RunStop', content)
def monitor_run_start():
raise NotImplementedError(
'Not this cycle. Code works, needs capped collection support')
r = requests.get(_server_path + '/run_start_capped')
content = ujson.loads(r.text)
yield utils.Document('RunStart', content)
def find_run_starts(range_floor=0, range_ceil=50, **kwargs):
"""Given search criteria, locate RunStart Documents.
As we describe in design document, time here is strictly the time
server entry was created, not IOC timestamp. For the difference, refer
to: nsls-ii.github.io
Parameters
----------
start_time : time-like, optional
time-like representation of the earliest time that a RunStart
was created. Valid options are:
- timestamps --> time.time()
- '2015'
- '2015-01'
- '2015-01-30'
- '2015-03-30 03:00:00'
- datetime.datetime.now()
stop_time : time-like, optional
timestamp of the latest time that a RunStart was created. See
docs for `start_time` for examples.
`beamline_id : str, optional
String identifier for a specific beamline
project : str, optional
Project name
owner : str, optional
The username of the logged-in user when the scan was performed
scan_id : int, optional
Integer scan identifier
uid : str, optional
Globally unique id string provided to metadatastore
_id : str or ObjectId, optional
The unique id generated by mongo
Returns
-------
content : iterable of list of json documents
We need lists to be able to JSON encode multiple dicts. We can return an iterator of
iterator?
Note
----
All documents that the RunStart Document points to are dereferenced.
These include RunStop, BeamlineConfig, and Sample.
Examples
--------
>>> find_run_starts(scan_id=123)
>>> find_run_starts(owner='arkilic')
>>> find_run_starts(start_time=1421176750.514707, stop_time=time.time()})
>>> find_run_starts(start_time=1421176750.514707, stop_time=time.time())
>>> find_run_starts(owner='arkilic', start_time=1421176750.514707,
... stop_time=time.time())
"""
_format_time(kwargs)
query = kwargs
increment = range_ceil - range_floor + 1
has_more = True
while has_more:
q_range = range_ceil - range_floor
query['range_floor'] = range_floor
query['range_ceil'] = range_ceil
r = requests.get(_server_path + "/run_start",
params=ujson.dumps(query))
r.raise_for_status()
content = ujson.loads(r.text)
if not content:
StopIteration()
break
else:
for c in content:
yield utils.Document('RunStart', c)
if len(content) <= q_range:
has_more = False
break
else:
range_ceil += increment
range_floor += increment
def find_events(descriptor=None, range_floor=0, range_ceil=1000, **kwargs):
"""
Parameters
-----------
start_time : time-like, optional
time-like representation of the earliest time that an Event
was created. Valid options are:
- timestamps --> time.time()
- '2015'
- '2015-01'
- '2015-01-30'
- '2015-03-30 03:00:00'
- datetime.datetime.now()
stop_time : time-like, optional
timestamp of the latest time that an Event was created. See
docs for `start_time` for examples.
descriptor : doc.Document or str, optional
Find events for a given EventDescriptor
uid : str, optional
Globally unique id string provided to metadatastore
Returns
-------
events : iterable of utils.Document objects
"""
if 'event_descriptor' in kwargs:
raise ValueError("Use 'descriptor', not 'event_descriptor'.")
query = kwargs
increment = range_ceil - range_floor + 1
if descriptor:
#TODO: Try to get from capped collection/cache instead of db hit
descriptor = descriptor_given_uid(descriptor)
query['descriptor_id'] = descriptor._id
_format_time(kwargs)
has_more = True
while has_more:
q_range = range_ceil - range_floor
query['range_floor'] = range_floor
query['range_ceil'] = range_ceil
r = requests.get(_server_path + "/run_stop", params=ujson.dumps(query))
r.raise_for_status()
content = ujson.loads(r.text)
if not content:
StopIteration()
break
else:
for c in content:
desc_id = c.pop('descriptor_id')
if descriptor:
c['descriptor'] = descriptor
else:
# if descriptor not provided, find and replace
# primarily for event based search to succeed
# should not be used
descriptor = next(find_descriptors(_id=desc_id))
yield utils.Document('Event', c)
if len(content) <= q_range:
has_more = False
break
else:
range_ceil += increment
range_floor += increment
def find_descriptors(run_start=None, range_floor=0, range_ceil=50, **kwargs):
"""Given search criteria, locate EventDescriptor Documents.
Parameters
----------
run_start : metadatastore.document.Document or uid, optional
The metadatastore run start document or the metadatastore uid to get
the corresponding run end for
run_start_uid : str
Globally unique id string provided to metadatastore for the
RunStart Document.
start_time : time-like, optional
time-like representation of the earliest time that an EventDescriptor
was created. Valid options are:
- timestamps --> time.time()
- '2015'
- '2015-01'
- '2015-01-30'
- '2015-03-30 03:00:00'
- datetime.datetime.now()
stop_time : time-like, optional
timestamp of the latest time that an EventDescriptor was created. See
docs for `start_time` for examples.
uid : str, optional
Globally unique id string provided to metadatastore
_id : str or ObjectId, optional
The unique id generated by mongo
Returns
-------
content : iterable of list of json documents
We need lists to be able to JSON encode multiple dicts. We can return an iterator of
iterator?
"""
_format_time(kwargs)
query = kwargs
increment = range_ceil - range_floor + 1
has_more = True
try:
query['run_start'] = run_start.uid
except AttributeError:
query['run_start'] = str(run_start)
while has_more:
q_range = range_ceil - range_floor
query['range_floor'] = range_floor
query['range_ceil'] = range_ceil
r = requests.get(_server_path + "/event_descriptor",
params=ujson.dumps(query))
r.raise_for_status()
content = ujson.loads(r.text)
if not content:
StopIteration()
break
else:
for c in content:
yield utils.Document('EventDescriptor', c)
if len(content) <= q_range:
has_more = False
break
else:
range_ceil += increment
range_floor += increment
def find_run_starts(**kwargs):
"""Given search criteria, locate RunStart Documents.
As we describe in design document, time here is strictly the time
server entry was created, not IOC timestamp. For the difference, refer
to: nsls-ii.github.io
Parameters
----------
start_time : time-like, optional
time-like representation of the earliest time that a RunStart
was created. Valid options are:
- timestamps --> time.time()
- '2015'
- '2015-01'
- '2015-01-30'
- '2015-03-30 03:00:00'
- datetime.datetime.now()
stop_time : time-like, optional
timestamp of the latest time that a RunStart was created. See
docs for `start_time` for examples.
beamline_id : str, optional
String identifier for a specific beamline
project : str, optional
Project name
owner : str, optional
The username of the logged-in user when the scan was performed
scan_id : int, optional
Integer scan identifier
uid : str, optional
Globally unique id string provided to metadatastore
Yields utils.Document
-------
RunStart documents if query returned something
Note
----
All documents that the RunStart Document points to are dereferenced.
These include RunStop, BeamlineConfig, and Sample.
Examples
--------
>>> find_run_starts(scan_id=123)
>>> find_run_starts(owner='arkilic')
>>> find_run_starts(start_time=1421176750.514707, stop_time=time.time()})
>>> find_run_starts(start_time=1421176750.514707, stop_time=time.time())
>>> find_run_starts(owner='arkilic', start_time=1421176750.514707,
... stop_time=time.time())
"""
_format_time(kwargs)
query = kwargs
r = requests.get(_server_path + "/run_start", params=ujson.dumps(query))
# r.raise_for_status()
# TODO: Done this way to replicate library behavior, we need to change this to raise_status?
if r.status_code != 200:
return None
content = ujson.loads(r.text)
if not content:
return None
else:
for c in content:
yield utils.Document('RunStart', c)
