def stop_by_start(run_start): """Given a RunStart return it's RunStop Raises if no RunStop exists. Parameters ---------- run_start : utils.Document or dict or str The RunStart to get the RunStop for. Can be either a Document/dict with a 'uid' key or a uid string. Returns ------- run_stop : utils.Document The RunStop document Raises ------ NoRunStop If no RunStop document exists for the given RunStart. """ rstart = doc_or_uid_to_uid(run_start) try: run_stop = next(find_run_stops(run_start=rstart)) return utils.Document('RunStop', run_stop) except StopIteration: raise NoRunStop("No run stop exists for {!r}".format(run_start))
def descriptors_by_start(run_start): """Given a RunStart return a list of it's descriptors Raises if no EventDescriptors exist. Parameters ---------- run_start : utils.Document or dict or str The RunStart to get the EventDescriptors for. Can be either a Document/dict with a 'uid' key or a uid string Returns ------- event_descriptors : list A list of EventDescriptor documents Raises ------ NoEventDescriptors If no EventDescriptor documents exist for the given RunStart """ run_start_id = doc_or_uid_to_uid(run_start) res = find_descriptors(run_start=run_start_id) e_descs = [utils.Document('EventDescriptor', r) for r in res] if not e_descs: raise NoEventDescriptors("No EventDescriptors exists " "for {!r}".format(run_start)) return e_descs
def stop_by_start(run_start): """Given a RunStart return it's RunStop Raises if no RunStop exists. Parameters ---------- run_start : doc.Document or dict or str The RunStart to get the RunStop for. Can be either a Document/dict with a 'uid' key or a uid string Returns ------- run_stop : utils.Document The RunStop document Raises ------ NoRunStop If no RunStop document exists for the given RunStart """ # run_start_uid = doc_or_uid_to_uid(run_start) # rstart = run_start_given_uid(run_start.uid) # if rstart: run_stop = next(find_run_stops(run_start=run_start.uid)) # run_stop['run_start'] = run_start return utils.Document('RunStop', run_stop)
def insert_run_start(time, scan_id, config, beamline_id, beamline_config={}, uid=None, owner=None, group=None, project=None, custom=None): """Provide a head for a sequence of events. Entry point for an experiment's run. Parameters ---------- time : float The date/time as found at the client side when an event is created. scan_id : int Unique scan identifier visible to the user and data analysis beamline_id : str Beamline String identifier. Not unique, just an indicator of beamline code for multiple beamline systems beamline_config : metadatastore.documents.Document or str if Document: The metadatastore beamline config document if str: uid of beamline config corresponding to a given run uid : str, optional Globally unique id string provided to metadatastore owner : str, optional A username associated with the entry group : str, optional A group (e.g., UNIX group) associated with the entry project : str, optional Any project name to help users locate the data custom: dict, optional Any additional information that data acquisition code/user wants to append to the Header at the start of the run. Returns ---------- utils.Document The run_start document that is successfully saved in mongo """ data = locals() if custom: data.update(custom) payload = ujson.dumps(data) r = requests.post(_server_path + '/run_start', data=payload) if r.status_code != 200: raise Exception("Server cannot complete the request", r) else: return utils.Document('RunStart', r.json())
def find_descriptors(run_start=None, **kwargs): """Given search criteria, locate EventDescriptor Documents. Parameters ---------- run_start : metadatastore.document.Document or uid, optional The metadatastore run start document or the metadatastore uid to get the corresponding run end for run_start_uid : str Globally unique id string provided to metadatastore for the RunStart Document. start_time : time-like, optional time-like representation of the earliest time that an EventDescriptor was created. Valid options are: - timestamps --> time.time() - '2015' - '2015-01' - '2015-01-30' - '2015-03-30 03:00:00' - datetime.datetime.now() stop_time : time-like, optional timestamp of the latest time that an EventDescriptor was created. See docs for `start_time` for examples. uid : str, optional Globally unique id string provided to metadatastore Returns ------- content : iterable of list of json documents We need lists to be able to JSON encode multiple dicts. We can return an iterator of iterator? """ _format_time(kwargs) query = kwargs rstart = None if run_start: query['run_start'] = doc_or_uid_to_uid(run_start) try: rstart = next(find_run_starts(uid=query['run_start'])) except StopIteration: raise NoRunStart() r = requests.get(_server_path + "/event_descriptor", params=ujson.dumps(query)) if r.status_code != 200: return None else: content = ujson.loads(r.text) if not content: return None else: for c in content: if rstart is None: rstart = next(find_run_starts(uid=c['run_start'])) c['run_start'] = rstart yield utils.Document('EventDescriptor', c)
def find_last(num=1): """Return the last 'num' many headers""" query = dict(num=num) r = requests.get(_server_path + "/run_start", params=ujson.dumps(query)) if r.status_code != 200: return None content = ujson.loads(r.text) if not content: return None else: for c in content: yield utils.Document('RunStart', c)
def find_events(descriptor=None, **kwargs): """ Parameters ----------- start_time : time-like, optional time-like representation of the earliest time that an Event was created. Valid options are: - timestamps --> time.time() - '2015' - '2015-01' - '2015-01-30' - '2015-03-30 03:00:00' - datetime.datetime.now() stop_time : time-like, optional timestamp of the latest time that an Event was created. See docs for `start_time` for examples. descriptor : doc.Document or str, optional Find events for a given EventDescriptor uid : str, optional Globally unique id string provided to metadatastore Yields ------- events : iterable of utils.Document objects """ # TODO: Add more tests!!! Make sure descriptor is returned for each event if 'event_descriptor' in kwargs: raise ValueError("Use 'descriptor', not 'event_descriptor'.") query = kwargs desc = None if descriptor: desc_uid = doc_or_uid_to_uid(descriptor) query['descriptor'] = desc_uid _format_time(query) r = requests.get(_server_path + "/event", params=ujson.dumps(query), stream=True) r.raise_for_status() content = ujson.loads(r.text) if not content: return None else: if descriptor: desc = next(find_descriptors(uid=doc_or_uid_to_uid(descriptor))) for c in content: if not desc: # Obvious bottleneck!!! # Fix using local caching!!!! desc = next(find_descriptors(uid=c['descriptor'])) c['descriptor'] = desc yield utils.Document('Event', c)
def run_start_given_uid(uid): """Given a uid, return the RunStart document Parameters ---------- uid : str The uid Returns ------- run_start : utils.Document The RunStart document. """ return utils.Document('RunStart', next(find_run_starts(uid=uid)))
def descriptor_given_uid(uid): """Given a uid, return the EventDescriptor document Parameters ---------- uid : str The uid Returns ------- EventDescriptor : utils.Document The EventDescriptor document fully de-referenced """ return utils.Document('EventDescriptor', next(find_descriptors(uid=uid)))
def find_run_stops(run_start=None, **kwargs): """Given search criteria, query for RunStop Documents. Parameters ---------- run_start : utils.Document or str, optional The metadatastore run start document or the metadatastore uid to get the corresponding run end for start_time : time-like, optional time-like representation of the earliest time that a RunStop was created. Valid options are: - timestamps --> time.time() - '2015' - '2015-01' - '2015-01-30' - '2015-03-30 03:00:00' - datetime.datetime.now() stop_time : time-like, optional timestamp of the latest time that a RunStop was created. See docs for `start_time` for examples. exit_status : {'success', 'fail', 'abort'}, optional provides information regarding the run success. reason : str, optional Long-form description of why the run was terminated. uid : str, optional Globally unique id string provided to metadatastore Yields utils.Document ------ RunStop documents if found in the query """ _format_time(kwargs) query = kwargs rstart = None if run_start: query['run_start'] = doc_or_uid_to_uid(run_start) rstart = next(find_run_starts(uid=query['run_start'])) r = requests.get(_server_path + "/run_stop", params=ujson.dumps(query)) if r.status_code != 200: return None content = ujson.loads(r.text) if not content: return None else: for c in content: if rstart is None: rstart = next(find_run_starts(uid=c['run_start'])) c['run_start'] = rstart yield utils.Document('RunStop', c)
def insert_run_stop(run_start, time, uid=None, config={}, exit_status='success', reason=None, custom=None): """ Provide an end to a sequence of events. Exit point for an experiment's run. Parameters ---------- run_start : metadatastore.documents.Document or str if Document: The metadatastore RunStart document if str: uid of RunStart object to associate with this record time : float The date/time as found at the client side when an event is created. uid : str, optional Globally unique id string provided to metadatastore exit_status : {'success', 'abort', 'fail'}, optional indicating reason run stopped, 'success' by default reason : str, optional more detailed exit status (stack trace, user remark, etc.) custom : dict, optional Any additional information that data acquisition code/user wants to append to the Header at the end of the run. Returns ------- run_stop : mongoengine.Document Inserted mongoengine object """ params = locals() rs_obj = params.pop('run_start') try: params['run_start'] = rs_obj.uid except AttributeError: params['run_start'] = rs_obj payload = ujson.dumps(params) r = requests.post(_server_path + '/run_stop', data=payload) if r.status_code != 200: raise Exception("Server cannot complete the request", r) else: return utils.Document('RunStop', r.json())
def runstop_given_uid(uid): """Given a uid, return the RunStop document. Parameters ---------- uid : str The uid Returns ------- run_stop : utils.Document The RunStop document fully de-referenced """ run_stop = dict(next(find_run_stops(uid=uid))) start_oid = run_stop.pop('run_start_id') run_stop['run_start'] = _run_start_given_oid(start_oid) return utils.Document('RunStop', run_stop)
def descriptor_given_uid(uid): """Given a uid, return the EventDescriptor document Parameters ---------- uid : str The uid Returns ------- descriptor : utils.Document The EventDescriptor document fully de-referenced """ descriptor = dict(next(find_descriptors(uid=uid))) start_oid = descriptor.pop('run_start_id') descriptor['run_start'] = _run_start_given_oid(start_oid) return utils.Document('EventDescriptor', descriptor)
def get_events_generator(descriptor): """A generator which yields all events from the event stream Parameters ---------- descriptor : doc.Document or dict or str The EventDescriptor to get the Events for. Can be either a Document/dict with a 'uid' key or a uid string Yields ------ event : utils.Document All events for the given EventDescriptor from oldest to newest """ descriptor_uid = doc_or_uid_to_uid(descriptor) ev_cur = find_events(descriptor=descriptor_uid) desc = next(find_descriptors(uid=descriptor_uid)) for ev in ev_cur: ev = dict(ev) ev['descriptor'] = desc yield utils.Document('Event', ev)
def fetch_events_generator(descriptor): """A generator which yields all events from the event stream Parameters ---------- descriptor : doc.Document or dict or str The EventDescriptor to get the Events for. Can be either a Document/dict with a 'uid' key or a uid string Yields ------ event : utils.Document All events for the given EventDescriptor from oldest to newest """ desc = descriptor_given_uid(descriptor.uid) raw_ev_gen = find_events(descriptor_id=str(desc['_id'])) for ev in raw_ev_gen: ev['descriptor'] = dict(desc) yield utils.Document('Event', ev)
def _run_start_given_oid(oid): """Get RunStart document given an ObjectId This is an internal function as ObjectIds should not be leaking out to user code. When we migrate to using uid as the primary key this function will be removed. Parameters ---------- oid : ObjectId Mongo's unique identifier for the document. This is currently used to implement foreign keys Returns ------- run_start : utils.Document The RunStart document. """ return utils.Document('RunStart', next(find_run_starts(_id=oid)))
def insert_descriptor(run_start, data_keys, time, uid=None, custom=None): """ Create an event_descriptor in metadatastore server backend Parameters ---------- run_start : metadatastore.documents.Document or str if Document: The metadatastore RunStart document if str: uid of RunStart object to associate with this record data_keys : dict Provides information about keys of the data dictionary in an event will contain time : float The date/time as found at the client side when an event descriptor is created. uid : str, optional Globally unique id string provided to metadatastore custom : dict, optional Any additional information that data acquisition code/user wants to append to the EventDescriptor. Returns ------- ev_desc : EventDescriptor The document added to the collection. """ payload = locals() tmp = payload['run_start'].uid payload.pop('run_start') payload['run_start'] = tmp r = requests.post(_server_path + '/event_descriptor', data=ujson.dumps(payload)) if r.status_code != 200: raise Exception("Server cannot complete the request", r) else: return utils.Document('EventDescriptor', dict(r.json()))
def _descriptor_given_oid(oid): """Get EventDescriptor document given an ObjectId This is an internal function as ObjectIds should not be leaking out to user code. When we migrate to using uid as the primary key this function will be removed. Parameters ---------- oid : ObjectId Mongo's unique identifier for the document. This is currently used to implement foreign keys Returns ------- descriptor : doc.Document The EventDescriptor document. """ #runstart replaced in server side descriptor = dict(next(find_descriptors(oid=oid))) return utils.Document('EventDescriptor', descriptor)
def monitor_run_stop(): r = requests.get(_server_path + '/run_stop_capped') content = ujson.loads(r.text) yield utils.Document('RunStop', content)
def monitor_run_start(): raise NotImplementedError( 'Not this cycle. Code works, needs capped collection support') r = requests.get(_server_path + '/run_start_capped') content = ujson.loads(r.text) yield utils.Document('RunStart', content)
def find_run_starts(range_floor=0, range_ceil=50, **kwargs): """Given search criteria, locate RunStart Documents. As we describe in design document, time here is strictly the time server entry was created, not IOC timestamp. For the difference, refer to: nsls-ii.github.io Parameters ---------- start_time : time-like, optional time-like representation of the earliest time that a RunStart was created. Valid options are: - timestamps --> time.time() - '2015' - '2015-01' - '2015-01-30' - '2015-03-30 03:00:00' - datetime.datetime.now() stop_time : time-like, optional timestamp of the latest time that a RunStart was created. See docs for `start_time` for examples. `beamline_id : str, optional String identifier for a specific beamline project : str, optional Project name owner : str, optional The username of the logged-in user when the scan was performed scan_id : int, optional Integer scan identifier uid : str, optional Globally unique id string provided to metadatastore _id : str or ObjectId, optional The unique id generated by mongo Returns ------- content : iterable of list of json documents We need lists to be able to JSON encode multiple dicts. We can return an iterator of iterator? Note ---- All documents that the RunStart Document points to are dereferenced. These include RunStop, BeamlineConfig, and Sample. Examples -------- >>> find_run_starts(scan_id=123) >>> find_run_starts(owner='arkilic') >>> find_run_starts(start_time=1421176750.514707, stop_time=time.time()}) >>> find_run_starts(start_time=1421176750.514707, stop_time=time.time()) >>> find_run_starts(owner='arkilic', start_time=1421176750.514707, ... stop_time=time.time()) """ _format_time(kwargs) query = kwargs increment = range_ceil - range_floor + 1 has_more = True while has_more: q_range = range_ceil - range_floor query['range_floor'] = range_floor query['range_ceil'] = range_ceil r = requests.get(_server_path + "/run_start", params=ujson.dumps(query)) r.raise_for_status() content = ujson.loads(r.text) if not content: StopIteration() break else: for c in content: yield utils.Document('RunStart', c) if len(content) <= q_range: has_more = False break else: range_ceil += increment range_floor += increment
def find_events(descriptor=None, range_floor=0, range_ceil=1000, **kwargs): """ Parameters ----------- start_time : time-like, optional time-like representation of the earliest time that an Event was created. Valid options are: - timestamps --> time.time() - '2015' - '2015-01' - '2015-01-30' - '2015-03-30 03:00:00' - datetime.datetime.now() stop_time : time-like, optional timestamp of the latest time that an Event was created. See docs for `start_time` for examples. descriptor : doc.Document or str, optional Find events for a given EventDescriptor uid : str, optional Globally unique id string provided to metadatastore Returns ------- events : iterable of utils.Document objects """ if 'event_descriptor' in kwargs: raise ValueError("Use 'descriptor', not 'event_descriptor'.") query = kwargs increment = range_ceil - range_floor + 1 if descriptor: #TODO: Try to get from capped collection/cache instead of db hit descriptor = descriptor_given_uid(descriptor) query['descriptor_id'] = descriptor._id _format_time(kwargs) has_more = True while has_more: q_range = range_ceil - range_floor query['range_floor'] = range_floor query['range_ceil'] = range_ceil r = requests.get(_server_path + "/run_stop", params=ujson.dumps(query)) r.raise_for_status() content = ujson.loads(r.text) if not content: StopIteration() break else: for c in content: desc_id = c.pop('descriptor_id') if descriptor: c['descriptor'] = descriptor else: # if descriptor not provided, find and replace # primarily for event based search to succeed # should not be used descriptor = next(find_descriptors(_id=desc_id)) yield utils.Document('Event', c) if len(content) <= q_range: has_more = False break else: range_ceil += increment range_floor += increment
def find_descriptors(run_start=None, range_floor=0, range_ceil=50, **kwargs): """Given search criteria, locate EventDescriptor Documents. Parameters ---------- run_start : metadatastore.document.Document or uid, optional The metadatastore run start document or the metadatastore uid to get the corresponding run end for run_start_uid : str Globally unique id string provided to metadatastore for the RunStart Document. start_time : time-like, optional time-like representation of the earliest time that an EventDescriptor was created. Valid options are: - timestamps --> time.time() - '2015' - '2015-01' - '2015-01-30' - '2015-03-30 03:00:00' - datetime.datetime.now() stop_time : time-like, optional timestamp of the latest time that an EventDescriptor was created. See docs for `start_time` for examples. uid : str, optional Globally unique id string provided to metadatastore _id : str or ObjectId, optional The unique id generated by mongo Returns ------- content : iterable of list of json documents We need lists to be able to JSON encode multiple dicts. We can return an iterator of iterator? """ _format_time(kwargs) query = kwargs increment = range_ceil - range_floor + 1 has_more = True try: query['run_start'] = run_start.uid except AttributeError: query['run_start'] = str(run_start) while has_more: q_range = range_ceil - range_floor query['range_floor'] = range_floor query['range_ceil'] = range_ceil r = requests.get(_server_path + "/event_descriptor", params=ujson.dumps(query)) r.raise_for_status() content = ujson.loads(r.text) if not content: StopIteration() break else: for c in content: yield utils.Document('EventDescriptor', c) if len(content) <= q_range: has_more = False break else: range_ceil += increment range_floor += increment
def find_run_starts(**kwargs): """Given search criteria, locate RunStart Documents. As we describe in design document, time here is strictly the time server entry was created, not IOC timestamp. For the difference, refer to: nsls-ii.github.io Parameters ---------- start_time : time-like, optional time-like representation of the earliest time that a RunStart was created. Valid options are: - timestamps --> time.time() - '2015' - '2015-01' - '2015-01-30' - '2015-03-30 03:00:00' - datetime.datetime.now() stop_time : time-like, optional timestamp of the latest time that a RunStart was created. See docs for `start_time` for examples. beamline_id : str, optional String identifier for a specific beamline project : str, optional Project name owner : str, optional The username of the logged-in user when the scan was performed scan_id : int, optional Integer scan identifier uid : str, optional Globally unique id string provided to metadatastore Yields utils.Document ------- RunStart documents if query returned something Note ---- All documents that the RunStart Document points to are dereferenced. These include RunStop, BeamlineConfig, and Sample. Examples -------- >>> find_run_starts(scan_id=123) >>> find_run_starts(owner='arkilic') >>> find_run_starts(start_time=1421176750.514707, stop_time=time.time()}) >>> find_run_starts(start_time=1421176750.514707, stop_time=time.time()) >>> find_run_starts(owner='arkilic', start_time=1421176750.514707, ... stop_time=time.time()) """ _format_time(kwargs) query = kwargs r = requests.get(_server_path + "/run_start", params=ujson.dumps(query)) # r.raise_for_status() # TODO: Done this way to replicate library behavior, we need to change this to raise_status? if r.status_code != 200: return None content = ujson.loads(r.text) if not content: return None else: for c in content: yield utils.Document('RunStart', c)