def run(run_start=None, sleep=0): if sleep != 0: raise NotImplementedError("A sleep time is not implemented for this " "example.") # Make the data ramp = common.stepped_ramp(start, stop, step, points_per_step) deadbanded_ramp = common.apply_deadband(ramp, deadband_size) rs = np.random.RandomState(5) point_det_data = rs.randn(num_exposures) # Create Event Descriptors data_keys1 = {'point_det': dict(source='PV:ES:PointDet', dtype='number')} data_keys2 = {'Tsam': dict(source='PV:ES:Tsam', dtype='number'), 'Troom': dict(source='PV:ES:Troom', dtype='number')} ev_desc1_uid = insert_event_descriptor(run_start=run_start, data_keys=data_keys1, time=common.get_time(), uid=str(uuid.uuid4())) ev_desc2_uid = insert_event_descriptor(run_start=run_start, data_keys=data_keys2, time=common.get_time(), uid=str(uuid.uuid4())) print('event descriptor 1 uid = %s' % ev_desc1_uid) print('event descriptor 2 uid = %s' % ev_desc2_uid) # Create Events. events = [] # Point Detector Events base_time = common.get_time() for i in range(num_exposures): time = float(i + 0.5 * rs.randn()) + base_time data = {'point_det': (point_det_data[i], time)} data = {'point_det': point_det_data[i]} timestamps = {'point_det': time} event_uid = insert_event(descriptor=ev_desc1_uid, seq_num=i, time=time, data=data, uid=str(uuid.uuid4()), timestamps=timestamps) event, = find_events(uid=event_uid) events.append(event) # Temperature Events for i, (time, temp) in enumerate(zip(*deadbanded_ramp)): time = float(time) + base_time data = {'Tsam': temp, 'Troom': temp + 10} timestamps = {'Tsam': time, 'Troom': time} event_uid = insert_event(descriptor=ev_desc2_uid, time=time, data=data, seq_num=i, uid=str(uuid.uuid4()), timestamps=timestamps) event, = find_events(uid=event_uid) events.append(event) return events
def run(run_start=None, sleep=0): if sleep != 0: raise NotImplementedError("A sleep time is not implemented for this " "example.") # Make the data ramp = common.stepped_ramp(start, stop, step, points_per_step) deadbanded_ramp = common.apply_deadband(ramp, deadband_size) rs = np.random.RandomState(5) point_det_data = rs.randn(num_exposures) # Create Event Descriptors data_keys1 = {'point_det': dict(source='PV:ES:PointDet', dtype='number')} data_keys2 = { 'Tsam': dict(source='PV:ES:Tsam', dtype='number'), 'Troom': dict(source='PV:ES:Troom', dtype='number') } ev_desc1 = insert_event_descriptor(run_start=run_start, data_keys=data_keys1, time=0., uid=str(uuid.uuid4())) ev_desc2 = insert_event_descriptor(run_start=run_start, data_keys=data_keys2, time=0., uid=str(uuid.uuid4())) # Create Events. events = [] # Point Detector Events for i in range(num_exposures): time = float(i + 0.01 * rs.randn()) data = {'point_det': (point_det_data[i], time)} event = insert_event(event_descriptor=ev_desc1, seq_num=i, time=time, data=data, uid=str(uuid.uuid4())) events.append(event) # Temperature Events for i, (time, temp) in enumerate(zip(*deadbanded_ramp)): time = float(time) data = {'Tsam': (temp, time), 'Troom': (temp + 10, time)} event = insert_event(event_descriptor=ev_desc2, time=time, data=data, seq_num=i, uid=str(uuid.uuid4())) events.append(event) return events
def run(run_start_uid=None, sleep=0): if sleep != 0: raise NotImplementedError("A sleep time is not implemented for this " "example.") # Make the data ramp = common.stepped_ramp(start, stop, step, points_per_step) deadbanded_ramp = common.apply_deadband(ramp, deadband_size) rs = np.random.RandomState(5) point_det_data = rs.randn(num_exposures) + np.arange(num_exposures) # Create Event Descriptors data_keys1 = {'point_det': dict(source='PV:ES:PointDet', dtype='number')} data_keys2 = {'Tsam': dict(source='PV:ES:Tsam', dtype='number')} ev_desc1_uid = insert_event_descriptor(run_start=run_start_uid, data_keys=data_keys1, time=common.get_time()) ev_desc2_uid = insert_event_descriptor(run_start=run_start_uid, data_keys=data_keys2, time=common.get_time()) # Create Events. events = [] # Point Detector Events base_time = common.get_time() for i in range(num_exposures): time = float(2 * i + 0.5 * rs.randn()) + base_time data = {'point_det': point_det_data[i]} timestamps = {'point_det': time} event_dict = dict(descriptor=ev_desc1_uid, seq_num=i, time=time, data=data, timestamps=timestamps) event_uid = insert_event(**event_dict) # grab the actual event from metadatastore event, = find_events(uid=event_uid) events.append(event) # Temperature Events for i, (time, temp) in enumerate(zip(*deadbanded_ramp)): time = float(time) + base_time data = {'Tsam': temp} timestamps = {'Tsam': time} event_dict = dict(descriptor=ev_desc2_uid, time=time, data=data, timestamps=timestamps, seq_num=i) event_uid = insert_event(**event_dict) event, = find_events(uid=event_uid) events.append(event) #todo insert run stop if run_start_uid is not None return events
def run(run_start=None, sleep=0): if sleep != 0: raise NotImplementedError("A sleep time is not implemented for this " "example.") # Make the data ramp = common.stepped_ramp(start, stop, step, points_per_step) deadbanded_ramp = common.apply_deadband(ramp, deadband_size) # Create Event Descriptors data_keys = {'Tsam': dict(source='PV:ES:Tsam', dtype='number'), 'point_det': dict(source='PV:ES:point_det', dtype='number')} ev_desc = insert_event_descriptor(run_start=run_start, data_keys=data_keys, time=0.) # Create Events. events = [] # Temperature Events for i, (time, temp) in enumerate(zip(*deadbanded_ramp)): time = float(time) point_det = np.random.randn() data = {'Tsam': temp, 'point_det': point_det} timestamps = {'Tsam': time, 'point_det': time} event_uid = insert_event(descriptor=ev_desc, time=time, data=data, seq_num=i, timestamps=timestamps) event, = find_events(uid=event_uid) events.append(event) return events
def test_data_key(): rs1_uid = insert_run_start(time=100., scan_id=1, owner='nedbrainard', beamline_id='example', beamline_config=insert_beamline_config( {}, time=0.)) rs2_uid = insert_run_start(time=200., scan_id=2, owner='nedbrainard', beamline_id='example', beamline_config=insert_beamline_config( {}, time=0.)) rs1, = find_run_starts(uid=rs1_uid) rs2, = find_run_starts(uid=rs2_uid) data_keys = {'fork': {'source': '_', 'dtype': 'number'}, 'spoon': {'source': '_', 'dtype': 'number'}} evd1_uid = insert_event_descriptor(run_start=rs1_uid, data_keys=data_keys, time=100.) insert_event_descriptor(run_start=rs2_uid, data_keys=data_keys, time=200.) result1 = db.find_headers(data_key='fork') result2 = db.find_headers(data_key='fork', start_time=150) assert_equal(len(result1), 2) assert_equal(len(result2), 1) actual = result2[0].run_start_uid assert_equal(actual, str(rs2.uid))
def hdf_data_io(): """ Save data to db and run test when data is retrieved. """ blc = insert_beamline_config({'cfg1': 1}, 0.0) run_start_uid = insert_run_start(time=0., scan_id=1, beamline_id='csx', uid=str(uuid.uuid4()), beamline_config=blc) # data keys entry data_keys = {'x_pos': dict(source='MCA:pos_x', dtype='number'), 'y_pos': dict(source='MCA:pos_y', dtype='number'), 'xrf_spectrum': dict(source='MCA:spectrum', dtype='array', #shape=(5,), external='FILESTORE:')} # save the event descriptor descriptor_uid = insert_event_descriptor( run_start=run_start_uid, data_keys=data_keys, time=0., uid=str(uuid.uuid4())) # number of positions to record, basically along a horizontal line num = 5 events = [] for i in range(num): v_pos = 0 h_pos = i spectrum_uid = get_data(v_pos, h_pos) # Put in actual ndarray data, as broker would do. data1 = {'xrf_spectrum': spectrum_uid, 'v_pos': v_pos, 'h_pos': h_pos} timestamps1 = {k: noisy(i) for k in data1} event_uid = insert_event(descriptor=descriptor_uid, seq_num=i, time=noisy(i), data=data1, uid=str(uuid.uuid4()), timestamps=timestamps1) event, = find_events(uid=event_uid) # test on retrieve data for all data sets events.append(event) return events
def setup(self, n): self.obj = range(n) self.bcfg = mdsc.insert_beamline_config(time=1315315135.5135, config_params={'param1': 1}) self.data_keys = {'linear_motor': {'source': 'PV:pv1', 'shape': None, 'dtype': 'number'}, 'scalar_detector': {'source': 'PV:pv2', 'shape': None, 'dtype': 'number'}, 'Tsam': {'source': 'PV:pv3', 'dtype': 'number', 'shape': None}} self.custom = {'custom_key': 'value'} self.scan_id = 1903 self.run_start = mdsc.insert_run_start(scan_id=int(self.scan_id), owner='benchmark_script', beamline_id='benchmark_b', time=1315315135.5135, beamline_config=self.bcfg, custom=self.custom) self.e_desc = mdsc.insert_event_descriptor(data_keys=self.data_keys, time=1315315135.5135, run_start=self.run_start) # Compose event data list for 1mil events in setup. # See params in event insert test to see how many of these are used func = np.cos num = EVENT_COUNT start = 0 stop = 10 sleep_time = .1 self.data = list() for idx, i in enumerate(np.linspace(start, stop, num)): self.data.append({'linear_motor': [i, 1315315135.5135], 'Tsam': [i + 5, 1315315135.5135], 'scalar_detector': [func(i) + np.random.randn() / 100, 1315315135.5135]})
def run(run_start=None, sleep=0): # Make the data rs = np.random.RandomState(5) # set up the data keys entry data_keys1 = {'linear_motor': dict(source='PV:ES:sam_x', dtype='number'), 'img': dict(source='CCD', shape=(5, 5), dtype='array', external='FILESTORE:'), 'total_img_sum': dict(source='CCD:sum', dtype='number'), 'img_x_max': dict(source='CCD:xmax', dtype='number'), 'img_y_max': dict(source='CCD:ymax', dtype='number'), 'img_sum_x': dict(source='CCD:xsum', dtype='array', shape=(5,), external='FILESTORE:'), 'img_sum_y': dict(source='CCD:ysum', dtype='array', shape=(5,), external='FILESTORE:') } data_keys2 = {'Tsam': dict(source='PV:ES:Tsam', dtype='number')} # save the first event descriptor e_desc1 = insert_event_descriptor( run_start=run_start, data_keys=data_keys1, time=0., uid=str(uuid.uuid4())) e_desc2 = insert_event_descriptor( run_start=run_start, data_keys=data_keys2, time=0., uid=str(uuid.uuid4())) # number of motor positions to fake num1 = 20 # number of temperatures to record per motor position num2 = 10 events = [] for idx1, i in enumerate(range(num1)): img = next(frame_generator) img_sum = float(img.sum()) img_sum_x = img.sum(axis=0) img_sum_y = img.sum(axis=1) img_x_max = float(img_sum_x.argmax()) img_y_max = float(img_sum_y.argmax()) fsid_img = save_ndarray(img) fsid_x = save_ndarray(img_sum_x) fsid_y = save_ndarray(img_sum_y) # Put in actual ndarray data, as broker would do. data1 = {'linear_motor': (i, noisy(i)), 'total_img_sum': (img_sum, noisy(i)), 'img': (fsid_img, noisy(i)), 'img_sum_x': (fsid_x, noisy(i)), 'img_sum_y': (fsid_y, noisy(i)), 'img_x_max': (img_x_max, noisy(i)), 'img_y_max': (img_y_max, noisy(i)) } event = insert_event(event_descriptor=e_desc1, seq_num=idx1, time=noisy(i), data=data1, uid=str(uuid.uuid4())) fill_event(event) events.append(event) for idx2, i2 in enumerate(range(num2)): time = noisy(i/num2) data2 = {'Tsam': (idx1 + np.random.randn()/100, time)} event = insert_event(event_descriptor=e_desc2, seq_num=idx2+idx1, time=time, data=data2, uid=str(uuid.uuid4())) events.append(event) ttime.sleep(sleep) return events
begin_runs = db.begin_run_event.find() for br in begin_runs: the_run_start = insert_run_start( time=br["time"], beamline_id=br["beamline_id"], beamline_config=the_bc, owner=br["owner"], scan_id=br["scan_id"], custom=br.get("custom", {}), uid=br["uid"], ) event_descs = db.event_descriptor.find({"begin_run_id": br["_id"]}) max_time = 0.0 for e_desc in event_descs: the_e_desc = insert_event_descriptor( run_start=the_run_start, data_keys=e_desc["data_keys"], time=e_desc["time"], uid=e_desc["uid"] ) events = db.event.find({"descriptor_id": e_desc["_id"]}) for ev in events: if ev["time"] > max_time: max_time = ev["time"] insert_event( event_descriptor=the_e_desc, time=ev["time"], data=ev["data"], seq_num=ev["seq_num"], uid=ev["uid"] ) insert_run_stop(run_start=the_run_start, time=float(max_time), exit_status="success", reason=None, uid=None) run_start_mapping = dict() run_starts = db.run_start.find() for rs in run_starts: time = rs.pop("time") beamline_id = rs.pop("beamline_id")
def _start_scan(self, run_start=None, detectors=None, data=None, positioners=None, **kwargs): dets = detectors triggers = [det for det in dets if isinstance(det, Detector)] # creation of the event descriptor should be delayed until the first # event comes in. Set it to None for now event_descriptor = None # provide header for formatted list of positioners and detectors in # INFO channel names = list() for pos in positioners + dets: names.extend(pos.describe().keys()) self.logger.info(self._demunge_names(names)) seq_num = 0 while self._scan_state is True: self.logger.debug( 'self._scan_state is True in self._start_scan') posvals = self._move_positioners(positioners=positioners, **kwargs) self.logger.debug('moved positioners') # if we're done iterating over positions, get outta Dodge if posvals is None: break # Trigger detector acquisision acq_status = [trig.acquire() for trig in triggers] while any([not stat.done for stat in acq_status]): time.sleep(0.05) time.sleep(0.05) # Read detector values tmp_detvals = {} for det in dets + positioners: tmp_detvals.update(det.read()) detvals = mds.format_events(tmp_detvals) # pass data onto Demuxer for distribution self.logger.info(self._demunge_values(detvals, names)) # grab the current time as a timestamp that describes when the # event data was bundled together bundle_time = time.time() # actually insert the event into metadataStore try: self.logger.debug( 'inserting event %d------------------', seq_num) event = mds.insert_event(event_descriptor=event_descriptor, time=bundle_time, data=detvals, seq_num=seq_num) except mds.EventDescriptorIsNoneError: # the time when the event descriptor was created self.logger.debug( 'event_descriptor has not been created. ' 'creating it now...') evdesc_creation_time = time.time() data_key_info = _get_info( positioners=positioners, detectors=dets, data=detvals) event_descriptor = mds.insert_event_descriptor( run_start=run_start, time=evdesc_creation_time, data_keys=mds.format_data_keys(data_key_info)) self.logger.debug( 'event_descriptor: %s', vars(event_descriptor)) # insert the event again. this time it better damn well work self.logger.debug( 'inserting event %d------------------', seq_num) event = mds.insert_event(event_descriptor=event_descriptor, time=bundle_time, data=detvals, seq_num=seq_num) self.logger.debug('event %d--------', seq_num) self.logger.debug('%s', vars(event)) seq_num += 1 # update the 'data' object from detvals dict for k, v in detvals.items(): data[k].append(v) if not positioners: break self._scan_state = False return
beamline_cfg_mapping[bc['_id']] = the_bc begin_runs = db.begin_run_event.find() for br in begin_runs: the_run_start = insert_run_start(time=br['time'], beamline_id=br['beamline_id'], beamline_config=the_bc, owner=br['owner'], scan_id=br['scan_id'], custom=br.get('custom', {}), uid=br['uid']) event_descs = db.event_descriptor.find({'begin_run_id': br['_id']}) max_time = 0.0 for e_desc in event_descs: the_e_desc = insert_event_descriptor(run_start=the_run_start, data_keys=e_desc['data_keys'], time=e_desc['time'], uid=e_desc['uid']) events = db.event.find({'descriptor_id': e_desc['_id']}) for ev in events: if ev['time'] > max_time: max_time = ev['time'] insert_event(event_descriptor=the_e_desc, time=ev['time'], data=ev['data'], seq_num=ev['seq_num'], uid=ev['uid']) insert_run_stop(run_start=the_run_start, time=float(max_time), exit_status='success', reason=None, uid=None)
def time_single_descriptor(self, n): for _ in self.obj: mdsc.insert_event_descriptor(data_keys=self.data_keys, time=1315315135.5135, run_start=self.run_start)
"Tsam": {"source": "PV:pv3", "dtype": "number", "shape": None}, } try: last_hdr = next(find_last()) scan_id = int(last_hdr.scan_id) + 1 except (IndexError, TypeError): scan_id = 1 custom = {"plotx": "linear_motor", "ploty": "scalar_detector"} # Create a BeginRunEvent that serves as entry point for a run rs = insert_run_start(scan_id=scan_id, beamline_id="csx", time=time.time(), beamline_config=b_config, custom=custom) # Create an EventDescriptor that indicates the data # keys and serves as header for set of Event(s) e_desc = insert_event_descriptor(data_keys=data_keys, time=time.time(), run_start=rs) func = np.cos num = 1000 start = 0 stop = 10 sleep_time = 0.1 for idx, i in enumerate(np.linspace(start, stop, num)): data = { "linear_motor": [i, time.time()], "Tsam": [i + 5, time.time()], "scalar_detector": [func(i) + np.random.randn() / 100, time.time()], } e = insert_event(event_descriptor=e_desc, seq_num=idx, time=time.time(), data=data) last_run = next(find_last()) try: if str(last_run.id) != str(rs.id):
def run(run_start_uid=None, sleep=0): frame_generator = frame_generators.brownian(img_size, step_scale=.5, I_fluc_function=I_func_gaus, step_fluc_function=scale_fluc) # Make the data rs = np.random.RandomState(5) # set up the data keys entry data_keys1 = {'linear_motor': dict(source='PV:ES:sam_x', dtype='number'), 'img': dict(source='CCD', shape=(5, 5), dtype='array', external='FILESTORE:'), 'total_img_sum': dict(source='CCD:sum', dtype='number'), 'img_x_max': dict(source='CCD:xmax', dtype='number'), 'img_y_max': dict(source='CCD:ymax', dtype='number'), 'img_sum_x': dict(source='CCD:xsum', dtype='array', shape=(5,), external='FILESTORE:'), 'img_sum_y': dict(source='CCD:ysum', dtype='array', shape=(5,), external='FILESTORE:') } data_keys2 = {'Tsam': dict(source='PV:ES:Tsam', dtype='number')} # save the first event descriptor descriptor1_uid = insert_event_descriptor( run_start=run_start_uid, data_keys=data_keys1, time=0., uid=str(uuid.uuid4())) descriptor2_uid = insert_event_descriptor( run_start=run_start_uid, data_keys=data_keys2, time=0., uid=str(uuid.uuid4())) events = [] for idx1, i in enumerate(range(num1)): img = next(frame_generator) img_sum = float(img.sum()) img_sum_x = img.sum(axis=0) img_sum_y = img.sum(axis=1) img_x_max = float(img_sum_x.argmax()) img_y_max = float(img_sum_y.argmax()) fsid_img = save_ndarray(img) fsid_x = save_ndarray(img_sum_x) fsid_y = save_ndarray(img_sum_y) # Put in actual ndarray data, as broker would do. data1 = {'linear_motor': i, 'total_img_sum': img_sum, 'img': fsid_img, 'img_sum_x': fsid_x, 'img_sum_y': fsid_y, 'img_x_max': img_x_max, 'img_y_max': img_y_max } timestamps1 = {k: noisy(i) for k in data1} event_uid = insert_event(descriptor=descriptor1_uid, seq_num=idx1, time=noisy(i), data=data1, timestamps=timestamps1, uid=str(uuid.uuid4())) event, = find_events(uid=event_uid) events.append(event) for idx2, i2 in enumerate(range(num2)): time = noisy(i/num2) data2 = {'Tsam': idx1 + np.random.randn()} timestamps2 = {'Tsam': time} event_uid = insert_event(descriptor=descriptor2_uid, seq_num=idx2+idx1, time=time, data=data2, uid=str(uuid.uuid4()), timestamps=timestamps2) event, = find_events(uid=event_uid) events.append(event) ttime.sleep(sleep) return events