def test_AD_round_trip(self): # Retrieve the data. for i, datum_id in enumerate(self.datum_ids): data = retrieve(datum_id) known_data = i * np.ones((1, 2, 2)) assert_array_equal(data, known_data)
def test_AD_round_trip(self): # Retrieve the data. for i, datum_id in zip(np.arange(len(self.datum_ids)), self.datum_ids): data = retrieve(datum_id) known_data = 2 * i assert_array_equal(data, known_data)
def __init__(self, headers, name): """ Load images from a detector for given Header(s). Parameters ---------- headers : Header or list of Headers name : str alias (data key) of a detector Example ------- >>> header = DataBroker[-1] >>> images = Images(header, 'my_detector_lightfield') >>> for image in images: # do something """ all_events = DataBroker.fetch_events(headers, fill=False) # TODO Make this smarter by only grabbing events from the relevant # descriptors. For now, we will do it this way to take advantage of # header validation logical happening in fetch_events. events = [event for event in all_events if name in event.data.keys()] self._datum_uids = [event.data[name] for event in events] self._len = len(self._datum_uids) example_frame = retrieve(self._datum_uids[0]) self._dtype = example_frame.dtype self._shape = example_frame.shape
def test_insert_funcs(func): shape = (25, 32) mod_ids = func('syn-mod', shape, 10) for j, r_id in enumerate(mod_ids): data = retrieve(r_id) known_data = np.mod(np.arange(np.prod(shape)), j + 1).reshape(shape) assert_array_equal(data, known_data)
def test_maps_spectrum_round_trip(self): sn = np.sin(self.th) with handler_context({'hdf5_maps': HDFM}): for eid, sc in zip(self.eids_spectrum, self.scale): print(eid) data = retrieve(eid) assert_array_equal(data, sc * sn)
def fill_event(event): """ Populate events with externally stored data. """ is_external = _inspect_descriptor(event.descriptor) for data_key, value in six.iteritems(event.data): if is_external[data_key]: # Retrieve a numpy array from filestore event.data[data_key] = retrieve(value)
def test_custom(): test_path = os.path.join(BASE_PATH, str(uuid.uuid4()) + '.npy') dd = np.random.rand(500, 500) with fs_write.NpyWriter(test_path, resource_kwargs={'mmap_mode': 'r'}) as f: eid = f.add_data(dd) with fsa.handler_context({'npy': fs_read.NpyHandler}): ret = fsa.retrieve(eid) assert_array_equal(dd, ret)
def fill_event(event): """ Populate events with externally stored data. """ is_external = _inspect_descriptor(event.descriptor) for data_key, (value, timestamp) in event.data.items(): if is_external[data_key]: # Retrieve a numpy array from filestore event.data[data_key][0] = retrieve(value)
def load_image_filestore(datum_id): if datum_id is None: raise IOError("Image doesn't exist yet") try: return np.asarray(fsapi.retrieve(datum_id)).squeeze() except Exception as ex: print('Filestore load failed (datum={}): ({}) {}' ''.format(datum_id, ex.__class__.__name__, ex)) raise
def event(self, event_document): event_time = datetime.fromtimestamp(event_document['time']).time() rounded_time = str(event_time)[:10] row = [event_document['seq_num'], rounded_time] for field in self.fields: val = event_document['data'].get(field, '') if field in self._filestore_keys: try: import filestore.api as fsapi val = fsapi.retrieve(val) except Exception as exc: warnings.warn( UserWarning, "Attempt to read {0} raised {1}" "".format(field, exc)) val = 'Not Available' if isinstance(val, np.ndarray) or isinstance(val, list): val = np.sum(np.asarray(val)) try: val = format_num(val, max_len=self.data_field_width, pre=self.max_pre_decimal, post=self.max_post_decimal) except Exception: val = str(val)[:self.data_field_width] row.append(val) self.table.add_row(row) if self.rowwise: # Print the last row of data only. # [-1] is the bottom border print(str(self.table).split('\n')[-2]) # only print header intermittently for rowwise table printing if self.num_events_since_last_header >= self.print_header_interval: self._print_table_header() self.num_events_since_last_header = 0 self.num_events_since_last_header += 1 else: # print the whole table print(self.table) sys.stdout.flush()
def event(self, event_document): event_time = datetime.fromtimestamp(event_document['time']).time() rounded_time = str(event_time)[:10] row = [event_document['seq_num'], rounded_time] for field in self.fields: val = event_document['data'].get(field, '') if field in self._filestore_keys: try: import filestore.api as fsapi val = fsapi.retrieve(val) except Exception as exc: warnings.warn(UserWarning, "Attempt to read {0} raised {1}" "".format(field, exc)) val = 'Not Available' if isinstance(val, np.ndarray) or isinstance(val, list): val = np.sum(np.asarray(val)) try: val = format_num(val, max_len=self.data_field_width, pre=self.max_pre_decimal, post=self.max_post_decimal) except Exception: val = str(val)[:self.data_field_width] row.append(val) self.table.add_row(row) if self.rowwise: # Print the last row of data only. # [-1] is the bottom border print(str(self.table).split('\n')[-2]) # only print header intermittently for rowwise table printing if self.num_events_since_last_header >= self.print_header_interval: self._print_table_header() self.num_events_since_last_header = 0 self.num_events_since_last_header += 1 else: # print the whole table print(self.table) sys.stdout.flush()
def __init__(self, headers, name): """ Load images from a detector for given Header(s). Parameters ---------- headers : Header or list of Headers name : str field name (data key) of a detector Example ------- >>> header = DataBroker[-1] >>> images = Images(header, 'my_detector_lightfield') >>> for image in images: # do something """ events = get_events(headers, [name], fill=False) self._datum_uids = [event.data[name] for event in events] self._len = len(self._datum_uids) example_frame = retrieve(self._datum_uids[0]) self._dtype = example_frame.dtype self._shape = example_frame.shape
def _test_retrieve_data(event): uid = event['data']['xrf_spectrum'] data = retrieve(uid) assert_equal(data.size, 20) assert_not_equal(np.sum(data), 0)
def test_maps_plane_round_trip(self): base = self.scale.reshape(self.N, self.M) with handler_context({'hdf5_planes': HDFE}): for eid, v in zip(self.eids_planes, np.sin(self.th)): data = retrieve(eid) assert_array_equal(data, base * v)
def test_non_exist(): with pytest.raises(DatumNotFound): retrieve('aardvark')
def _npsave_helper(dd, base_path): eid = fs_write.save_ndarray(dd, base_path) with fsa.handler_context({'npy': fs_read.NpyHandler}): ret = fsa.retrieve(eid) assert_array_equal(dd, ret)
def get_table(headers, fields=None, fill=True, convert_times=True): """ Make a table (pandas.DataFrame) from given run(s). Parameters ---------- headers : Header or iterable of Headers The headers to fetch the events for fields : list, optional whitelist of field names of interest; if None, all are returned fill : bool, optional Whether externally-stored data should be filled in. Defaults to True convert_times : bool, optional Whether to convert times from float (seconds since 1970) to numpy datetime64, using pandas. True by default. Returns ------- table : pandas.DataFrame """ # A word about the 'fields' argument: # Notice that we assume that the same field name cannot occur in # more than one descriptor. We could relax this assumption, but # we current enforce it in bluesky, so it is safe for now. try: headers.items() except AttributeError: pass else: headers = [headers] if fields is None: fields = [] fields = set(fields) dfs = [] for header in headers: descriptors = find_descriptors(header['start']['uid']) for descriptor in descriptors: all_fields = set(descriptor['data_keys']) if fields: discard_fields = all_fields - fields else: discard_fields = [] if discard_fields == all_fields: continue is_external = _inspect_descriptor(descriptor) payload = get_events_table(descriptor) descriptor, data, seq_nums, times, uids, timestamps = payload df = pd.DataFrame(index=seq_nums) if convert_times: times = pd.to_datetime( pd.Series(times), unit='s', utc=True).dt.tz_localize(TZ) df['time'] = times for field, values in six.iteritems(data): if field in discard_fields: logger.debug('Discarding field %s', field) continue if is_external[field] and fill: logger.debug('filling data for %s', field) # TODO someday we will have bulk retrieve in FS values = [fs.retrieve(value) for value in values] df[field] = values dfs.append(df) if dfs: return pd.concat(dfs) else: # edge case: no data return pd.DataFrame()
def test_retrieval(self): with handler_context({'npy_FRAMEWISE': NpyFrameWise}): for i, datum_id in enumerate(self.datum_ids): data = retrieve(datum_id) known_data = i * np.ones((9, 8)) assert_array_equal(data, known_data)
def event(self, doc): uid = doc['data'][self.field] data = fsapi.retrieve(uid) self.cs.update_image(data) self.cs._fig.canvas.draw_idle()
def event(self, doc): import filestore.api as fsapi uid = doc['data'][self.field] data = fsapi.retrieve(uid) self.cs.update_image(data) self.cs._fig.canvas.draw_idle()
def get_frame(self, i): img = retrieve(self._datum_uids[i]) return Frame(img, frame_no=i)
plt.show() hdrs = db( run_folder='/mnt/bulk-data/research_data/USC_beamtime/APS_March_2016/S1/temp_exp') hdr = hdrs[0] print(hdr['start']['run_folder'], hdr['start']['uid']) # Get calibrations if not hdr['start']['is_calibration']: cals = [db[u]['start']['poni'][0] for u in hdr['start']['calibration']] else: cals = [p for p in hdr['start']['poni']] geos = [retrieve(p) for p in cals] cal_dists = np.asarray( [g.dist for g in geos]) * 100 # convert to meters events = get_events(hdr) ev0 = events.next() detz = ev0['data']['detz'] cal_idx = np.argmin((detz - cal_dists) ** 2) geo = geos[cal_idx] img = retrieve(ev0['data']['img']) # Correct for polarization img /= geo.polarization(img.shape, .95) r = geo.rArray(img.shape) q = geo.qArray(img.shape) / 10 # pyFAI works in nm**-1, we want A**-1 fq = geo.qArray(img.shape).ravel()
def event(self, doc): uid = doc['data'][self.field] data = fsapi.retrieve(uid) self.cs.update_image(data) self.cs._fig.canvas.draw() self.cs._fig.canvas.flush_events()
from filestore.api import retrieve from xpd_workflow.mask_tools import * from sidewinder_spec.utils.handlers import * import matplotlib.pyplot as plt from filestore.api import db_connect as fs_db_connect from metadatastore.api import db_connect as mds_db_connect fs_db_connect(**{'database': 'data-processing-dev', 'host': 'localhost', 'port': 27017}) mds_db_connect(**{'database': 'data-processing-dev', 'host': 'localhost', 'port': 27017}) # Get headers of interest hdrs = [db[-1]] for hdr in hdrs: time_dept_bg = True # Get calibrations geos = [retrieve(p) for p in hdr['start']['poni']] cal_dists = np.asarray( [g.dist for g in geos]) * 10 # pyFAI reports in meters # Get starting masks # start_masks = [retrieve(p) for p in hdr['start']['mask']] for event in get_events(hdr): # Pull relevent data into local vars data = event['data'] img = data['img'] detz = data['detz'] # Find the correct calibration file, it's the one with the dist close # to the recorded detector dist cal_idx = np.argmin((detz - cal_dists) ** 2) geo = geos[cal_idx] # start_mask = start_masks[cal_idx]