def extract_darkfield(header, dark_key): cam_desc = [d for d in header['descriptors'] if dark_key in d['data_keys']][0] events = get_events_generator(cam_desc) events = list(((ev, fill_event(ev))[0] for ev in events)) event = events[0] ims = (event['data'][dark_key] <<2) >>2 return ims.mean(axis=0)
def extract_darkfield(header, dark_key): cam_desc = [d for d in header["descriptors"] if dark_key in d["data_keys"]][0] events = get_events_generator(cam_desc) events = list(((ev, fill_event(ev))[0] for ev in events)) event = events[0] ims = (event["data"][dark_key] << 2) >> 2 return ims.mean(axis=0)
def export(headers, filename): """ Parameters ---------- headers : a Header or a list of Headers objects retruned by the Data Broker filename : string path to a new or existing HDF5 file """ if isinstance(headers, Header): headers = [headers] with h5py.File(filename) as f: for header in headers: header = dict(header) try: descriptors = header.pop('descriptors') except KeyError: warnings.warn( "Header with uid {header.uid} contains no " "data.".format(header), UserWarning) continue top_group_name = header['start']['uid'] group = f.create_group(top_group_name) _safe_attrs_assignment(group, header) for i, descriptor in enumerate(descriptors): # make sure it's a dictionary and trim any spurious keys descriptor = dict(descriptor) descriptor.pop('_name', None) desc_group = group.create_group(descriptor['uid']) data_keys = descriptor.pop('data_keys') _safe_attrs_assignment(desc_group, descriptor) events = list(get_events_generator(descriptor=descriptor)) event_times = [e['time'] for e in events] desc_group.create_dataset('time', data=event_times, compression='gzip', fletcher32=True) data_group = desc_group.create_group('data') ts_group = desc_group.create_group('timestamps') [fill_event(e) for e in events] for key, value in data_keys.items(): value = dict(value) timestamps = [e['timestamps'][key] for e in events] ts_group.create_dataset(key, data=timestamps, compression='gzip', fletcher32=True) data = [e['data'][key] for e in events] dataset = data_group.create_dataset(key, data=data, compression='gzip', fletcher32=True) # Put contents of this data key (source, etc.) # into an attribute on the associated data set. _safe_attrs_assignment(dataset, dict(value))
def export(headers, filename): """ Parameters ---------- headers : a Header or a list of Headers objects retruned by the Data Broker filename : string path to a new or existing HDF5 file """ if isinstance(headers, Header): headers = [headers] with h5py.File(filename) as f: for header in headers: header = dict(header) try: descriptors = header.pop('descriptors') except KeyError: warnings.warn("Header with uid {header.uid} contains no " "data.".format(header), UserWarning) continue top_group_name = header['start']['uid'] group = f.create_group(top_group_name) _safe_attrs_assignment(group, header) for i, descriptor in enumerate(descriptors): # make sure it's a dictionary and trim any spurious keys descriptor = dict(descriptor) descriptor.pop('_name', None) desc_group = group.create_group(descriptor['uid']) data_keys = descriptor.pop('data_keys') _safe_attrs_assignment(desc_group, descriptor) events = list(get_events_generator(descriptor=descriptor)) event_times = [e['time'] for e in events] desc_group.create_dataset('time', data=event_times, compression='gzip', fletcher32=True) data_group = desc_group.create_group('data') ts_group = desc_group.create_group('timestamps') [fill_event(e) for e in events] for key, value in data_keys.items(): value = dict(value) timestamps = [e['timestamps'][key] for e in events] ts_group.create_dataset(key, data=timestamps, compression='gzip', fletcher32=True) data = [e['data'][key] for e in events] dataset = data_group.create_dataset( key, data=data, compression='gzip', fletcher32=True) # Put contents of this data key (source, etc.) # into an attribute on the associated data set. _safe_attrs_assignment(dataset, dict(value))
def clean_images(header, pivot_key, timesource_key, dark_images=None, static_keys=None): if static_keys is None: static_keys = ['sx', 'sy', 'temp_a', 'temp_b', 'sz'] # sort out which descriptor has the key we want to pivot on pv_desc = [d for d in header['descriptors'] if pivot_key in d['data_keys']][0] # sort out which descriptor has the key that we want to zip with to get time stamps ts_desc = [d for d in header['descriptors'] if timesource_key in d['data_keys']][0] ts_events = get_events_generator(ts_desc) pv_events = get_events_generator(pv_desc) # fill_event works in place, sillyness to work around that pv_events = ((ev, fill_event(ev))[0] for ev in pv_events) pivoted_events = pivot_timeseries(pv_events, [pivot_key], static_keys) if dark_images: pivoted_events = correct_events(pivoted_events, pivot_key, dark_images) merged_events = zip_events(pivoted_events, ts_events) out_ev = reset_time(merged_events, timesource_key) yield from out_ev
def clean_images(header, pivot_key, timesource_key, dark_images=None, static_keys=None): if static_keys is None: static_keys = ["sx", "sy", "temp_a", "temp_b", "sz"] # sort out which descriptor has the key we want to pivot on pv_desc = [d for d in header["descriptors"] if pivot_key in d["data_keys"]][0] # sort out which descriptor has the key that we want to zip with to get time stamps ts_desc = [d for d in header["descriptors"] if timesource_key in d["data_keys"]][0] ts_events = get_events_generator(ts_desc) pv_events = get_events_generator(pv_desc) # fill_event works in place, sillyness to work around that pv_events = ((ev, fill_event(ev))[0] for ev in pv_events) pivoted_events = pivot_timeseries(pv_events, [pivot_key], static_keys) if dark_images: pivoted_events = correct_events(pivoted_events, pivot_key, dark_images) merged_events = zip_events(pivoted_events, ts_events) out_ev = reset_time(merged_events, timesource_key) yield from out_ev
def export(headers, filename, mds, stream_name=None, fields=None, timestamps=True, use_uid=True): """ Create hdf5 file to preserve the structure of databroker. Parameters ---------- headers : a Header or a list of Headers objects retruned by the Data Broker filename : string path to a new or existing HDF5 file mds : metadatastore object metadatastore object or alike, like db.mds from databroker stream_name : string, optional None means save all the data from each descriptor, i.e., user can define stream_name as primary, so only data with descriptor.name == primary will be saved. The default is None. fields : list, optional whitelist of names of interest; if None, all are returned; This is consistent with name convension in databroker. The default is None. timestamps : Bool, optional save timestamps or not use_uid : Bool, optional Create group name at hdf file based on uid if this value is set as True. Otherwise group name is created based on beamline id and run id. """ if isinstance(headers, Header): headers = [headers] with h5py.File(filename) as f: for header in headers: header = dict(header) try: descriptors = header.pop('descriptors') except KeyError: warnings.warn("Header with uid {header.uid} contains no " "data.".format(header), UserWarning) continue if use_uid: top_group_name = header['start']['uid'] else: top_group_name = str(header['start']['beamline_id']) + '_' + str(header['start']['scan_id']) group = f.create_group(top_group_name) _safe_attrs_assignment(group, header) for i, descriptor in enumerate(descriptors): # make sure it's a dictionary and trim any spurious keys descriptor = dict(descriptor) if stream_name: if descriptor['name'] != stream_name: continue descriptor.pop('_name', None) if use_uid: desc_group = group.create_group(descriptor['uid']) else: desc_group = group.create_group(descriptor['name']) data_keys = descriptor.pop('data_keys') _safe_attrs_assignment(desc_group, descriptor) events = list(mds.get_events_generator(descriptor)) event_times = [e['time'] for e in events] desc_group.create_dataset('time', data=event_times, compression='gzip', fletcher32=True) data_group = desc_group.create_group('data') if timestamps: ts_group = desc_group.create_group('timestamps') [fill_event(e) for e in events] for key, value in data_keys.items(): if fields is not None: if key not in fields: continue if timestamps: timestamps = [e['timestamps'][key] for e in events] ts_group.create_dataset(key, data=timestamps, compression='gzip', fletcher32=True) data = [e['data'][key] for e in events] data = np.array(data) if value['dtype'].lower() == 'string': # 1D of string data_len = len(data[0]) data = data.astype('|S'+str(data_len)) dataset = data_group.create_dataset( key, data=data, compression='gzip') elif data.dtype.kind in ['S', 'U']: # 2D of string, we can't tell from dytpe, they are shown as array only. if data.ndim == 2: data_len = 1 for v in data[0]: data_len = max(data_len, len(v)) data = data.astype('|S'+str(data_len)) dataset = data_group.create_dataset( key, data=data, compression='gzip') else: raise ValueError('Array of str with ndim >= 3 can not be saved.') else: # save numerical data dataset = data_group.create_dataset( key, data=data, compression='gzip', fletcher32=True) # Put contents of this data key (source, etc.) # into an attribute on the associated data set. _safe_attrs_assignment(dataset, dict(value))
def export(headers, filename, mds, stream_name=None, fields=None, timestamps=True, use_uid=True): """ Create NeXus HDF5 file to preserve the structure of databroker. Parameters ---------- headers : a Header or a list of Headers objects retruned by the Data Broker filename : string path to a new or existing HDF5 file mds : metadatastore object metadatastore object or alike, like db.mds from databroker stream_name : string, optional None means save all the data from each descriptor, i.e., user can define stream_name as primary, so only data with descriptor.name == primary will be saved. The default is None. fields : list, optional whitelist of names of interest; if None, all are returned; This is consistent with name convension in databroker. The default is None. timestamps : Bool, optional save timestamps or not use_uid : Bool, optional Create group name at hdf file based on uid if this value is set as True. Otherwise group name is created based on beamline id and run id. """ if isinstance(headers, Header): headers = [headers] with h5py.File(filename) as f: # :see: http://download.nexusformat.org/doc/html/classes/base_classes/NXroot.html f.attrs['file_name'] = filename f.attrs['file_time'] = str(datetime.datetime.now()) f.attrs['creator'] = 'https://github.com/NSLS-II/suitcase/suitcase/nexus.py' f.attrs['HDF5_Version'] = h5py.version.hdf5_version for header in headers: header = dict(header) try: descriptors = header.pop('descriptors') except KeyError: warnings.warn("Header with uid {header.uid} contains no " "data.".format(header), UserWarning) continue if use_uid: proposed_name = header['start']['uid'] else: proposed_name = str(header['start']['beamline_id']) proposed_name += '_' + str(header['start']['scan_id']) nxentry = f.create_group(pick_NeXus_safe_name(proposed_name)) nxentry.attrs["NX_class"] = "NXentry" #header.pop('_name') _safe_attrs_assignment(nxentry, header) # TODO: improve this if f.attrs.get("default") is None: f.attrs["default"] = nxentry.name.split("/")[-1] for i, descriptor in enumerate(descriptors): # make sure it's a dictionary and trim any spurious keys descriptor = dict(descriptor) if stream_name: if descriptor['name'] != stream_name: continue descriptor.pop('_name', None) if use_uid: proposed_name = descriptor['uid'] else: proposed_name = descriptor['name'] nxlog = nxentry.create_group( pick_NeXus_safe_name(proposed_name)) nxlog.attrs["NX_class"] = "NXlog" data_keys = descriptor.pop('data_keys') _safe_attrs_assignment(nxlog, descriptor) # TODO: possible to create a useful NXinstrument group? nxdata = nxentry.create_group( pick_NeXus_safe_name(proposed_name + '_data')) nxdata.attrs["NX_class"] = "NXdata" if nxentry.attrs.get("default") is None: nxentry.attrs["default"] = nxdata.name.split("/")[-1] ''' structure (under nxlog:NXlog): [data_keys] @axes = data_key_timestamps [data_keys]_timestamps time (must be renamed or converted) Is this necessary? :see: http://download.nexusformat.org/doc/html/classes/base_classes/NXlog.html ''' events = list(mds.get_events_generator(descriptor)) event_times = np.array([e['time'] for e in events]) start = event_times[0] ds = nxlog.create_dataset( 'time', data=event_times-start, compression='gzip', fletcher32=True) ds.attrs['units'] = 's' datetime_string = time.asctime(time.gmtime(start)) ds.attrs['start'] = dateutil.parser.parse(datetime_string).isoformat() [fill_event(e) for e in events] for key, value in data_keys.items(): if fields is not None: if key not in fields: continue safename = pick_NeXus_safe_name(key) if timestamps: timestamps = [e['timestamps'][safename] for e in events] ts = nxlog.create_dataset(safename+'_timestamps', data=timestamps, compression='gzip', fletcher32=True) ts.attrs['key_name'] = key else: ts = None data = [e['data'][key] for e in events] data = np.array(data) if value['dtype'].lower() == 'string': # 1D of string data_len = len(data[0]) data = data.astype('|S'+str(data_len)) dataset = nxlog.create_dataset( safename, data=data, compression='gzip') elif data.dtype.kind in ['S', 'U']: # 2D of string, we can't tell from dytpe, they are shown as array only. if data.ndim == 2: data_len = 1 for v in data[0]: data_len = max(data_len, len(v)) data = data.astype('|S'+str(data_len)) dataset = nxlog.create_dataset( safename, data=data, compression='gzip') else: raise ValueError('Array of str with ndim >= 3 can not be saved.') else: # save numerical data dataset = nxlog.create_dataset( safename, data=data, compression='gzip', fletcher32=True) dataset.attrs['key_name'] = key # only link to the NXdata group if the data is numerical if value['dtype'] in ('number',): nxdata[safename] = dataset dataset.attrs['target'] = dataset.name if nxdata.attrs.get("h5py") is None: nxdata.attrs["signal"] = dataset.name.split("/")[-1] if ts is not None: # point to the associated timestamp array dataset.attrs['axes'] = ts.name.split('/')[-1] if value['dtype'] in ('number',): nxdata[dataset.attrs['axes']] = ts ts.attrs['target'] = ts.name # Put contents of this data key (source, etc.) # into an attribute on the associated data set. _safe_attrs_assignment(dataset, dict(value)) if nxdata.attrs.get("signal") is None: del nxdata # TODO: is this the correct way to delete the empty NXdata group?
def fill_all_events(*args, **kwargs): events = func(*args, **kwargs) for event in events: fill_event(event) return events