def extract_darkfield(header, dark_key):
cam_desc = [d for d in header['descriptors'] if dark_key in d['data_keys']][0]
events = get_events_generator(cam_desc)
events = list(((ev, fill_event(ev))[0] for ev in events))
event = events[0]
ims = (event['data'][dark_key] <<2) >>2
return ims.mean(axis=0)
def extract_darkfield(header, dark_key):
cam_desc = [d for d in header["descriptors"] if dark_key in d["data_keys"]][0]
events = get_events_generator(cam_desc)
events = list(((ev, fill_event(ev))[0] for ev in events))
event = events[0]
ims = (event["data"][dark_key] << 2) >> 2
return ims.mean(axis=0)
def export(headers, filename):
"""
Parameters
----------
headers : a Header or a list of Headers
objects retruned by the Data Broker
filename : string
path to a new or existing HDF5 file
"""
if isinstance(headers, Header):
headers = [headers]
with h5py.File(filename) as f:
for header in headers:
header = dict(header)
try:
descriptors = header.pop('descriptors')
except KeyError:
warnings.warn(
"Header with uid {header.uid} contains no "
"data.".format(header), UserWarning)
continue
top_group_name = header['start']['uid']
group = f.create_group(top_group_name)
_safe_attrs_assignment(group, header)
for i, descriptor in enumerate(descriptors):
# make sure it's a dictionary and trim any spurious keys
descriptor = dict(descriptor)
descriptor.pop('_name', None)
desc_group = group.create_group(descriptor['uid'])
data_keys = descriptor.pop('data_keys')
_safe_attrs_assignment(desc_group, descriptor)
events = list(get_events_generator(descriptor=descriptor))
event_times = [e['time'] for e in events]
desc_group.create_dataset('time',
data=event_times,
compression='gzip',
fletcher32=True)
data_group = desc_group.create_group('data')
ts_group = desc_group.create_group('timestamps')
[fill_event(e) for e in events]
for key, value in data_keys.items():
value = dict(value)
timestamps = [e['timestamps'][key] for e in events]
ts_group.create_dataset(key,
data=timestamps,
compression='gzip',
fletcher32=True)
data = [e['data'][key] for e in events]
dataset = data_group.create_dataset(key,
data=data,
compression='gzip',
fletcher32=True)
# Put contents of this data key (source, etc.)
# into an attribute on the associated data set.
_safe_attrs_assignment(dataset, dict(value))
def export(headers, filename):
"""
Parameters
----------
headers : a Header or a list of Headers
objects retruned by the Data Broker
filename : string
path to a new or existing HDF5 file
"""
if isinstance(headers, Header):
headers = [headers]
with h5py.File(filename) as f:
for header in headers:
header = dict(header)
try:
descriptors = header.pop('descriptors')
except KeyError:
warnings.warn("Header with uid {header.uid} contains no "
"data.".format(header), UserWarning)
continue
top_group_name = header['start']['uid']
group = f.create_group(top_group_name)
_safe_attrs_assignment(group, header)
for i, descriptor in enumerate(descriptors):
# make sure it's a dictionary and trim any spurious keys
descriptor = dict(descriptor)
descriptor.pop('_name', None)
desc_group = group.create_group(descriptor['uid'])
data_keys = descriptor.pop('data_keys')
_safe_attrs_assignment(desc_group, descriptor)
events = list(get_events_generator(descriptor=descriptor))
event_times = [e['time'] for e in events]
desc_group.create_dataset('time', data=event_times,
compression='gzip', fletcher32=True)
data_group = desc_group.create_group('data')
ts_group = desc_group.create_group('timestamps')
[fill_event(e) for e in events]
for key, value in data_keys.items():
value = dict(value)
timestamps = [e['timestamps'][key] for e in events]
ts_group.create_dataset(key, data=timestamps,
compression='gzip',
fletcher32=True)
data = [e['data'][key] for e in events]
dataset = data_group.create_dataset(
key, data=data, compression='gzip', fletcher32=True)
# Put contents of this data key (source, etc.)
# into an attribute on the associated data set.
_safe_attrs_assignment(dataset, dict(value))
def clean_images(header, pivot_key, timesource_key, dark_images=None, static_keys=None):
if static_keys is None:
static_keys = ['sx', 'sy', 'temp_a', 'temp_b', 'sz']
# sort out which descriptor has the key we want to pivot on
pv_desc = [d for d in header['descriptors'] if pivot_key in d['data_keys']][0]
# sort out which descriptor has the key that we want to zip with to get time stamps
ts_desc = [d for d in header['descriptors'] if timesource_key in d['data_keys']][0]
ts_events = get_events_generator(ts_desc)
pv_events = get_events_generator(pv_desc)
# fill_event works in place, sillyness to work around that
pv_events = ((ev, fill_event(ev))[0] for ev in pv_events)
pivoted_events = pivot_timeseries(pv_events, [pivot_key], static_keys)
if dark_images:
pivoted_events = correct_events(pivoted_events, pivot_key, dark_images)
merged_events = zip_events(pivoted_events, ts_events)
out_ev = reset_time(merged_events, timesource_key)
yield from out_ev
def clean_images(header, pivot_key, timesource_key, dark_images=None, static_keys=None):
if static_keys is None:
static_keys = ["sx", "sy", "temp_a", "temp_b", "sz"]
# sort out which descriptor has the key we want to pivot on
pv_desc = [d for d in header["descriptors"] if pivot_key in d["data_keys"]][0]
# sort out which descriptor has the key that we want to zip with to get time stamps
ts_desc = [d for d in header["descriptors"] if timesource_key in d["data_keys"]][0]
ts_events = get_events_generator(ts_desc)
pv_events = get_events_generator(pv_desc)
# fill_event works in place, sillyness to work around that
pv_events = ((ev, fill_event(ev))[0] for ev in pv_events)
pivoted_events = pivot_timeseries(pv_events, [pivot_key], static_keys)
if dark_images:
pivoted_events = correct_events(pivoted_events, pivot_key, dark_images)
merged_events = zip_events(pivoted_events, ts_events)
out_ev = reset_time(merged_events, timesource_key)
yield from out_ev
def export(headers, filename, mds,
stream_name=None, fields=None, timestamps=True, use_uid=True):
"""
Create hdf5 file to preserve the structure of databroker.
Parameters
----------
headers : a Header or a list of Headers
objects retruned by the Data Broker
filename : string
path to a new or existing HDF5 file
mds : metadatastore object
metadatastore object or alike, like db.mds from databroker
stream_name : string, optional
None means save all the data from each descriptor, i.e., user can define stream_name as primary,
so only data with descriptor.name == primary will be saved.
The default is None.
fields : list, optional
whitelist of names of interest; if None, all are returned;
This is consistent with name convension in databroker.
The default is None.
timestamps : Bool, optional
save timestamps or not
use_uid : Bool, optional
Create group name at hdf file based on uid if this value is set as True.
Otherwise group name is created based on beamline id and run id.
"""
if isinstance(headers, Header):
headers = [headers]
with h5py.File(filename) as f:
for header in headers:
header = dict(header)
try:
descriptors = header.pop('descriptors')
except KeyError:
warnings.warn("Header with uid {header.uid} contains no "
"data.".format(header), UserWarning)
continue
if use_uid:
top_group_name = header['start']['uid']
else:
top_group_name = str(header['start']['beamline_id']) + '_' + str(header['start']['scan_id'])
group = f.create_group(top_group_name)
_safe_attrs_assignment(group, header)
for i, descriptor in enumerate(descriptors):
# make sure it's a dictionary and trim any spurious keys
descriptor = dict(descriptor)
if stream_name:
if descriptor['name'] != stream_name:
continue
descriptor.pop('_name', None)
if use_uid:
desc_group = group.create_group(descriptor['uid'])
else:
desc_group = group.create_group(descriptor['name'])
data_keys = descriptor.pop('data_keys')
_safe_attrs_assignment(desc_group, descriptor)
events = list(mds.get_events_generator(descriptor))
event_times = [e['time'] for e in events]
desc_group.create_dataset('time', data=event_times,
compression='gzip', fletcher32=True)
data_group = desc_group.create_group('data')
if timestamps:
ts_group = desc_group.create_group('timestamps')
[fill_event(e) for e in events]
for key, value in data_keys.items():
if fields is not None:
if key not in fields:
continue
if timestamps:
timestamps = [e['timestamps'][key] for e in events]
ts_group.create_dataset(key, data=timestamps,
compression='gzip',
fletcher32=True)
data = [e['data'][key] for e in events]
data = np.array(data)
if value['dtype'].lower() == 'string': # 1D of string
data_len = len(data[0])
data = data.astype('|S'+str(data_len))
dataset = data_group.create_dataset(
key, data=data, compression='gzip')
elif data.dtype.kind in ['S', 'U']:
# 2D of string, we can't tell from dytpe, they are shown as array only.
if data.ndim == 2:
data_len = 1
for v in data[0]:
data_len = max(data_len, len(v))
data = data.astype('|S'+str(data_len))
dataset = data_group.create_dataset(
key, data=data, compression='gzip')
else:
raise ValueError('Array of str with ndim >= 3 can not be saved.')
else: # save numerical data
dataset = data_group.create_dataset(
key, data=data,
compression='gzip', fletcher32=True)
# Put contents of this data key (source, etc.)
# into an attribute on the associated data set.
_safe_attrs_assignment(dataset, dict(value))
def export(headers, filename, mds,
stream_name=None, fields=None, timestamps=True, use_uid=True):
"""
Create NeXus HDF5 file to preserve the structure of databroker.
Parameters
----------
headers : a Header or a list of Headers
objects retruned by the Data Broker
filename : string
path to a new or existing HDF5 file
mds : metadatastore object
metadatastore object or alike, like db.mds from databroker
stream_name : string, optional
None means save all the data from each descriptor, i.e., user can define stream_name as primary,
so only data with descriptor.name == primary will be saved.
The default is None.
fields : list, optional
whitelist of names of interest; if None, all are returned;
This is consistent with name convension in databroker.
The default is None.
timestamps : Bool, optional
save timestamps or not
use_uid : Bool, optional
Create group name at hdf file based on uid if this value is set as True.
Otherwise group name is created based on beamline id and run id.
"""
if isinstance(headers, Header):
headers = [headers]
with h5py.File(filename) as f:
# :see: http://download.nexusformat.org/doc/html/classes/base_classes/NXroot.html
f.attrs['file_name'] = filename
f.attrs['file_time'] = str(datetime.datetime.now())
f.attrs['creator'] = 'https://github.com/NSLS-II/suitcase/suitcase/nexus.py'
f.attrs['HDF5_Version'] = h5py.version.hdf5_version
for header in headers:
header = dict(header)
try:
descriptors = header.pop('descriptors')
except KeyError:
warnings.warn("Header with uid {header.uid} contains no "
"data.".format(header), UserWarning)
continue
if use_uid:
proposed_name = header['start']['uid']
else:
proposed_name = str(header['start']['beamline_id'])
proposed_name += '_' + str(header['start']['scan_id'])
nxentry = f.create_group(pick_NeXus_safe_name(proposed_name))
nxentry.attrs["NX_class"] = "NXentry"
#header.pop('_name')
_safe_attrs_assignment(nxentry, header) # TODO: improve this
if f.attrs.get("default") is None:
f.attrs["default"] = nxentry.name.split("/")[-1]
for i, descriptor in enumerate(descriptors):
# make sure it's a dictionary and trim any spurious keys
descriptor = dict(descriptor)
if stream_name:
if descriptor['name'] != stream_name:
continue
descriptor.pop('_name', None)
if use_uid:
proposed_name = descriptor['uid']
else:
proposed_name = descriptor['name']
nxlog = nxentry.create_group(
pick_NeXus_safe_name(proposed_name))
nxlog.attrs["NX_class"] = "NXlog"
data_keys = descriptor.pop('data_keys')
_safe_attrs_assignment(nxlog, descriptor)
# TODO: possible to create a useful NXinstrument group?
nxdata = nxentry.create_group(
pick_NeXus_safe_name(proposed_name + '_data'))
nxdata.attrs["NX_class"] = "NXdata"
if nxentry.attrs.get("default") is None:
nxentry.attrs["default"] = nxdata.name.split("/")[-1]
'''
structure (under nxlog:NXlog):
[data_keys]
@axes = data_key_timestamps
[data_keys]_timestamps
time (must be renamed or converted) Is this necessary?
:see: http://download.nexusformat.org/doc/html/classes/base_classes/NXlog.html
'''
events = list(mds.get_events_generator(descriptor))
event_times = np.array([e['time'] for e in events])
start = event_times[0]
ds = nxlog.create_dataset(
'time', data=event_times-start, compression='gzip', fletcher32=True)
ds.attrs['units'] = 's'
datetime_string = time.asctime(time.gmtime(start))
ds.attrs['start'] = dateutil.parser.parse(datetime_string).isoformat()
[fill_event(e) for e in events]
for key, value in data_keys.items():
if fields is not None:
if key not in fields:
continue
safename = pick_NeXus_safe_name(key)
if timestamps:
timestamps = [e['timestamps'][safename] for e in events]
ts = nxlog.create_dataset(safename+'_timestamps', data=timestamps,
compression='gzip',
fletcher32=True)
ts.attrs['key_name'] = key
else:
ts = None
data = [e['data'][key] for e in events]
data = np.array(data)
if value['dtype'].lower() == 'string': # 1D of string
data_len = len(data[0])
data = data.astype('|S'+str(data_len))
dataset = nxlog.create_dataset(
safename, data=data, compression='gzip')
elif data.dtype.kind in ['S', 'U']:
# 2D of string, we can't tell from dytpe, they are shown as array only.
if data.ndim == 2:
data_len = 1
for v in data[0]:
data_len = max(data_len, len(v))
data = data.astype('|S'+str(data_len))
dataset = nxlog.create_dataset(
safename, data=data, compression='gzip')
else:
raise ValueError('Array of str with ndim >= 3 can not be saved.')
else: # save numerical data
dataset = nxlog.create_dataset(
safename, data=data,
compression='gzip', fletcher32=True)
dataset.attrs['key_name'] = key
# only link to the NXdata group if the data is numerical
if value['dtype'] in ('number',):
nxdata[safename] = dataset
dataset.attrs['target'] = dataset.name
if nxdata.attrs.get("h5py") is None:
nxdata.attrs["signal"] = dataset.name.split("/")[-1]
if ts is not None:
# point to the associated timestamp array
dataset.attrs['axes'] = ts.name.split('/')[-1]
if value['dtype'] in ('number',):
nxdata[dataset.attrs['axes']] = ts
ts.attrs['target'] = ts.name
# Put contents of this data key (source, etc.)
# into an attribute on the associated data set.
_safe_attrs_assignment(dataset, dict(value))
if nxdata.attrs.get("signal") is None:
del nxdata # TODO: is this the correct way to delete the empty NXdata group?
def fill_all_events(*args, **kwargs):
events = func(*args, **kwargs)
for event in events:
fill_event(event)
return events
