def extract_darkfield(header, dark_key):
cam_desc = [d for d in header["descriptors"] if dark_key in d["data_keys"]][0]
events = get_events_generator(cam_desc)
events = list(((ev, fill_event(ev))[0] for ev in events))
event = events[0]
ims = (event["data"][dark_key] << 2) >> 2
return ims.mean(axis=0)
def extract_darkfield(header, dark_key):
cam_desc = [d for d in header['descriptors'] if dark_key in d['data_keys']][0]
events = get_events_generator(cam_desc)
events = list(((ev, fill_event(ev))[0] for ev in events))
event = events[0]
ims = (event['data'][dark_key] <<2) >>2
return ims.mean(axis=0)
def export(headers, filename):
"""
Parameters
----------
headers : a Header or a list of Headers
objects retruned by the Data Broker
filename : string
path to a new or existing HDF5 file
"""
if isinstance(headers, Header):
headers = [headers]
with h5py.File(filename) as f:
for header in headers:
header = dict(header)
try:
descriptors = header.pop('descriptors')
except KeyError:
warnings.warn(
"Header with uid {header.uid} contains no "
"data.".format(header), UserWarning)
continue
top_group_name = header['start']['uid']
group = f.create_group(top_group_name)
_safe_attrs_assignment(group, header)
for i, descriptor in enumerate(descriptors):
# make sure it's a dictionary and trim any spurious keys
descriptor = dict(descriptor)
descriptor.pop('_name', None)
desc_group = group.create_group(descriptor['uid'])
data_keys = descriptor.pop('data_keys')
_safe_attrs_assignment(desc_group, descriptor)
events = list(get_events_generator(descriptor=descriptor))
event_times = [e['time'] for e in events]
desc_group.create_dataset('time',
data=event_times,
compression='gzip',
fletcher32=True)
data_group = desc_group.create_group('data')
ts_group = desc_group.create_group('timestamps')
[fill_event(e) for e in events]
for key, value in data_keys.items():
value = dict(value)
timestamps = [e['timestamps'][key] for e in events]
ts_group.create_dataset(key,
data=timestamps,
compression='gzip',
fletcher32=True)
data = [e['data'][key] for e in events]
dataset = data_group.create_dataset(key,
data=data,
compression='gzip',
fletcher32=True)
# Put contents of this data key (source, etc.)
# into an attribute on the associated data set.
_safe_attrs_assignment(dataset, dict(value))
def clean_images(header, pivot_key, timesource_key, dark_images=None, static_keys=None):
if static_keys is None:
static_keys = ['sx', 'sy', 'temp_a', 'temp_b', 'sz']
# sort out which descriptor has the key we want to pivot on
pv_desc = [d for d in header['descriptors'] if pivot_key in d['data_keys']][0]
# sort out which descriptor has the key that we want to zip with to get time stamps
ts_desc = [d for d in header['descriptors'] if timesource_key in d['data_keys']][0]
ts_events = get_events_generator(ts_desc)
pv_events = get_events_generator(pv_desc)
# fill_event works in place, sillyness to work around that
pv_events = ((ev, fill_event(ev))[0] for ev in pv_events)
pivoted_events = pivot_timeseries(pv_events, [pivot_key], static_keys)
if dark_images:
pivoted_events = correct_events(pivoted_events, pivot_key, dark_images)
merged_events = zip_events(pivoted_events, ts_events)
out_ev = reset_time(merged_events, timesource_key)
yield from out_ev
def clean_images(header, pivot_key, timesource_key, dark_images=None, static_keys=None):
if static_keys is None:
static_keys = ["sx", "sy", "temp_a", "temp_b", "sz"]
# sort out which descriptor has the key we want to pivot on
pv_desc = [d for d in header["descriptors"] if pivot_key in d["data_keys"]][0]
# sort out which descriptor has the key that we want to zip with to get time stamps
ts_desc = [d for d in header["descriptors"] if timesource_key in d["data_keys"]][0]
ts_events = get_events_generator(ts_desc)
pv_events = get_events_generator(pv_desc)
# fill_event works in place, sillyness to work around that
pv_events = ((ev, fill_event(ev))[0] for ev in pv_events)
pivoted_events = pivot_timeseries(pv_events, [pivot_key], static_keys)
if dark_images:
pivoted_events = correct_events(pivoted_events, pivot_key, dark_images)
merged_events = zip_events(pivoted_events, ts_events)
out_ev = reset_time(merged_events, timesource_key)
yield from out_ev
def export(headers, filename):
"""
Parameters
----------
headers : a Header or a list of Headers
objects retruned by the Data Broker
filename : string
path to a new or existing HDF5 file
"""
if isinstance(headers, Header):
headers = [headers]
with h5py.File(filename) as f:
for header in headers:
header = dict(header)
try:
descriptors = header.pop('descriptors')
except KeyError:
warnings.warn("Header with uid {header.uid} contains no "
"data.".format(header), UserWarning)
continue
top_group_name = header['start']['uid']
group = f.create_group(top_group_name)
_safe_attrs_assignment(group, header)
for i, descriptor in enumerate(descriptors):
# make sure it's a dictionary and trim any spurious keys
descriptor = dict(descriptor)
descriptor.pop('_name', None)
desc_group = group.create_group(descriptor['uid'])
data_keys = descriptor.pop('data_keys')
_safe_attrs_assignment(desc_group, descriptor)
events = list(get_events_generator(descriptor=descriptor))
event_times = [e['time'] for e in events]
desc_group.create_dataset('time', data=event_times,
compression='gzip', fletcher32=True)
data_group = desc_group.create_group('data')
ts_group = desc_group.create_group('timestamps')
[fill_event(e) for e in events]
for key, value in data_keys.items():
value = dict(value)
timestamps = [e['timestamps'][key] for e in events]
ts_group.create_dataset(key, data=timestamps,
compression='gzip',
fletcher32=True)
data = [e['data'][key] for e in events]
dataset = data_group.create_dataset(
key, data=data, compression='gzip', fletcher32=True)
# Put contents of this data key (source, etc.)
# into an attribute on the associated data set.
_safe_attrs_assignment(dataset, dict(value))
def test_bulk_insert():
num = 50
rs, e_desc, data_keys = setup_syn()
all_data = syn_data(data_keys, num)
mdsc.bulk_insert_events(e_desc, all_data, validate=False)
mdsc.insert_run_stop(rs, ttime.time(), uid=str(uuid.uuid4()))
ev_gen = mdsc.get_events_generator(e_desc)
for ret, expt in zip(ev_gen, all_data):
assert ret['descriptor']['uid'] == e_desc
for k in ['data', 'timestamps', 'time', 'uid', 'seq_num']:
assert ret[k] == expt[k]
def test_bulk_insert():
num = 50
rs, e_desc, data_keys = setup_syn()
all_data = syn_data(data_keys, num)
mdsc.bulk_insert_events(e_desc, all_data, validate=False)
mdsc.insert_run_stop(rs, ttime.time(), uid=str(uuid.uuid4()))
ev_gen = mdsc.get_events_generator(e_desc)
for ret, expt in zip(ev_gen, all_data):
assert_equal(ret['descriptor']['uid'], e_desc)
for k in ['data', 'timestamps', 'time', 'uid', 'seq_num']:
assert_equal(ret[k], expt[k])
def get_events(headers, fields=None, fill=True):
"""
Get Events from given run(s).
Parameters
----------
headers : Header or iterable of Headers
The headers to fetch the events for
fields : list, optional
whitelist of field names of interest; if None, all are returned
fill : bool, optional
Whether externally-stored data should be filled in. Defaults to True
Yields
------
event : Event
The event, optionally with non-scalar data filled in
"""
# A word about the 'fields' argument:
# Notice that we assume that the same field name cannot occur in
# more than one descriptor. We could relax this assumption, but
# we current enforce it in bluesky, so it is safe for now.
try:
headers.items()
except AttributeError:
pass
else:
headers = [headers]
if fields is None:
fields = []
fields = set(fields)
for header in headers:
descriptors = find_descriptors(header['start']['uid'])
for descriptor in descriptors:
all_fields = set(descriptor['data_keys'])
if fields:
discard_fields = all_fields - fields
else:
discard_fields = []
if discard_fields == all_fields:
continue
for event in get_events_generator(descriptor):
for field in discard_fields:
del event.data[field]
del event.timestamps[field]
if fill:
fill_event(event)
yield event
def test_spec_to_document(sf):
map = {
'start': insert_run_start,
'stop': insert_run_stop,
'descriptor': insert_descriptor,
'event': insert_event
}
start_uids = list()
for document_name, document in spec_to_document(sf, validate=True):
document = dict(document)
del document['_name']
if not isinstance(document_name, str):
document_name = document_name.name
# insert the documents
if document_name == 'start':
document['beamline_id'] = 'test'
start_uids.append(document['uid'])
map[document_name](**document)
# make sure we are not trying to add duplicates
assert len(start_uids) == len(set(start_uids))
# smoketest the retrieval
hdrs = []
for uid in start_uids:
hdr = db[uid]
# make sure we only get one back
assert isinstance(hdr, Header)
hdrs.append(hdr)
# make sure we are not getting duplicates back out
hdr_uids = [hdr.start.uid for hdr in hdrs]
assert len(hdr_uids) == len(set(hdr_uids))
if isinstance(sf, Specscan):
sf = [sf]
if isinstance(sf, str):
sf = Specfile(sf)
for hdr, specscan in zip(hdrs, sf):
for descriptor in hdr.descriptors:
ev = list(get_events_generator(descriptor))
if descriptor.get('name') == 'baseline':
# we better only have one baseline event
assert len(ev) == 1
else:
assert len(specscan.scan_data) == len(ev)
def test_spec_to_document(sf):
map = {
'start': insert_run_start,
'stop': insert_run_stop,
'descriptor': insert_descriptor,
'event': insert_event
}
start_uids = list()
for document_name, document in spec_to_document(sf, validate=True):
document = dict(document)
del document['_name']
if not isinstance(document_name, str):
document_name = document_name.name
# insert the documents
if document_name == 'start':
document['beamline_id'] = 'test'
start_uids.append(document['uid'])
map[document_name](**document)
# make sure we are not trying to add duplicates
assert len(start_uids) == len(set(start_uids))
# smoketest the retrieval
hdrs = []
for uid in start_uids:
hdr = db[uid]
# make sure we only get one back
assert isinstance(hdr, Header)
hdrs.append(hdr)
# make sure we are not getting duplicates back out
hdr_uids = [hdr.start.uid for hdr in hdrs]
assert len(hdr_uids) == len(set(hdr_uids))
if isinstance(sf, Specscan):
sf = [sf]
if isinstance(sf, str):
sf = Specfile(sf)
for hdr, specscan in zip(hdrs, sf):
for descriptor in hdr.descriptors:
ev = list(get_events_generator(descriptor))
if descriptor.get('name') == 'baseline':
# we better only have one baseline event
assert len(ev) == 1
else:
assert len(specscan.scan_data) == len(ev)