예제 #1
0
def extract_darkfield(header, dark_key):
    cam_desc = [d for d in header["descriptors"] if dark_key in d["data_keys"]][0]
    events = get_events_generator(cam_desc)
    events = list(((ev, fill_event(ev))[0] for ev in events))
    event = events[0]
    ims = (event["data"][dark_key] << 2) >> 2
    return ims.mean(axis=0)
예제 #2
0
def extract_darkfield(header, dark_key):
    cam_desc = [d for d in header['descriptors'] if dark_key in d['data_keys']][0]
    events = get_events_generator(cam_desc)
    events = list(((ev, fill_event(ev))[0] for ev in events))
    event = events[0]
    ims = (event['data'][dark_key] <<2) >>2
    return ims.mean(axis=0)
예제 #3
0
def export(headers, filename):
    """
    Parameters
    ----------
    headers : a Header or a list of Headers
        objects retruned by the Data Broker
    filename : string
        path to a new or existing HDF5 file
    """
    if isinstance(headers, Header):
        headers = [headers]
    with h5py.File(filename) as f:
        for header in headers:
            header = dict(header)
            try:
                descriptors = header.pop('descriptors')
            except KeyError:
                warnings.warn(
                    "Header with uid {header.uid} contains no "
                    "data.".format(header), UserWarning)
                continue
            top_group_name = header['start']['uid']
            group = f.create_group(top_group_name)
            _safe_attrs_assignment(group, header)
            for i, descriptor in enumerate(descriptors):
                # make sure it's a dictionary and trim any spurious keys
                descriptor = dict(descriptor)
                descriptor.pop('_name', None)

                desc_group = group.create_group(descriptor['uid'])

                data_keys = descriptor.pop('data_keys')
                _safe_attrs_assignment(desc_group, descriptor)

                events = list(get_events_generator(descriptor=descriptor))
                event_times = [e['time'] for e in events]
                desc_group.create_dataset('time',
                                          data=event_times,
                                          compression='gzip',
                                          fletcher32=True)
                data_group = desc_group.create_group('data')
                ts_group = desc_group.create_group('timestamps')
                [fill_event(e) for e in events]
                for key, value in data_keys.items():
                    value = dict(value)
                    timestamps = [e['timestamps'][key] for e in events]
                    ts_group.create_dataset(key,
                                            data=timestamps,
                                            compression='gzip',
                                            fletcher32=True)
                    data = [e['data'][key] for e in events]
                    dataset = data_group.create_dataset(key,
                                                        data=data,
                                                        compression='gzip',
                                                        fletcher32=True)
                    # Put contents of this data key (source, etc.)
                    # into an attribute on the associated data set.
                    _safe_attrs_assignment(dataset, dict(value))
예제 #4
0
def clean_images(header, pivot_key, timesource_key, dark_images=None, static_keys=None):
    if static_keys is None:
        static_keys = ['sx', 'sy', 'temp_a', 'temp_b', 'sz']
    # sort out which descriptor has the key we want to pivot on
    pv_desc = [d for d in header['descriptors'] if pivot_key in d['data_keys']][0]
    # sort out which descriptor has the key that we want to zip with to get time stamps
    ts_desc = [d for d in header['descriptors'] if timesource_key in d['data_keys']][0]

    ts_events = get_events_generator(ts_desc)
    pv_events = get_events_generator(pv_desc)
    # fill_event works in place, sillyness to work around that
    pv_events = ((ev, fill_event(ev))[0] for ev in pv_events)
    pivoted_events = pivot_timeseries(pv_events, [pivot_key], static_keys)

    if dark_images:
        pivoted_events = correct_events(pivoted_events, pivot_key, dark_images)
    merged_events = zip_events(pivoted_events, ts_events)
    out_ev = reset_time(merged_events, timesource_key)
    yield from out_ev
예제 #5
0
def clean_images(header, pivot_key, timesource_key, dark_images=None, static_keys=None):
    if static_keys is None:
        static_keys = ["sx", "sy", "temp_a", "temp_b", "sz"]
    # sort out which descriptor has the key we want to pivot on
    pv_desc = [d for d in header["descriptors"] if pivot_key in d["data_keys"]][0]
    # sort out which descriptor has the key that we want to zip with to get time stamps
    ts_desc = [d for d in header["descriptors"] if timesource_key in d["data_keys"]][0]

    ts_events = get_events_generator(ts_desc)
    pv_events = get_events_generator(pv_desc)
    # fill_event works in place, sillyness to work around that
    pv_events = ((ev, fill_event(ev))[0] for ev in pv_events)
    pivoted_events = pivot_timeseries(pv_events, [pivot_key], static_keys)

    if dark_images:
        pivoted_events = correct_events(pivoted_events, pivot_key, dark_images)
    merged_events = zip_events(pivoted_events, ts_events)
    out_ev = reset_time(merged_events, timesource_key)
    yield from out_ev
예제 #6
0
def export(headers, filename):
    """
    Parameters
    ----------
    headers : a Header or a list of Headers
        objects retruned by the Data Broker
    filename : string
        path to a new or existing HDF5 file
    """
    if isinstance(headers, Header):
        headers = [headers]
    with h5py.File(filename) as f:
        for header in headers:
            header = dict(header)
            try:
                descriptors = header.pop('descriptors')
            except KeyError:
                warnings.warn("Header with uid {header.uid} contains no "
                              "data.".format(header), UserWarning)
                continue
            top_group_name = header['start']['uid']
            group = f.create_group(top_group_name)
            _safe_attrs_assignment(group, header)
            for i, descriptor in enumerate(descriptors):
                # make sure it's a dictionary and trim any spurious keys
                descriptor = dict(descriptor)
                descriptor.pop('_name', None)

                desc_group = group.create_group(descriptor['uid'])

                data_keys = descriptor.pop('data_keys')
                _safe_attrs_assignment(desc_group, descriptor)

                events = list(get_events_generator(descriptor=descriptor))
                event_times = [e['time'] for e in events]
                desc_group.create_dataset('time', data=event_times,
                                          compression='gzip', fletcher32=True)
                data_group = desc_group.create_group('data')
                ts_group = desc_group.create_group('timestamps')
                [fill_event(e) for e in events]
                for key, value in data_keys.items():
                    value = dict(value)
                    timestamps = [e['timestamps'][key] for e in events]
                    ts_group.create_dataset(key, data=timestamps,
                                            compression='gzip',
                                            fletcher32=True)
                    data = [e['data'][key] for e in events]
                    dataset = data_group.create_dataset(
                        key, data=data, compression='gzip', fletcher32=True)
                    # Put contents of this data key (source, etc.)
                    # into an attribute on the associated data set.
                    _safe_attrs_assignment(dataset, dict(value))
예제 #7
0
def test_bulk_insert():
    num = 50
    rs, e_desc, data_keys = setup_syn()
    all_data = syn_data(data_keys, num)

    mdsc.bulk_insert_events(e_desc, all_data, validate=False)
    mdsc.insert_run_stop(rs, ttime.time(), uid=str(uuid.uuid4()))

    ev_gen = mdsc.get_events_generator(e_desc)

    for ret, expt in zip(ev_gen, all_data):
        assert ret['descriptor']['uid'] == e_desc
        for k in ['data', 'timestamps', 'time', 'uid', 'seq_num']:
            assert ret[k] == expt[k]
예제 #8
0
def test_bulk_insert():
    num = 50
    rs, e_desc, data_keys = setup_syn()
    all_data = syn_data(data_keys, num)

    mdsc.bulk_insert_events(e_desc, all_data, validate=False)
    mdsc.insert_run_stop(rs, ttime.time(), uid=str(uuid.uuid4()))

    ev_gen = mdsc.get_events_generator(e_desc)

    for ret, expt in zip(ev_gen, all_data):
        assert_equal(ret['descriptor']['uid'], e_desc)
        for k in ['data', 'timestamps', 'time', 'uid', 'seq_num']:
            assert_equal(ret[k], expt[k])
예제 #9
0
def get_events(headers, fields=None, fill=True):
    """
    Get Events from given run(s).

    Parameters
    ----------
    headers : Header or iterable of Headers
        The headers to fetch the events for
    fields : list, optional
        whitelist of field names of interest; if None, all are returned
    fill : bool, optional
        Whether externally-stored data should be filled in. Defaults to True

    Yields
    ------
    event : Event
        The event, optionally with non-scalar data filled in
    """
    # A word about the 'fields' argument:
    # Notice that we assume that the same field name cannot occur in
    # more than one descriptor. We could relax this assumption, but
    # we current enforce it in bluesky, so it is safe for now.
    try:
        headers.items()
    except AttributeError:
        pass
    else:
        headers = [headers]

    if fields is None:
        fields = []
    fields = set(fields)

    for header in headers:
        descriptors = find_descriptors(header['start']['uid'])
        for descriptor in descriptors:
            all_fields = set(descriptor['data_keys'])
            if fields:
                discard_fields = all_fields - fields
            else:
                discard_fields = []
            if discard_fields == all_fields:
                continue
            for event in get_events_generator(descriptor):
                for field in discard_fields:
                    del event.data[field]
                    del event.timestamps[field]
                if fill:
                    fill_event(event)
                yield event
예제 #10
0
def test_spec_to_document(sf):
    map = {
        'start': insert_run_start,
        'stop': insert_run_stop,
        'descriptor': insert_descriptor,
        'event': insert_event
    }
    start_uids = list()
    for document_name, document in spec_to_document(sf, validate=True):
        document = dict(document)
        del document['_name']
        if not isinstance(document_name, str):
            document_name = document_name.name
        # insert the documents
        if document_name == 'start':
            document['beamline_id'] = 'test'
            start_uids.append(document['uid'])
        map[document_name](**document)

    # make sure we are not trying to add duplicates
    assert len(start_uids) == len(set(start_uids))

    # smoketest the retrieval
    hdrs = []
    for uid in start_uids:
        hdr = db[uid]
        # make sure we only get one back
        assert isinstance(hdr, Header)
        hdrs.append(hdr)

    # make sure we are not getting duplicates back out
    hdr_uids = [hdr.start.uid for hdr in hdrs]
    assert len(hdr_uids) == len(set(hdr_uids))
    if isinstance(sf, Specscan):
        sf = [sf]
    if isinstance(sf, str):
        sf = Specfile(sf)
    for hdr, specscan in zip(hdrs, sf):
        for descriptor in hdr.descriptors:
            ev = list(get_events_generator(descriptor))
            if descriptor.get('name') == 'baseline':
                # we better only have one baseline event
                assert len(ev) == 1
            else:
                assert len(specscan.scan_data) == len(ev)
예제 #11
0
def test_spec_to_document(sf):
    map = {
        'start': insert_run_start,
        'stop': insert_run_stop,
        'descriptor': insert_descriptor,
        'event': insert_event
    }
    start_uids = list()
    for document_name, document in spec_to_document(sf, validate=True):
        document = dict(document)
        del document['_name']
        if not isinstance(document_name, str):
            document_name = document_name.name
        # insert the documents
        if document_name == 'start':
            document['beamline_id'] = 'test'
            start_uids.append(document['uid'])
        map[document_name](**document)

    # make sure we are not trying to add duplicates
    assert len(start_uids) == len(set(start_uids))

    # smoketest the retrieval
    hdrs = []
    for uid in start_uids:
        hdr = db[uid]
        # make sure we only get one back
        assert isinstance(hdr, Header)
        hdrs.append(hdr)

    # make sure we are not getting duplicates back out
    hdr_uids = [hdr.start.uid for hdr in hdrs]
    assert len(hdr_uids) == len(set(hdr_uids))
    if isinstance(sf, Specscan):
        sf = [sf]
    if isinstance(sf, str):
        sf = Specfile(sf)
    for hdr, specscan in zip(hdrs, sf):
        for descriptor in hdr.descriptors:
            ev = list(get_events_generator(descriptor))
            if descriptor.get('name') == 'baseline':
                # we better only have one baseline event
                assert len(ev) == 1
            else:
                assert len(specscan.scan_data) == len(ev)