Example #1
0
    def test_AD_round_trip(self):

        # Retrieve the data.
        for i, datum_id in enumerate(self.datum_ids):
            data = retrieve(datum_id)
            known_data = i * np.ones((1, 2, 2))
            assert_array_equal(data, known_data)
Example #2
0
    def test_AD_round_trip(self):

        # Retrieve the data.
        for i, datum_id in zip(np.arange(len(self.datum_ids)), self.datum_ids):
            data = retrieve(datum_id)
            known_data = 2 * i
            assert_array_equal(data, known_data)
Example #3
0
    def __init__(self, headers, name):
        """
        Load images from a detector for given Header(s).

        Parameters
        ----------
        headers : Header or list of Headers
        name : str
            alias (data key) of a detector

        Example
        -------
        >>> header = DataBroker[-1]
        >>> images = Images(header, 'my_detector_lightfield')
        >>> for image in images:
                # do something
        """
        all_events = DataBroker.fetch_events(headers, fill=False)
        # TODO Make this smarter by only grabbing events from the relevant
        # descriptors. For now, we will do it this way to take advantage of
        # header validation logical happening in fetch_events.
        events = [event for event in all_events if name in event.data.keys()]
        self._datum_uids = [event.data[name] for event in events]
        self._len = len(self._datum_uids)
        example_frame = retrieve(self._datum_uids[0])
        self._dtype = example_frame.dtype
        self._shape = example_frame.shape
Example #4
0
def test_insert_funcs(func):
    shape = (25, 32)
    mod_ids = func('syn-mod', shape, 10)

    for j, r_id in enumerate(mod_ids):
        data = retrieve(r_id)
        known_data = np.mod(np.arange(np.prod(shape)), j + 1).reshape(shape)
        assert_array_equal(data, known_data)
Example #5
0
    def test_maps_spectrum_round_trip(self):
        sn = np.sin(self.th)

        with handler_context({'hdf5_maps': HDFM}):
            for eid, sc in zip(self.eids_spectrum, self.scale):
                print(eid)
                data = retrieve(eid)
                assert_array_equal(data, sc * sn)
Example #6
0
def fill_event(event):
    """
    Populate events with externally stored data.
    """
    is_external = _inspect_descriptor(event.descriptor)
    for data_key, value in six.iteritems(event.data):
        if is_external[data_key]:
            # Retrieve a numpy array from filestore
            event.data[data_key] = retrieve(value)
Example #7
0
def test_custom():
    test_path = os.path.join(BASE_PATH, str(uuid.uuid4()) + '.npy')
    dd = np.random.rand(500, 500)
    with fs_write.NpyWriter(test_path, resource_kwargs={'mmap_mode': 'r'}) as f:
        eid = f.add_data(dd)
    with fsa.handler_context({'npy': fs_read.NpyHandler}):
        ret = fsa.retrieve(eid)

    assert_array_equal(dd, ret)
Example #8
0
def fill_event(event):
    """
    Populate events with externally stored data.
    """
    is_external = _inspect_descriptor(event.descriptor)
    for data_key, (value, timestamp) in event.data.items():
        if is_external[data_key]:
            # Retrieve a numpy array from filestore
            event.data[data_key][0] = retrieve(value)
Example #9
0
def load_image_filestore(datum_id):
    if datum_id is None:
        raise IOError("Image doesn't exist yet")

    try:
        return np.asarray(fsapi.retrieve(datum_id)).squeeze()
    except Exception as ex:
        print('Filestore load failed (datum={}): ({}) {}'
              ''.format(datum_id, ex.__class__.__name__, ex))
        raise
Example #10
0
    def event(self, event_document):
        event_time = datetime.fromtimestamp(event_document['time']).time()
        rounded_time = str(event_time)[:10]
        row = [event_document['seq_num'], rounded_time]
        for field in self.fields:
            val = event_document['data'].get(field, '')
            if field in self._filestore_keys:
                try:
                    import filestore.api as fsapi
                    val = fsapi.retrieve(val)
                except Exception as exc:
                    warnings.warn(
                        UserWarning, "Attempt to read {0} raised {1}"
                        "".format(field, exc))
                    val = 'Not Available'
            if isinstance(val, np.ndarray) or isinstance(val, list):
                val = np.sum(np.asarray(val))
            try:
                val = format_num(val,
                                 max_len=self.data_field_width,
                                 pre=self.max_pre_decimal,
                                 post=self.max_post_decimal)
            except Exception:
                val = str(val)[:self.data_field_width]
            row.append(val)
        self.table.add_row(row)

        if self.rowwise:
            # Print the last row of data only.
            # [-1] is the bottom border
            print(str(self.table).split('\n')[-2])
            # only print header intermittently for rowwise table printing
            if self.num_events_since_last_header >= self.print_header_interval:
                self._print_table_header()
                self.num_events_since_last_header = 0
            self.num_events_since_last_header += 1
        else:
            # print the whole table
            print(self.table)

        sys.stdout.flush()
Example #11
0
    def event(self, event_document):
        event_time = datetime.fromtimestamp(event_document['time']).time()
        rounded_time = str(event_time)[:10]
        row = [event_document['seq_num'], rounded_time]
        for field in self.fields:
            val = event_document['data'].get(field, '')
            if field in self._filestore_keys:
                try:
                    import filestore.api as fsapi
                    val = fsapi.retrieve(val)
                except Exception as exc:
                    warnings.warn(UserWarning, "Attempt to read {0} raised {1}"
                                  "".format(field, exc))
                    val = 'Not Available'
            if isinstance(val, np.ndarray) or isinstance(val, list):
                val = np.sum(np.asarray(val))
            try:
                val = format_num(val,
                                 max_len=self.data_field_width,
                                 pre=self.max_pre_decimal,
                                 post=self.max_post_decimal)
            except Exception:
                val = str(val)[:self.data_field_width]
            row.append(val)
        self.table.add_row(row)

        if self.rowwise:
            # Print the last row of data only.
            # [-1] is the bottom border
            print(str(self.table).split('\n')[-2])
            # only print header intermittently for rowwise table printing
            if self.num_events_since_last_header >= self.print_header_interval:
                self._print_table_header()
                self.num_events_since_last_header = 0
            self.num_events_since_last_header += 1
        else:
            # print the whole table
            print(self.table)

        sys.stdout.flush()
Example #12
0
    def __init__(self, headers, name):
        """
        Load images from a detector for given Header(s).

        Parameters
        ----------
        headers : Header or list of Headers
        name : str
            field name (data key) of a detector

        Example
        -------
        >>> header = DataBroker[-1]
        >>> images = Images(header, 'my_detector_lightfield')
        >>> for image in images:
                # do something
        """
        events = get_events(headers, [name], fill=False)
        self._datum_uids = [event.data[name] for event in events]
        self._len = len(self._datum_uids)
        example_frame = retrieve(self._datum_uids[0])
        self._dtype = example_frame.dtype
        self._shape = example_frame.shape
Example #13
0
def _test_retrieve_data(event):
    uid = event['data']['xrf_spectrum']
    data = retrieve(uid)
    assert_equal(data.size, 20)
    assert_not_equal(np.sum(data), 0)
Example #14
0
 def test_maps_plane_round_trip(self):
     base = self.scale.reshape(self.N, self.M)
     with handler_context({'hdf5_planes': HDFE}):
         for eid, v in zip(self.eids_planes, np.sin(self.th)):
             data = retrieve(eid)
             assert_array_equal(data, base * v)
Example #15
0
def test_non_exist():
    with pytest.raises(DatumNotFound):
        retrieve('aardvark')
Example #16
0
def _npsave_helper(dd, base_path):
    eid = fs_write.save_ndarray(dd, base_path)
    with fsa.handler_context({'npy': fs_read.NpyHandler}):
        ret = fsa.retrieve(eid)

    assert_array_equal(dd, ret)
Example #17
0
def get_table(headers, fields=None, fill=True, convert_times=True):
    """
    Make a table (pandas.DataFrame) from given run(s).

    Parameters
    ----------
    headers : Header or iterable of Headers
        The headers to fetch the events for
    fields : list, optional
        whitelist of field names of interest; if None, all are returned
    fill : bool, optional
        Whether externally-stored data should be filled in. Defaults to True
    convert_times : bool, optional
        Whether to convert times from float (seconds since 1970) to
        numpy datetime64, using pandas. True by default.

    Returns
    -------
    table : pandas.DataFrame
    """
    # A word about the 'fields' argument:
    # Notice that we assume that the same field name cannot occur in
    # more than one descriptor. We could relax this assumption, but
    # we current enforce it in bluesky, so it is safe for now.
    try:
        headers.items()
    except AttributeError:
        pass
    else:
        headers = [headers]

    if fields is None:
        fields = []
    fields = set(fields)

    dfs = []
    for header in headers:
        descriptors = find_descriptors(header['start']['uid'])
        for descriptor in descriptors:
            all_fields = set(descriptor['data_keys'])
            if fields:
                discard_fields = all_fields - fields
            else:
                discard_fields = []
            if discard_fields == all_fields:
                continue
            is_external = _inspect_descriptor(descriptor)

            payload = get_events_table(descriptor)
            descriptor, data, seq_nums, times, uids, timestamps = payload
            df = pd.DataFrame(index=seq_nums)
            if convert_times:
                times = pd.to_datetime(
                    pd.Series(times), unit='s', utc=True).dt.tz_localize(TZ)
            df['time'] = times
            for field, values in six.iteritems(data):
                if field in discard_fields:
                    logger.debug('Discarding field %s', field)
                    continue
                if is_external[field] and fill:
                    logger.debug('filling data for %s', field)
                    # TODO someday we will have bulk retrieve in FS
                    values = [fs.retrieve(value) for value in values]
                df[field] = values
            dfs.append(df)
    if dfs:
        return pd.concat(dfs)
    else:
        # edge case: no data
        return pd.DataFrame()
Example #18
0
 def test_retrieval(self):
     with handler_context({'npy_FRAMEWISE': NpyFrameWise}):
         for i, datum_id in enumerate(self.datum_ids):
             data = retrieve(datum_id)
             known_data = i * np.ones((9, 8))
             assert_array_equal(data, known_data)
Example #19
0
 def event(self, doc):
     uid = doc['data'][self.field]
     data = fsapi.retrieve(uid)
     self.cs.update_image(data)
     self.cs._fig.canvas.draw_idle()
Example #20
0
 def event(self, doc):
     import filestore.api as fsapi
     uid = doc['data'][self.field]
     data = fsapi.retrieve(uid)
     self.cs.update_image(data)
     self.cs._fig.canvas.draw_idle()
Example #21
0
 def get_frame(self, i):
     img = retrieve(self._datum_uids[i])
     return Frame(img, frame_no=i)
Example #22
0
        plt.show()

hdrs = db(
    run_folder='/mnt/bulk-data/research_data/USC_beamtime/APS_March_2016/S1/temp_exp')

hdr = hdrs[0]
print(hdr['start']['run_folder'], hdr['start']['uid'])

# Get calibrations
if not hdr['start']['is_calibration']:
    cals = [db[u]['start']['poni'][0] for u in
            hdr['start']['calibration']]
else:
    cals = [p for p in hdr['start']['poni']]

geos = [retrieve(p) for p in cals]
cal_dists = np.asarray(
    [g.dist for g in geos]) * 100  # convert to meters

events = get_events(hdr)
ev0 = events.next()
detz = ev0['data']['detz']
cal_idx = np.argmin((detz - cal_dists) ** 2)
geo = geos[cal_idx]
img = retrieve(ev0['data']['img'])
# Correct for polarization
img /= geo.polarization(img.shape, .95)

r = geo.rArray(img.shape)
q = geo.qArray(img.shape) / 10  # pyFAI works in nm**-1, we want A**-1
fq = geo.qArray(img.shape).ravel()
Example #23
0
 def event(self, doc):
     uid = doc['data'][self.field]
     data = fsapi.retrieve(uid)
     self.cs.update_image(data)
     self.cs._fig.canvas.draw()
     self.cs._fig.canvas.flush_events()
from filestore.api import retrieve
from xpd_workflow.mask_tools import *
from sidewinder_spec.utils.handlers import *
import matplotlib.pyplot as plt
from filestore.api import db_connect as fs_db_connect
from metadatastore.api import db_connect as mds_db_connect
fs_db_connect(**{'database': 'data-processing-dev', 'host': 'localhost', 'port': 27017})
mds_db_connect(**{'database': 'data-processing-dev', 'host': 'localhost', 'port': 27017})

# Get headers of interest
hdrs = [db[-1]]
for hdr in hdrs:
    time_dept_bg = True

    # Get calibrations
    geos = [retrieve(p) for p in hdr['start']['poni']]
    cal_dists = np.asarray(
        [g.dist for g in geos]) * 10  # pyFAI reports in meters
    # Get starting masks
    # start_masks = [retrieve(p) for p in hdr['start']['mask']]
    for event in get_events(hdr):
        # Pull relevent data into local vars
        data = event['data']
        img = data['img']
        detz = data['detz']

        # Find the correct calibration file, it's the one with the dist close
        # to the recorded detector dist
        cal_idx = np.argmin((detz - cal_dists) ** 2)
        geo = geos[cal_idx]
        # start_mask = start_masks[cal_idx]