def test_stack_data(mock_fxe_raw_run): test_run = RunDirectory(mock_fxe_raw_run) tid, data = test_run.train_from_id(10000, devices=[('*/DET/*', 'image.data')]) comb = stack_data(data, 'image.data') assert comb.shape == (128, 1, 16, 256, 256)
def agipd_frame(mock_run): run = RunDirectory(mock_run) print(run.instrument_sources) print(run.keys_for_source('SPB_DET_AGIPD1M-1/DET/10CH0:xtdf')) tid, train_data = run.select('*/DET/*', 'image.data').train_from_index(0) ary = stack_detector_data(train_data, 'image.data') return ary[0]
def test_stack_detector_data_stackview(mock_fxe_raw_run): test_run = RunDirectory(mock_fxe_raw_run) tid, data = test_run.train_from_id(10000, devices=[('*/DET/*', 'image.data')]) # Three variants of missing data: # 1. Source missing del data['FXE_DET_LPD1M-1/DET/3CH0:xtdf'] # 2. Key missing del data['FXE_DET_LPD1M-1/DET/7CH0:xtdf']['image.data'] # 3. Empty array missing = ['FXE_DET_LPD1M-1/DET/{}CH0:xtdf'.format(m) for m in (1, 5, 9, 15)] for module in missing: data[module]['image.data'] = np.zeros((0, 1, 256, 256), dtype=np.uint16) comb = stack_detector_data(data, 'image.data', fillvalue=22, real_array=False) assert comb.shape == (128, 1, 16, 256, 256) assert not (comb[:, :, 0] == 22).any() # Control assert (comb[:, :, 3] == 22).all() # Source missing assert (comb[:, :, 7] == 22).all() # Key missing assert (comb[:, :, 5] == 22).all() # Empty array # Slice across all modules pulse = comb[0, 0] assert pulse.shape == (16, 256, 256) assert not (pulse[0] == 22).any() assert (pulse[3] == 22).all() assert (pulse[7] == 22).all() assert (pulse[5] == 22).all() pulse_arr = pulse.asarray() assert pulse_arr.shape == (16, 256, 256) assert pulse_arr.max() == 22 assert pulse_arr.min() == 0
def test_select_by(mock_spb_raw_run): run = RunDirectory(mock_spb_raw_run) am0 = run['SPB_DET_AGIPD1M-1/DET/0CH0:xtdf', 'image.data'] subrun = run.select(am0) assert subrun.all_sources == {am0.source} assert subrun.keys_for_source(am0.source) == {am0.key}
def test_merge_detector(mock_fxe_raw_run, mock_fxe_control_data, mock_spb_proc_run): with RunDirectory(mock_fxe_raw_run) as run: for tid, data in _iter_trains(run, merge_detector=True): assert 'FXE_DET_LPD1M-1/DET/APPEND' in data assert 'FXE_DET_LPD1M-1/DET/0CH0:xtdf' not in data shape = data['FXE_DET_LPD1M-1/DET/APPEND']['image.data'].shape assert shape == (128, 1, 16, 256, 256) break for tid, data in _iter_trains(run): assert 'FXE_DET_LPD1M-1/DET/0CH0:xtdf' in data shape = data['FXE_DET_LPD1M-1/DET/0CH0:xtdf']['image.data'].shape assert shape == (128, 1, 256, 256) break with H5File(mock_fxe_control_data) as run: for tid, data in _iter_trains(run, merge_detector=True): assert frozenset(data) == run.select_trains(by_id[[tid ]]).all_sources break with RunDirectory(mock_spb_proc_run) as run: for tid, data in _iter_trains(run, merge_detector=True): shape = data['SPB_DET_AGIPD1M-1/DET/APPEND']['image.data'].shape assert shape == (64, 16, 512, 128) shape = data['SPB_DET_AGIPD1M-1/DET/APPEND']['image.gain'].shape assert shape == (64, 16, 512, 128) shape = data['SPB_DET_AGIPD1M-1/DET/APPEND']['image.mask'].shape assert shape == (64, 16, 512, 128) break
def test_run_get_series_select_trains(mock_fxe_raw_run): run = RunDirectory(mock_fxe_raw_run) sel = run.select_trains(by_id[10100:10150]) s = sel.get_series('SA1_XTD2_XGM/DOOCS/MAIN', "beamPosition.iyPos.value") assert isinstance(s, pd.Series) assert len(s) == 50 assert list(s.index) == list(range(10100, 10150))
def test_stack_detector_data_missing(mock_fxe_raw_run): test_run = RunDirectory(mock_fxe_raw_run) tid, data = test_run.train_from_id(10000, devices=[('*/DET/*', 'image.data')]) # Three variants of missing data: # 1. Source missing del data['FXE_DET_LPD1M-1/DET/3CH0:xtdf'] # 2. Key missing del data['FXE_DET_LPD1M-1/DET/7CH0:xtdf']['image.data'] # 3. Empty array missing = [ 'FXE_DET_LPD1M-1/DET/{}CH0:xtdf'.format(m) for m in (1, 5, 9, 15) ] for module in missing: data[module]['image.data'] = np.zeros((0, 1, 256, 256), dtype=np.uint16) comb = stack_detector_data(data, 'image.data', fillvalue=22) assert comb.shape == (128, 1, 16, 256, 256) assert not (comb[:, :, 0] == 22).any() # Control assert (comb[:, :, 3] == 22).all() # Source missing assert (comb[:, :, 7] == 22).all() # Key missing assert (comb[:, :, 5] == 22).all() # Empty array # default fillvalue for int is 0 comb = stack_detector_data(data, 'image.data') assert (comb[:, :, 3] == 0).all() with pytest.raises(ValueError): comb = stack_detector_data(data, 'image.data', fillvalue=np.nan)
def test_train_from_index_fxe_run(mock_fxe_raw_run): run = RunDirectory(mock_fxe_raw_run) _, data = run.train_from_index(479) assert 'FXE_DET_LPD1M-1/DET/15CH0:xtdf' in data assert 'image.data' in data['FXE_DET_LPD1M-1/DET/15CH0:xtdf'] assert 'FXE_XAD_GEC/CAM/CAMERA' in data assert 'firmwareVersion.value' in data['FXE_XAD_GEC/CAM/CAMERA']
def test_iterate_spb_raw_run(mock_spb_raw_run): run = RunDirectory(mock_spb_raw_run) trains_iter = run.trains() tid, data = next(trains_iter) assert tid == 10000 device = 'SPB_IRU_CAM/CAM/SIDEMIC:daqOutput' assert device in data assert data[device]['data.image.pixels'].shape == (1024, 768)
def test_union_raw_proc(mock_spb_raw_run, mock_spb_proc_run): raw_run = RunDirectory(mock_spb_raw_run) proc_run = RunDirectory(mock_spb_proc_run) run = raw_run.deselect('*AGIPD1M*').union(proc_run) assert run.all_sources == (raw_run.all_sources | proc_run.all_sources) if raw_run.run_metadata()['dataFormatVersion'] != '0.5': assert run.is_single_run
def test_run_get_array_empty(mock_fxe_raw_run): run = RunDirectory(mock_fxe_raw_run) arr = run.get_array('FXE_XAD_GEC/CAM/CAMERA_NODATA:daqOutput', 'data.image.pixels') assert isinstance(arr, DataArray) assert arr.dims[0] == 'trainId' assert arr.shape == (0, 255, 1024)
def test_read_fxe_raw_run_selective(mock_fxe_raw_run): run = RunDirectory(mock_fxe_raw_run, include='*DA*') assert run.train_ids == list(range(10000, 10480)) assert 'SA1_XTD2_XGM/DOOCS/MAIN' in run.control_sources assert 'FXE_DET_LPD1M-1/DET/0CH0:xtdf' not in run.detector_sources run = RunDirectory(mock_fxe_raw_run, include='*LPD*') assert run.train_ids == list(range(10000, 10480)) assert 'SA1_XTD2_XGM/DOOCS/MAIN' not in run.control_sources assert 'FXE_DET_LPD1M-1/DET/0CH0:xtdf' in run.detector_sources
def test_iterate_fxe_run(mock_fxe_raw_run): run = RunDirectory(mock_fxe_raw_run) trains_iter = run.trains() tid, data = next(trains_iter) assert tid == 10000 assert 'FXE_DET_LPD1M-1/DET/15CH0:xtdf' in data assert 'image.data' in data['FXE_DET_LPD1M-1/DET/15CH0:xtdf'] assert 'FXE_XAD_GEC/CAM/CAMERA' in data assert 'firmwareVersion.value' in data['FXE_XAD_GEC/CAM/CAMERA']
def test_iterate_run_glob_devices(mock_fxe_raw_run): run = RunDirectory(mock_fxe_raw_run) trains_iter = run.trains([("*/DET/*", "image.data")]) tid, data = next(trains_iter) assert tid == 10000 assert 'FXE_DET_LPD1M-1/DET/15CH0:xtdf' in data assert 'image.data' in data['FXE_DET_LPD1M-1/DET/15CH0:xtdf'] assert 'detector.data' not in data['FXE_DET_LPD1M-1/DET/15CH0:xtdf'] assert 'FXE_XAD_GEC/CAM/CAMERA' not in data
def test_stack_detector_data_jungfrau(mock_jungfrau_run): run = RunDirectory(mock_jungfrau_run) _, data = run.select('*JF4M/DET/*', 'data.adc').train_from_index(0) comb = stack_detector_data(data, 'data.adc', modules=8, pattern=r'/DET/JNGFR(\d+)', starts_at=1) assert comb.shape == (16, 8, 512, 1024)
def test_run_get_array(mock_fxe_raw_run): run = RunDirectory(mock_fxe_raw_run) arr = run.get_array('SA1_XTD2_XGM/DOOCS/MAIN:output', 'data.intensityTD', extra_dims=['pulse']) assert isinstance(arr, DataArray) assert arr.dims == ('trainId', 'pulse') assert arr.shape == (480, 1000) assert arr.coords['trainId'][0] == 10000
def test_run_get_array_multiple_per_train(mock_fxe_raw_run): run = RunDirectory(mock_fxe_raw_run) sel = run.select_trains(np.s_[:2]) arr = sel.get_array('FXE_DET_LPD1M-1/DET/6CH0:xtdf', 'image.data', roi=np.s_[:, 10:20, 20:40]) assert isinstance(arr, DataArray) assert arr.shape == (256, 1, 10, 20) np.testing.assert_array_equal(arr.coords['trainId'], np.repeat([10000, 10001], 128))
def test_stack_detector_data_type_error(mock_fxe_raw_run): test_run = RunDirectory(mock_fxe_raw_run) tid, data = test_run.train_from_id(10000, devices=[('*/DET/*', 'image.data')]) module = 'FXE_DET_LPD1M-1/DET/3CH0:xtdf' data[module]['image.data'] = data[module]['image.data'].astype(np.float32) with pytest.raises(ValueError) as excinfo: comb = stack_detector_data(data, 'image.data') assert "dtype('float32')" in str(excinfo.value)
def test_iterate_select_trains(mock_fxe_raw_run): run = RunDirectory(mock_fxe_raw_run) tids = [tid for (tid, _) in run.trains(train_range=by_id[10004:10006])] assert tids == [10004, 10005] tids = [tid for (tid, _) in run.trains(train_range=by_id[:10003])] assert tids == [10000, 10001, 10002] # Overlap with start of run tids = [tid for (tid, _) in run.trains(train_range=by_id[9000:10003])] assert tids == [10000, 10001, 10002] # Overlap with end of run tids = [tid for (tid, _) in run.trains(train_range=by_id[10478:10500])] assert tids == [10478, 10479] # Not overlapping with catch_warnings(record=True) as w: tids = [tid for (tid, _) in run.trains(train_range=by_id[9000:9050])] assert tids == [] assert 'before' in str(w[0].message) with catch_warnings(record=True) as w: tids = [tid for (tid, _) in run.trains(train_range=by_id[10500:10550])] assert tids == [] assert 'after' in str(w[0].message) tids = [tid for (tid, _) in run.trains(train_range=by_index[4:6])] assert tids == [10004, 10005]
def test_iterate_select_trains(mock_fxe_raw_run): run = RunDirectory(mock_fxe_raw_run) tids = [tid for (tid, _) in run.trains(train_range=by_id[10004:10006])] assert tids == [10004, 10005] tids = [tid for (tid, _) in run.trains(train_range=by_id[:10003])] assert tids == [10000, 10001, 10002] # Overlap with start of run tids = [tid for (tid, _) in run.trains(train_range=by_id[9000:10003])] assert tids == [10000, 10001, 10002] # Overlap with end of run tids = [tid for (tid, _) in run.trains(train_range=by_id[10478:10500])] assert tids == [10478, 10479] # Not overlapping with pytest.raises(ValueError) as excinfo: list(run.trains(train_range=by_id[9000:9050])) assert 'before' in str(excinfo.value) with pytest.raises(ValueError) as excinfo: list(run.trains(train_range=by_id[10500:10550])) assert 'after' in str(excinfo.value) tids = [tid for (tid, _) in run.trains(train_range=by_index[4:6])] assert tids == [10004, 10005]
def test_stack_detector_data_wrong_shape(mock_fxe_raw_run): test_run = RunDirectory(mock_fxe_raw_run) tid, data = test_run.train_from_id(10000, devices=[('*/DET/*', 'image.data')]) misshaped = ['FXE_DET_LPD1M-1/DET/{}CH0:xtdf'.format(m) for m in (0, 15)] for module in misshaped: data[module]['image.data'] = np.zeros((128, 1, 512, 128), dtype=np.uint16) with pytest.raises(ValueError) as excinfo: comb = stack_detector_data(data, 'image.data') assert '(128, 1, 512, 128)' in str(excinfo.value)
def test_run_get_dask_array(mock_fxe_raw_run): import dask.array as da run = RunDirectory(mock_fxe_raw_run) arr = run.get_dask_array( 'SA1_XTD2_XGM/DOOCS/MAIN:output', 'data.intensityTD', ) assert isinstance(arr, da.Array) assert arr.shape == (480, 1000) assert arr.dtype == np.float32
def test_run_get_virtual_dataset_filename(mock_fxe_raw_run, tmpdir): run = RunDirectory(mock_fxe_raw_run) path = str(tmpdir / 'test-vds.h5') ds = run.get_virtual_dataset('FXE_DET_LPD1M-1/DET/6CH0:xtdf', 'image.data', filename=path) assert_isfile(path) assert ds.file.filename == path assert isinstance(ds, h5py.Dataset) assert ds.is_virtual assert ds.shape == (61440, 1, 256, 256)
def test_split_trains(mock_fxe_raw_run): run = RunDirectory(mock_fxe_raw_run) assert len(run.train_ids) == 480 chunks = list(run.split_trains(3)) assert len(chunks) == 3 assert {len(c.train_ids) for c in chunks} == {160} chunks = list(run.split_trains(4, trains_per_part=100)) assert len(chunks) == 5 assert {len(c.train_ids) for c in chunks} == {96}
def test_run_get_dataframe(mock_fxe_raw_run): run = RunDirectory(mock_fxe_raw_run) df = run.get_dataframe(fields=[("*_XGM/*", "*.i[xy]Pos*")]) assert len(df.columns) == 4 assert "SA1_XTD2_XGM/DOOCS/MAIN/beamPosition.ixPos" in df.columns df2 = run.get_dataframe(fields=[("*_XGM/*", "*.i[xy]Pos*")], timestamps=True) assert len(df2.columns) == 8 assert "SA1_XTD2_XGM/DOOCS/MAIN/beamPosition.ixPos" in df2.columns assert "SA1_XTD2_XGM/DOOCS/MAIN/beamPosition.ixPos.timestamp" in df2.columns
def test_stack_detector_data_extra_mods(mock_fxe_raw_run): test_run = RunDirectory(mock_fxe_raw_run) tid, data = test_run.train_from_id(10000, devices=[('*/DET/*', 'image.data')]) data.setdefault( 'FXE_DET_LPD1M-1/DET/16CH0:xtdf', {'image.data': np.zeros((128, 1, 256, 256), dtype=np.uint16)}, ) with pytest.raises(IndexError) as excinfo: comb = stack_detector_data(data, 'image.data') assert "16" in str(excinfo.value)
def test_select(mock_fxe_raw_run): run = RunDirectory(mock_fxe_raw_run) assert 'SPB_XTD9_XGM/DOOCS/MAIN' in run.control_sources sel = run.select('*/DET/*', 'image.pulseId') assert 'SPB_XTD9_XGM/DOOCS/MAIN' not in sel.control_sources assert 'FXE_DET_LPD1M-1/DET/0CH0:xtdf' in sel.instrument_sources _, data = sel.train_from_id(10000) for source, source_data in data.items(): print(source) assert set(source_data.keys()) == {'image.pulseId', 'metadata'}
def test_run_get_dask_array_labelled(mock_fxe_raw_run): import dask.array as da run = RunDirectory(mock_fxe_raw_run) arr = run.get_dask_array('SA1_XTD2_XGM/DOOCS/MAIN:output', 'data.intensityTD', labelled=True) assert isinstance(arr, DataArray) assert isinstance(arr.data, da.Array) assert arr.dims == ('trainId', 'dim_0') assert arr.shape == (480, 1000) assert arr.coords['trainId'][0] == 10000
def test_run_get_virtual_dataset(mock_fxe_raw_run): run = RunDirectory(mock_fxe_raw_run) ds = run.get_virtual_dataset('FXE_DET_LPD1M-1/DET/6CH0:xtdf', 'image.data') assert isinstance(ds, h5py.Dataset) assert ds.is_virtual assert ds.shape == (61440, 1, 256, 256) # Across two sequence files ds = run.get_virtual_dataset('FXE_XAD_GEC/CAM/CAMERA:daqOutput', 'data.image.pixels') assert isinstance(ds, h5py.Dataset) assert ds.is_virtual assert ds.shape == (480, 255, 1024)
def test_read_spb_proc_run(mock_spb_proc_run): run = RunDirectory(mock_spb_proc_run) #Test for calib data assert len(run.files) == 16 # only 16 detector modules for calib data assert run.train_ids == list(range(10000, 10064)) #64 trains tid, data = next(run.trains()) device = 'SPB_DET_AGIPD1M-1/DET/15CH0:xtdf' assert tid == 10000 for prop in ('image.gain', 'image.mask', 'image.data'): assert prop in data[device] assert 'u1' == data[device]['image.gain'].dtype assert 'u4' == data[device]['image.mask'].dtype assert 'f4' == data[device]['image.data'].dtype run.info() # Smoke test