def test_stack_data(mock_fxe_raw_run):
test_run = RunDirectory(mock_fxe_raw_run)
tid, data = test_run.train_from_id(10000,
devices=[('*/DET/*', 'image.data')])
comb = stack_data(data, 'image.data')
assert comb.shape == (128, 1, 16, 256, 256)
def agipd_frame(mock_run):
run = RunDirectory(mock_run)
print(run.instrument_sources)
print(run.keys_for_source('SPB_DET_AGIPD1M-1/DET/10CH0:xtdf'))
tid, train_data = run.select('*/DET/*', 'image.data').train_from_index(0)
ary = stack_detector_data(train_data, 'image.data')
return ary[0]
def test_stack_detector_data_stackview(mock_fxe_raw_run):
test_run = RunDirectory(mock_fxe_raw_run)
tid, data = test_run.train_from_id(10000, devices=[('*/DET/*', 'image.data')])
# Three variants of missing data:
# 1. Source missing
del data['FXE_DET_LPD1M-1/DET/3CH0:xtdf']
# 2. Key missing
del data['FXE_DET_LPD1M-1/DET/7CH0:xtdf']['image.data']
# 3. Empty array
missing = ['FXE_DET_LPD1M-1/DET/{}CH0:xtdf'.format(m) for m in (1, 5, 9, 15)]
for module in missing:
data[module]['image.data'] = np.zeros((0, 1, 256, 256), dtype=np.uint16)
comb = stack_detector_data(data, 'image.data', fillvalue=22, real_array=False)
assert comb.shape == (128, 1, 16, 256, 256)
assert not (comb[:, :, 0] == 22).any() # Control
assert (comb[:, :, 3] == 22).all() # Source missing
assert (comb[:, :, 7] == 22).all() # Key missing
assert (comb[:, :, 5] == 22).all() # Empty array
# Slice across all modules
pulse = comb[0, 0]
assert pulse.shape == (16, 256, 256)
assert not (pulse[0] == 22).any()
assert (pulse[3] == 22).all()
assert (pulse[7] == 22).all()
assert (pulse[5] == 22).all()
pulse_arr = pulse.asarray()
assert pulse_arr.shape == (16, 256, 256)
assert pulse_arr.max() == 22
assert pulse_arr.min() == 0
def test_select_by(mock_spb_raw_run):
run = RunDirectory(mock_spb_raw_run)
am0 = run['SPB_DET_AGIPD1M-1/DET/0CH0:xtdf', 'image.data']
subrun = run.select(am0)
assert subrun.all_sources == {am0.source}
assert subrun.keys_for_source(am0.source) == {am0.key}
def test_merge_detector(mock_fxe_raw_run, mock_fxe_control_data,
mock_spb_proc_run):
with RunDirectory(mock_fxe_raw_run) as run:
for tid, data in _iter_trains(run, merge_detector=True):
assert 'FXE_DET_LPD1M-1/DET/APPEND' in data
assert 'FXE_DET_LPD1M-1/DET/0CH0:xtdf' not in data
shape = data['FXE_DET_LPD1M-1/DET/APPEND']['image.data'].shape
assert shape == (128, 1, 16, 256, 256)
break
for tid, data in _iter_trains(run):
assert 'FXE_DET_LPD1M-1/DET/0CH0:xtdf' in data
shape = data['FXE_DET_LPD1M-1/DET/0CH0:xtdf']['image.data'].shape
assert shape == (128, 1, 256, 256)
break
with H5File(mock_fxe_control_data) as run:
for tid, data in _iter_trains(run, merge_detector=True):
assert frozenset(data) == run.select_trains(by_id[[tid
]]).all_sources
break
with RunDirectory(mock_spb_proc_run) as run:
for tid, data in _iter_trains(run, merge_detector=True):
shape = data['SPB_DET_AGIPD1M-1/DET/APPEND']['image.data'].shape
assert shape == (64, 16, 512, 128)
shape = data['SPB_DET_AGIPD1M-1/DET/APPEND']['image.gain'].shape
assert shape == (64, 16, 512, 128)
shape = data['SPB_DET_AGIPD1M-1/DET/APPEND']['image.mask'].shape
assert shape == (64, 16, 512, 128)
break
def test_run_get_series_select_trains(mock_fxe_raw_run):
run = RunDirectory(mock_fxe_raw_run)
sel = run.select_trains(by_id[10100:10150])
s = sel.get_series('SA1_XTD2_XGM/DOOCS/MAIN', "beamPosition.iyPos.value")
assert isinstance(s, pd.Series)
assert len(s) == 50
assert list(s.index) == list(range(10100, 10150))
def test_stack_detector_data_missing(mock_fxe_raw_run):
test_run = RunDirectory(mock_fxe_raw_run)
tid, data = test_run.train_from_id(10000,
devices=[('*/DET/*', 'image.data')])
# Three variants of missing data:
# 1. Source missing
del data['FXE_DET_LPD1M-1/DET/3CH0:xtdf']
# 2. Key missing
del data['FXE_DET_LPD1M-1/DET/7CH0:xtdf']['image.data']
# 3. Empty array
missing = [
'FXE_DET_LPD1M-1/DET/{}CH0:xtdf'.format(m) for m in (1, 5, 9, 15)
]
for module in missing:
data[module]['image.data'] = np.zeros((0, 1, 256, 256),
dtype=np.uint16)
comb = stack_detector_data(data, 'image.data', fillvalue=22)
assert comb.shape == (128, 1, 16, 256, 256)
assert not (comb[:, :, 0] == 22).any() # Control
assert (comb[:, :, 3] == 22).all() # Source missing
assert (comb[:, :, 7] == 22).all() # Key missing
assert (comb[:, :, 5] == 22).all() # Empty array
# default fillvalue for int is 0
comb = stack_detector_data(data, 'image.data')
assert (comb[:, :, 3] == 0).all()
with pytest.raises(ValueError):
comb = stack_detector_data(data, 'image.data', fillvalue=np.nan)
def test_train_from_index_fxe_run(mock_fxe_raw_run):
run = RunDirectory(mock_fxe_raw_run)
_, data = run.train_from_index(479)
assert 'FXE_DET_LPD1M-1/DET/15CH0:xtdf' in data
assert 'image.data' in data['FXE_DET_LPD1M-1/DET/15CH0:xtdf']
assert 'FXE_XAD_GEC/CAM/CAMERA' in data
assert 'firmwareVersion.value' in data['FXE_XAD_GEC/CAM/CAMERA']
def test_iterate_spb_raw_run(mock_spb_raw_run):
run = RunDirectory(mock_spb_raw_run)
trains_iter = run.trains()
tid, data = next(trains_iter)
assert tid == 10000
device = 'SPB_IRU_CAM/CAM/SIDEMIC:daqOutput'
assert device in data
assert data[device]['data.image.pixels'].shape == (1024, 768)
def test_union_raw_proc(mock_spb_raw_run, mock_spb_proc_run):
raw_run = RunDirectory(mock_spb_raw_run)
proc_run = RunDirectory(mock_spb_proc_run)
run = raw_run.deselect('*AGIPD1M*').union(proc_run)
assert run.all_sources == (raw_run.all_sources | proc_run.all_sources)
if raw_run.run_metadata()['dataFormatVersion'] != '0.5':
assert run.is_single_run
def test_run_get_array_empty(mock_fxe_raw_run):
run = RunDirectory(mock_fxe_raw_run)
arr = run.get_array('FXE_XAD_GEC/CAM/CAMERA_NODATA:daqOutput',
'data.image.pixels')
assert isinstance(arr, DataArray)
assert arr.dims[0] == 'trainId'
assert arr.shape == (0, 255, 1024)
def test_read_fxe_raw_run_selective(mock_fxe_raw_run):
run = RunDirectory(mock_fxe_raw_run, include='*DA*')
assert run.train_ids == list(range(10000, 10480))
assert 'SA1_XTD2_XGM/DOOCS/MAIN' in run.control_sources
assert 'FXE_DET_LPD1M-1/DET/0CH0:xtdf' not in run.detector_sources
run = RunDirectory(mock_fxe_raw_run, include='*LPD*')
assert run.train_ids == list(range(10000, 10480))
assert 'SA1_XTD2_XGM/DOOCS/MAIN' not in run.control_sources
assert 'FXE_DET_LPD1M-1/DET/0CH0:xtdf' in run.detector_sources
def test_iterate_fxe_run(mock_fxe_raw_run):
run = RunDirectory(mock_fxe_raw_run)
trains_iter = run.trains()
tid, data = next(trains_iter)
assert tid == 10000
assert 'FXE_DET_LPD1M-1/DET/15CH0:xtdf' in data
assert 'image.data' in data['FXE_DET_LPD1M-1/DET/15CH0:xtdf']
assert 'FXE_XAD_GEC/CAM/CAMERA' in data
assert 'firmwareVersion.value' in data['FXE_XAD_GEC/CAM/CAMERA']
def test_iterate_run_glob_devices(mock_fxe_raw_run):
run = RunDirectory(mock_fxe_raw_run)
trains_iter = run.trains([("*/DET/*", "image.data")])
tid, data = next(trains_iter)
assert tid == 10000
assert 'FXE_DET_LPD1M-1/DET/15CH0:xtdf' in data
assert 'image.data' in data['FXE_DET_LPD1M-1/DET/15CH0:xtdf']
assert 'detector.data' not in data['FXE_DET_LPD1M-1/DET/15CH0:xtdf']
assert 'FXE_XAD_GEC/CAM/CAMERA' not in data
def test_stack_detector_data_jungfrau(mock_jungfrau_run):
run = RunDirectory(mock_jungfrau_run)
_, data = run.select('*JF4M/DET/*', 'data.adc').train_from_index(0)
comb = stack_detector_data(data,
'data.adc',
modules=8,
pattern=r'/DET/JNGFR(\d+)',
starts_at=1)
assert comb.shape == (16, 8, 512, 1024)
def test_run_get_array(mock_fxe_raw_run):
run = RunDirectory(mock_fxe_raw_run)
arr = run.get_array('SA1_XTD2_XGM/DOOCS/MAIN:output',
'data.intensityTD',
extra_dims=['pulse'])
assert isinstance(arr, DataArray)
assert arr.dims == ('trainId', 'pulse')
assert arr.shape == (480, 1000)
assert arr.coords['trainId'][0] == 10000
def test_run_get_array_multiple_per_train(mock_fxe_raw_run):
run = RunDirectory(mock_fxe_raw_run)
sel = run.select_trains(np.s_[:2])
arr = sel.get_array('FXE_DET_LPD1M-1/DET/6CH0:xtdf',
'image.data',
roi=np.s_[:, 10:20, 20:40])
assert isinstance(arr, DataArray)
assert arr.shape == (256, 1, 10, 20)
np.testing.assert_array_equal(arr.coords['trainId'],
np.repeat([10000, 10001], 128))
def test_stack_detector_data_type_error(mock_fxe_raw_run):
test_run = RunDirectory(mock_fxe_raw_run)
tid, data = test_run.train_from_id(10000, devices=[('*/DET/*', 'image.data')])
module = 'FXE_DET_LPD1M-1/DET/3CH0:xtdf'
data[module]['image.data'] = data[module]['image.data'].astype(np.float32)
with pytest.raises(ValueError) as excinfo:
comb = stack_detector_data(data, 'image.data')
assert "dtype('float32')" in str(excinfo.value)
def test_iterate_select_trains(mock_fxe_raw_run):
run = RunDirectory(mock_fxe_raw_run)
tids = [tid for (tid, _) in run.trains(train_range=by_id[10004:10006])]
assert tids == [10004, 10005]
tids = [tid for (tid, _) in run.trains(train_range=by_id[:10003])]
assert tids == [10000, 10001, 10002]
# Overlap with start of run
tids = [tid for (tid, _) in run.trains(train_range=by_id[9000:10003])]
assert tids == [10000, 10001, 10002]
# Overlap with end of run
tids = [tid for (tid, _) in run.trains(train_range=by_id[10478:10500])]
assert tids == [10478, 10479]
# Not overlapping
with catch_warnings(record=True) as w:
tids = [tid for (tid, _) in run.trains(train_range=by_id[9000:9050])]
assert tids == []
assert 'before' in str(w[0].message)
with catch_warnings(record=True) as w:
tids = [tid for (tid, _) in run.trains(train_range=by_id[10500:10550])]
assert tids == []
assert 'after' in str(w[0].message)
tids = [tid for (tid, _) in run.trains(train_range=by_index[4:6])]
assert tids == [10004, 10005]
def test_iterate_select_trains(mock_fxe_raw_run):
run = RunDirectory(mock_fxe_raw_run)
tids = [tid for (tid, _) in run.trains(train_range=by_id[10004:10006])]
assert tids == [10004, 10005]
tids = [tid for (tid, _) in run.trains(train_range=by_id[:10003])]
assert tids == [10000, 10001, 10002]
# Overlap with start of run
tids = [tid for (tid, _) in run.trains(train_range=by_id[9000:10003])]
assert tids == [10000, 10001, 10002]
# Overlap with end of run
tids = [tid for (tid, _) in run.trains(train_range=by_id[10478:10500])]
assert tids == [10478, 10479]
# Not overlapping
with pytest.raises(ValueError) as excinfo:
list(run.trains(train_range=by_id[9000:9050]))
assert 'before' in str(excinfo.value)
with pytest.raises(ValueError) as excinfo:
list(run.trains(train_range=by_id[10500:10550]))
assert 'after' in str(excinfo.value)
tids = [tid for (tid, _) in run.trains(train_range=by_index[4:6])]
assert tids == [10004, 10005]
def test_stack_detector_data_wrong_shape(mock_fxe_raw_run):
test_run = RunDirectory(mock_fxe_raw_run)
tid, data = test_run.train_from_id(10000, devices=[('*/DET/*', 'image.data')])
misshaped = ['FXE_DET_LPD1M-1/DET/{}CH0:xtdf'.format(m) for m in (0, 15)]
for module in misshaped:
data[module]['image.data'] = np.zeros((128, 1, 512, 128), dtype=np.uint16)
with pytest.raises(ValueError) as excinfo:
comb = stack_detector_data(data, 'image.data')
assert '(128, 1, 512, 128)' in str(excinfo.value)
def test_run_get_dask_array(mock_fxe_raw_run):
import dask.array as da
run = RunDirectory(mock_fxe_raw_run)
arr = run.get_dask_array(
'SA1_XTD2_XGM/DOOCS/MAIN:output',
'data.intensityTD',
)
assert isinstance(arr, da.Array)
assert arr.shape == (480, 1000)
assert arr.dtype == np.float32
def test_run_get_virtual_dataset_filename(mock_fxe_raw_run, tmpdir):
run = RunDirectory(mock_fxe_raw_run)
path = str(tmpdir / 'test-vds.h5')
ds = run.get_virtual_dataset('FXE_DET_LPD1M-1/DET/6CH0:xtdf',
'image.data',
filename=path)
assert_isfile(path)
assert ds.file.filename == path
assert isinstance(ds, h5py.Dataset)
assert ds.is_virtual
assert ds.shape == (61440, 1, 256, 256)
def test_split_trains(mock_fxe_raw_run):
run = RunDirectory(mock_fxe_raw_run)
assert len(run.train_ids) == 480
chunks = list(run.split_trains(3))
assert len(chunks) == 3
assert {len(c.train_ids) for c in chunks} == {160}
chunks = list(run.split_trains(4, trains_per_part=100))
assert len(chunks) == 5
assert {len(c.train_ids) for c in chunks} == {96}
def test_run_get_dataframe(mock_fxe_raw_run):
run = RunDirectory(mock_fxe_raw_run)
df = run.get_dataframe(fields=[("*_XGM/*", "*.i[xy]Pos*")])
assert len(df.columns) == 4
assert "SA1_XTD2_XGM/DOOCS/MAIN/beamPosition.ixPos" in df.columns
df2 = run.get_dataframe(fields=[("*_XGM/*", "*.i[xy]Pos*")],
timestamps=True)
assert len(df2.columns) == 8
assert "SA1_XTD2_XGM/DOOCS/MAIN/beamPosition.ixPos" in df2.columns
assert "SA1_XTD2_XGM/DOOCS/MAIN/beamPosition.ixPos.timestamp" in df2.columns
def test_stack_detector_data_extra_mods(mock_fxe_raw_run):
test_run = RunDirectory(mock_fxe_raw_run)
tid, data = test_run.train_from_id(10000, devices=[('*/DET/*', 'image.data')])
data.setdefault(
'FXE_DET_LPD1M-1/DET/16CH0:xtdf',
{'image.data': np.zeros((128, 1, 256, 256), dtype=np.uint16)},
)
with pytest.raises(IndexError) as excinfo:
comb = stack_detector_data(data, 'image.data')
assert "16" in str(excinfo.value)
def test_select(mock_fxe_raw_run):
run = RunDirectory(mock_fxe_raw_run)
assert 'SPB_XTD9_XGM/DOOCS/MAIN' in run.control_sources
sel = run.select('*/DET/*', 'image.pulseId')
assert 'SPB_XTD9_XGM/DOOCS/MAIN' not in sel.control_sources
assert 'FXE_DET_LPD1M-1/DET/0CH0:xtdf' in sel.instrument_sources
_, data = sel.train_from_id(10000)
for source, source_data in data.items():
print(source)
assert set(source_data.keys()) == {'image.pulseId', 'metadata'}
def test_run_get_dask_array_labelled(mock_fxe_raw_run):
import dask.array as da
run = RunDirectory(mock_fxe_raw_run)
arr = run.get_dask_array('SA1_XTD2_XGM/DOOCS/MAIN:output',
'data.intensityTD',
labelled=True)
assert isinstance(arr, DataArray)
assert isinstance(arr.data, da.Array)
assert arr.dims == ('trainId', 'dim_0')
assert arr.shape == (480, 1000)
assert arr.coords['trainId'][0] == 10000
def test_run_get_virtual_dataset(mock_fxe_raw_run):
run = RunDirectory(mock_fxe_raw_run)
ds = run.get_virtual_dataset('FXE_DET_LPD1M-1/DET/6CH0:xtdf', 'image.data')
assert isinstance(ds, h5py.Dataset)
assert ds.is_virtual
assert ds.shape == (61440, 1, 256, 256)
# Across two sequence files
ds = run.get_virtual_dataset('FXE_XAD_GEC/CAM/CAMERA:daqOutput',
'data.image.pixels')
assert isinstance(ds, h5py.Dataset)
assert ds.is_virtual
assert ds.shape == (480, 255, 1024)
def test_read_spb_proc_run(mock_spb_proc_run):
run = RunDirectory(mock_spb_proc_run) #Test for calib data
assert len(run.files) == 16 # only 16 detector modules for calib data
assert run.train_ids == list(range(10000, 10064)) #64 trains
tid, data = next(run.trains())
device = 'SPB_DET_AGIPD1M-1/DET/15CH0:xtdf'
assert tid == 10000
for prop in ('image.gain', 'image.mask', 'image.data'):
assert prop in data[device]
assert 'u1' == data[device]['image.gain'].dtype
assert 'u4' == data[device]['image.mask'].dtype
assert 'f4' == data[device]['image.data'].dtype
run.info() # Smoke test
