def test_train_from_index_fxe_run(mock_fxe_run):
run = RunDirectory(mock_fxe_run)
_, data = run.train_from_index(479)
assert 'FXE_DET_LPD1M-1/DET/15CH0:xtdf' in data
assert 'image.data' in data['FXE_DET_LPD1M-1/DET/15CH0:xtdf']
assert 'FXE_XAD_GEC/CAM/CAMERA' in data
assert 'firmwareVersion.value' in data['FXE_XAD_GEC/CAM/CAMERA']
def test_run_info(capsys):
test_run = RunDirectory(RUNPATH)
test_run.info()
captured = capsys.readouterr()
print(captured)
assert captured[0] == ('Run information\n'
'\tDuration: 0:08:57.200000\n'
'\tFirst train ID: 1472806005\n'
'\tLast train ID: 1472811377\n'
'\t# of trains: 542\n'
'\n'
'Devices\n'
'\tInstruments\n'
'\t- SPB_DET_AGIPD1M-1/DET/0CH0:xtdf\n'
'\t- SPB_DET_AGIPD1M-1/DET/10CH0:xtdf\n'
'\t- SPB_DET_AGIPD1M-1/DET/1CH0:xtdf\n'
'\t- SPB_DET_AGIPD1M-1/DET/2CH0:xtdf\n'
'\t- SPB_DET_AGIPD1M-1/DET/3CH0:xtdf\n'
'\t- SPB_DET_AGIPD1M-1/DET/4CH0:xtdf\n'
'\t- SPB_DET_AGIPD1M-1/DET/5CH0:xtdf\n'
'\t- SPB_DET_AGIPD1M-1/DET/6CH0:xtdf\n'
'\t- SPB_DET_AGIPD1M-1/DET/7CH0:xtdf\n'
'\t- SPB_DET_AGIPD1M-1/DET/8CH0:xtdf\n'
'\t- SPB_DET_AGIPD1M-1/DET/9CH0:xtdf\n'
'\tControls\n'
'\t-\n')
def __init__(self, run_folder):
"""Initialization.
:param str run_folder: full path of the run folder.
"""
self._run = RunDirectory(run_folder)
self._sources = {
'MONO': 'SA3_XTD10_MONO/MDL/PHOTON_ENERGY',
'XGM': 'SCS_BLU_XGM/XGM/DOOCS',
'XGM_OUTPUT': 'SCS_BLU_XGM/XGM/DOOCS:output',
'SA3_XGM': 'SA3_XTD10_XGM/XGM/DOOCS',
'SA3_XGM_OUTPUT': 'SA3_XTD10_XGM/XGM/DOOCS:output'
}
# get the DataFrame for XGM control data
self._xgm_df = self._run.get_dataframe(fields=[(self._sources['XGM'],
'*value')])
self._xgm_df.rename(columns=lambda x: x.split('/')[-1], inplace=True)
self._sa3_xgm_df = self._run.get_dataframe(
fields=[(self._sources['SA3_XGM'], '*value')])
self._sa3_xgm_df.rename(columns=lambda x: x.split('/')[-1],
inplace=True)
# get the DataFrame for SoftMono control data
self._mono_df = self._run.get_dataframe(fields=[(self._sources['MONO'],
'*value')])
self._mono_df.rename(columns=lambda x: x.split('/')[-1], inplace=True)
self._photon_energies = None # photon energies for each pulse
self._I0 = None
self._I1 = OrderedDict()
self._data = None # pulse-resolved data in DataFrame
def test_stack_detector_data(mock_fxe_run):
test_run = RunDirectory(mock_fxe_run)
tid, data = test_run.train_from_id(10000,
devices=[('*/DET/*', 'image.data')])
comb = stack_detector_data(data, 'image.data')
assert comb.shape == (128, 1, 16, 256, 256)
def test_run_get_series_select_trains(mock_fxe_raw_run):
run = RunDirectory(mock_fxe_raw_run)
sel = run.select_trains(by_id[10100:10150])
s = sel.get_series('SA1_XTD2_XGM/DOOCS/MAIN', "beamPosition.iyPos.value")
assert isinstance(s, pd.Series)
assert len(s) == 50
assert list(s.index) == list(range(10100, 10150))
def test_stack_detector_data_missing(mock_fxe_raw_run):
test_run = RunDirectory(mock_fxe_raw_run)
tid, data = test_run.train_from_id(10000,
devices=[('*/DET/*', 'image.data')])
# Three variants of missing data:
# 1. Source missing
del data['FXE_DET_LPD1M-1/DET/3CH0:xtdf']
# 2. Key missing
del data['FXE_DET_LPD1M-1/DET/7CH0:xtdf']['image.data']
# 3. Empty array
missing = [
'FXE_DET_LPD1M-1/DET/{}CH0:xtdf'.format(m) for m in (1, 5, 9, 15)
]
for module in missing:
data[module]['image.data'] = np.zeros((0, 1, 256, 256),
dtype=np.uint16)
comb = stack_detector_data(data, 'image.data', fillvalue=22)
assert comb.shape == (128, 1, 16, 256, 256)
assert not (comb[:, :, 0] == 22).any() # Control
assert (comb[:, :, 3] == 22).all() # Source missing
assert (comb[:, :, 7] == 22).all() # Key missing
assert (comb[:, :, 5] == 22).all() # Empty array
def test_stack_detector_data():
test_run = RunDirectory(RUNPATH)
tid, data = test_run.train_from_id(1472810853)
comb = stack_detector_data(data, 'image.data', only='AGIPD1M-1')
print(comb.shape)
assert comb.shape == (60, 16, 512, 128)
np.testing.assert_equal(comb[:, 0, ...], data['SPB_DET_AGIPD1M-1/DET/0CH0:xtdf']['image.data'])
def test_run_get_array_empty(mock_fxe_run):
run = RunDirectory(mock_fxe_run)
arr = run.get_array('FXE_XAD_GEC/CAM/CAMERA_NODATA:daqOutput',
'data.image.pixels')
assert isinstance(arr, DataArray)
assert arr.dims[0] == 'trainId'
assert arr.shape == (0, 255, 1024)
def test_iterate_spb_raw_run(mock_spb_raw_run):
run = RunDirectory(mock_spb_raw_run)
trains_iter = run.trains()
tid, data = next(trains_iter)
assert tid == 10000
device = 'SPB_IRU_CAM/CAM/SIDEMIC:daqOutput'
assert device in data
assert data[device]['data.image.pixels'].shape == (1024, 768)
def test_stack_data_2():
test_run = RunDirectory(RUNPATH)
tid, data = test_run.train_from_id(1472810853)
skip = ['SPB_DET_AGIPD1M-1/DET/0CH0:xtdf', 'SPB_DET_AGIPD1M-1/DET/9CH0:xtdf']
comb = stack_data(data, 'image.data', axis=0, xcept=skip)
print(comb.shape)
assert comb.shape == (8, 60, 512, 128)
np.testing.assert_equal(comb[0, ...], data['SPB_DET_AGIPD1M-1/DET/1CH0:xtdf']['image.data'])
def test_iterate_run_glob_devices(mock_fxe_run):
run = RunDirectory(mock_fxe_run)
trains_iter = run.trains([("*/DET/*", "image.data")])
tid, data = next(trains_iter)
assert tid == 10000
assert 'FXE_DET_LPD1M-1/DET/15CH0:xtdf' in data
assert 'image.data' in data['FXE_DET_LPD1M-1/DET/15CH0:xtdf']
assert 'detector.data' not in data['FXE_DET_LPD1M-1/DET/15CH0:xtdf']
assert 'FXE_XAD_GEC/CAM/CAMERA' not in data
def test_iterate_fxe_run(mock_fxe_raw_run):
run = RunDirectory(mock_fxe_raw_run)
trains_iter = run.trains()
tid, data = next(trains_iter)
assert tid == 10000
assert 'FXE_DET_LPD1M-1/DET/15CH0:xtdf' in data
assert 'image.data' in data['FXE_DET_LPD1M-1/DET/15CH0:xtdf']
assert 'FXE_XAD_GEC/CAM/CAMERA' in data
assert 'firmwareVersion.value' in data['FXE_XAD_GEC/CAM/CAMERA']
def test_read_fxe_raw_run_selective(mock_fxe_raw_run):
run = RunDirectory(mock_fxe_raw_run, include='*DA*')
assert run.train_ids == list(range(10000, 10480))
assert 'SA1_XTD2_XGM/DOOCS/MAIN' in run.control_sources
assert 'FXE_DET_LPD1M-1/DET/0CH0:xtdf' not in run.detector_sources
run = RunDirectory(mock_fxe_raw_run, include='*LPD*')
assert run.train_ids == list(range(10000, 10480))
assert 'SA1_XTD2_XGM/DOOCS/MAIN' not in run.control_sources
assert 'FXE_DET_LPD1M-1/DET/0CH0:xtdf' in run.detector_sources
def test_run_get_array(mock_fxe_run):
run = RunDirectory(mock_fxe_run)
arr = run.get_array('SA1_XTD2_XGM/DOOCS/MAIN:output',
'data.intensityTD',
extra_dims=['pulse'])
assert isinstance(arr, DataArray)
assert arr.dims == ('trainId', 'pulse')
assert arr.shape == (480, 1000)
assert arr.coords['trainId'][0] == 10000
def test_run_get_array_multiple_per_train(mock_fxe_raw_run):
run = RunDirectory(mock_fxe_raw_run)
sel = run.select_trains(by_index[:2])
arr = sel.get_array('FXE_DET_LPD1M-1/DET/6CH0:xtdf',
'image.data',
roi=by_index[:, 10:20, 20:40])
assert isinstance(arr, DataArray)
assert arr.shape == (256, 1, 10, 20)
np.testing.assert_array_equal(arr.coords['trainId'],
np.repeat([10000, 10001], 128))
def serve_files(path,
port,
fast_devices=None,
require_all=False,
repeat_stream=True,
**kwargs):
"""Stream data from files through a TCP socket.
Parameters
----------
path: str
Path to the HDF5 file or file folder.
port: int
Local TCP port to bind socket to.
slow_devices: list of tuples
[('src', 'prop')]
fast_devices: list of tuples
[('src', 'prop')]
require_all: bool
If set to True, will stream only trainIDs that has data
corresponding to keys specified in fast_devices.
Default: False
repeat_stream: bool
If set to True, will continue streaming when trains()
iterator is empty. Trainids will be monotonically increasing.
Default: False
"""
try:
corr_data = RunDirectory(path)
num_trains = len(corr_data.train_ids)
except Exception as ex:
print(repr(ex))
return
streamer = ZMQStreamer(port, **kwargs)
streamer.start()
counter = 0
repeat_stream = False
while True:
for tid, train_data in corr_data.trains(devices=fast_devices,
require_all=require_all):
# loop over corrected DataCollection
if train_data:
# Generate fake meta data with monotically increasing
# trainids only after the actual trains in corrected data
meta = generate_meta(train_data.keys(), tid +
counter) if counter > 0 else None
streamer.feed(train_data, metadata=meta)
if not repeat_stream:
break
# increase the counter by total number of trains in a run
counter += num_trains
streamer.stop()
def test_iterate_select_trains(mock_fxe_run):
run = RunDirectory(mock_fxe_run)
tids = [tid for (tid, _) in run.trains(train_range=by_id[10004:10006])]
assert tids == [10004, 10005]
tids = [tid for (tid, _) in run.trains(train_range=by_id[:10003])]
assert tids == [10000, 10001, 10002]
# Overlap with start of run
tids = [tid for (tid, _) in run.trains(train_range=by_id[9000:10003])]
assert tids == [10000, 10001, 10002]
# Overlap with end of run
tids = [tid for (tid, _) in run.trains(train_range=by_id[10478:10500])]
assert tids == [10478, 10479]
# Not overlapping
with pytest.raises(ValueError) as excinfo:
list(run.trains(train_range=by_id[9000:9050]))
assert 'before' in str(excinfo.value)
with pytest.raises(ValueError) as excinfo:
list(run.trains(train_range=by_id[10500:10550]))
assert 'after' in str(excinfo.value)
tids = [tid for (tid, _) in run.trains(train_range=by_index[4:6])]
assert tids == [10004, 10005]
def test_run_get_array_select_trains(mock_fxe_raw_run):
run = RunDirectory(mock_fxe_raw_run)
sel = run.select_trains(by_id[10100:10150])
arr = sel.get_array('SA1_XTD2_XGM/DOOCS/MAIN:output',
'data.intensityTD',
extra_dims=['pulse'])
assert isinstance(arr, DataArray)
assert arr.dims == ('trainId', 'pulse')
assert arr.shape == (50, 1000)
assert arr.coords['trainId'][0] == 10100
def test_run_get_virtual_dataset_filename(mock_fxe_raw_run, tmpdir):
run = RunDirectory(mock_fxe_raw_run)
path = str(tmpdir / 'test-vds.h5')
ds = run.get_virtual_dataset('FXE_DET_LPD1M-1/DET/6CH0:xtdf',
'image.data',
filename=path)
assert_isfile(path)
assert ds.file.filename == path
assert isinstance(ds, h5py.Dataset)
assert ds.is_virtual
assert ds.shape == (61440, 1, 256, 256)
def test_run_get_dask_array(mock_fxe_raw_run):
import dask.array as da
run = RunDirectory(mock_fxe_raw_run)
arr = run.get_dask_array(
'SA1_XTD2_XGM/DOOCS/MAIN:output',
'data.intensityTD',
)
assert isinstance(arr, da.Array)
assert arr.shape == (480, 1000)
assert arr.dtype == np.float32
def test_stack_detector_data_type_error(mock_fxe_raw_run):
test_run = RunDirectory(mock_fxe_raw_run)
tid, data = test_run.train_from_id(10000,
devices=[('*/DET/*', 'image.data')])
module = 'FXE_DET_LPD1M-1/DET/3CH0:xtdf'
data[module]['image.data'] = data[module]['image.data'].astype(np.float32)
with pytest.raises(ValueError) as excinfo:
comb = stack_detector_data(data, 'image.data')
assert "dtype('float32')" in str(excinfo.value)
def test_run_get_dataframe(mock_fxe_run):
run = RunDirectory(mock_fxe_run)
df = run.get_dataframe(fields=[("*_XGM/*", "*.i[xy]Pos*")])
assert len(df.columns) == 4
assert "SA1_XTD2_XGM/DOOCS/MAIN/beamPosition.ixPos" in df.columns
df2 = run.get_dataframe(fields=[("*_XGM/*", "*.i[xy]Pos*")],
timestamps=True)
assert len(df2.columns) == 8
assert "SA1_XTD2_XGM/DOOCS/MAIN/beamPosition.ixPos" in df2.columns
assert "SA1_XTD2_XGM/DOOCS/MAIN/beamPosition.ixPos.timestamp" in df2.columns
def test_select(mock_fxe_raw_run):
run = RunDirectory(mock_fxe_raw_run)
assert 'SPB_XTD9_XGM/DOOCS/MAIN' in run.control_sources
sel = run.select('*/DET/*', 'image.pulseId')
assert 'SPB_XTD9_XGM/DOOCS/MAIN' not in sel.control_sources
assert 'FXE_DET_LPD1M-1/DET/0CH0:xtdf' in sel.instrument_sources
_, data = sel.train_from_id(10000)
for source, source_data in data.items():
print(source)
assert set(source_data.keys()) == {'image.pulseId', 'metadata'}
def load_data(run, proposal, exp, VERBOSE=False):
ddir = "/gpfs/exfel/exp/SPB/{}/{}/raw/{}/".format(exp, proposal, run)
print("Loading data from: {}".format(ddir))
data = RunDirectory(ddir)
if VERBOSE:
data.info()
return data
def test_stack_detector_data_2():
test_run = RunDirectory(RUNPATH)
tid, data = test_run.train_from_id(1472810853)
skip = ['SPB_DET_AGIPD1M-1/DET/{}CH0:xtdf'.format(i) for i in range(3,8)]
comb = stack_detector_data(data, 'image.data', axis=0, only='AGIPD1M-1', xcept=skip)
print(comb.shape)
assert comb.shape == (16, 60, 512, 128)
for i in (0, 1, 2, 8, 9, 10):
np.testing.assert_equal(comb[i, ...], data['SPB_DET_AGIPD1M-1/DET/{}CH0:xtdf'.format(i)]['image.data'])
for i in (3, 4, 5, 6, 7, 11, 12, 13, 14, 15):
np.testing.assert_equal(comb[i, ...], np.full((60, 512, 128), np.nan, dtype=np.float32))
def test_read_spb_proc_run(mock_spb_proc_run):
run = RunDirectory(mock_spb_proc_run) #Test for calib data
assert len(run.files) == 16 # only 16 detector modules for calib data
assert run.train_ids == list(range(10000, 10064)) #64 trains
tid, data = next(run.trains())
device = 'SPB_DET_AGIPD1M-1/DET/15CH0:xtdf'
assert tid == 10000
for prop in ('image.gain', 'image.mask', 'image.data'):
assert prop in data[device]
assert 'u1' == data[device]['image.gain'].dtype
assert 'u4' == data[device]['image.mask'].dtype
assert 'f4' == data[device]['image.data'].dtype
run.info() # Smoke test
def test_run_get_virtual_dataset(mock_fxe_raw_run):
run = RunDirectory(mock_fxe_raw_run)
ds = run.get_virtual_dataset('FXE_DET_LPD1M-1/DET/6CH0:xtdf', 'image.data')
assert isinstance(ds, h5py.Dataset)
assert ds.is_virtual
assert ds.shape == (61440, 1, 256, 256)
# Across two sequence files
ds = run.get_virtual_dataset('FXE_XAD_GEC/CAM/CAMERA:daqOutput',
'data.image.pixels')
assert isinstance(ds, h5py.Dataset)
assert ds.is_virtual
assert ds.shape == (480, 255, 1024)
def test_stack_detector_data_extra_mods(mock_fxe_raw_run):
test_run = RunDirectory(mock_fxe_raw_run)
tid, data = test_run.train_from_id(10000,
devices=[('*/DET/*', 'image.data')])
data.setdefault(
'FXE_DET_LPD1M-1/DET/16CH0:xtdf',
{'image.data': np.zeros((128, 1, 256, 256), dtype=np.uint16)},
)
with pytest.raises(IndexError) as excinfo:
comb = stack_detector_data(data, 'image.data')
assert "16" in str(excinfo.value)
def test_stack_detector_data_wrong_pulses(mock_fxe_raw_run):
test_run = RunDirectory(mock_fxe_raw_run)
tid, data = test_run.train_from_id(10000,
devices=[('*/DET/*', 'image.data')])
misshaped = ['FXE_DET_LPD1M-1/DET/{}CH0:xtdf'.format(m) for m in (12, 13)]
for module in misshaped:
data[module]['image.data'] = np.zeros((64, 1, 256, 256),
dtype=np.uint16)
with pytest.raises(ValueError) as excinfo:
comb = stack_detector_data(data, 'image.data')
assert '(64, 1, 256, 256)' in str(excinfo.value)
def test_deselect(mock_fxe_raw_run):
run = RunDirectory(mock_fxe_raw_run)
xtd9_xgm = 'SPB_XTD9_XGM/DOOCS/MAIN'
assert xtd9_xgm in run.control_sources
sel = run.deselect('*_XGM/DOOCS*')
assert xtd9_xgm not in sel.control_sources
assert 'FXE_DET_LPD1M-1/DET/0CH0:xtdf' in sel.instrument_sources
sel = run.deselect('*_XGM/DOOCS*', '*.ixPos')
assert xtd9_xgm in sel.control_sources
assert 'beamPosition.ixPos.value' not in sel.selection[xtd9_xgm]
assert 'beamPosition.iyPos.value' in sel.selection[xtd9_xgm]
