def __init__(self, filename, source_ind=0):
if not karabo_installed:
raise IOError('karabo_data is required to load karabo h5 files')
self.f = H5File(filename)
self.series_max = len(self.f.train_ids)
self.sources = [
s for s in self.f.instrument_sources if "daqOutput" in s
]
self.current_source = self.sources[source_ind]
def test_open_file(mock_sa3_control_data):
f = H5File(mock_sa3_control_data)
file_access = f.files[0]
assert file_access.format_version in ('0.5', '1.0')
assert 'SA3_XTD10_VAC/TSENS/S30180K' in f.control_sources
if file_access.format_version == '0.5':
assert 'METADATA/dataSourceId' in file_access.file
else:
assert 'METADATA/dataSources/dataSourceId' in file_access.file
def test_file_get_array(mock_fxe_control_data):
with H5File(mock_fxe_control_data) as f:
arr = f.get_array('FXE_XAD_GEC/CAM/CAMERA:daqOutput',
'data.image.pixels')
assert isinstance(arr, DataArray)
assert arr.dims == ('trainId', 'dim_0', 'dim_1')
assert arr.shape == (400, 255, 1024)
assert arr.coords['trainId'][0] == 10000
def test_select_require_all(mock_sa3_control_data, select_str):
# De-select two sources in this example set, which have no trains
# at all, to allow matching trains across all sources with the same
# result.
run = H5File(mock_sa3_control_data) \
.deselect([('SA3_XTD10_MCP/ADC/1:*', '*'),
('SA3_XTD10_IMGFEL/CAM/BEAMVIEW:*', '*')])
subrun = run.select(select_str, require_all=True)
np.testing.assert_array_equal(subrun.train_ids, run.train_ids[1::2])
def test_still_valid_elsewhere(agipd_file_tid_very_high, mock_sa3_control_data):
dc = H5File(
agipd_file_tid_very_high, inc_suspect_trains=False
).union(H5File(mock_sa3_control_data))
assert dc.train_ids == list(range(10000, 10500))
agipd_src = 'SPB_DET_AGIPD1M-1/DET/7CH0:xtdf'
tsens_src = 'SA3_XTD10_VAC/TSENS/S30250K'
sel = dc.select({
agipd_src: {'image.pulseId'},
tsens_src: {'value.value'}
})
assert sel.all_sources == {agipd_src, tsens_src}
_, t1 = sel.train_from_id(10200, flat_keys=True)
assert set(t1) >= {(agipd_src, 'image.pulseId'), (tsens_src, 'value.value')}
_, t2 = sel.train_from_id(10400, flat_keys=True)
assert (agipd_src, 'image.pulseId') not in t2
assert (tsens_src, 'value.value') in t2
tids_from_iter, data_from_iter = [], []
for tid, d in sel.trains(flat_keys=True):
if tid in (10200, 10400):
tids_from_iter.append(tid)
data_from_iter.append(d)
assert tids_from_iter == [10200, 10400]
assert [set(d) for d in data_from_iter] == [set(t1), set(t2)]
# Check that select with require_all respects the valid train filtering:
sel2 = dc.select(agipd_src, require_all=True)
assert len(sel2.train_ids) == 249
dc_inc = H5File(agipd_file_tid_very_high, inc_suspect_trains=True)\
.union(H5File(mock_sa3_control_data))
sel_inc = dc_inc.select(sel)
_, t2_inc = sel_inc.train_from_id(10400, flat_keys=True)
assert set(t2_inc) == set(t1)
sel2_inc = dc_inc.select(agipd_src, require_all=True)
assert len(sel2_inc.train_ids) == 250
def test_file_get_array_missing_trains(mock_sa3_control_data):
with H5File(mock_sa3_control_data) as f:
sel = f.select_trains(by_index[:6])
arr = sel.get_array('SA3_XTD10_IMGFEL/CAM/BEAMVIEW2:daqOutput',
'data.image.dims')
assert isinstance(arr, DataArray)
assert arr.dims == ('trainId', 'dim_0')
assert arr.shape == (3, 2)
np.testing.assert_array_less(arr.coords['trainId'], 10006)
np.testing.assert_array_less(10000, arr.coords['trainId'])
def test_iterate_trains_fxe(mock_fxe_control_data):
with H5File(mock_fxe_control_data) as f:
for train_id, data in islice(f.trains(), 10):
assert train_id in range(10000, 10400)
assert 'SA1_XTD2_XGM/DOOCS/MAIN' in data.keys()
assert 'beamPosition.ixPos.value' in data[
'SA1_XTD2_XGM/DOOCS/MAIN']
assert 'data.image.pixels' in data[
'FXE_XAD_GEC/CAM/CAMERA:daqOutput']
assert 'data.image.pixels' not in data[
'FXE_XAD_GEC/CAM/CAMERA_NODATA:daqOutput']
def test_get_train_bad_device_name(mock_spb_control_data_badname):
# Check that we can handle devices which don't have the standard Karabo
# name structure A/B/C.
with H5File(mock_spb_control_data_badname) as f:
train_id, data = f.train_from_id(10004)
assert train_id == 10004
device = 'SPB_IRU_SIDEMIC_CAM:daqOutput'
assert device in data
assert 'data.image.dims' in data[device]
dims = data[device]['data.image.dims']
assert list(dims) == [1000, 1000]
def test_get_run_value(mock_fxe_control_data):
f = H5File(mock_fxe_control_data)
src = 'FXE_XAD_GEC/CAM/CAMERA'
val = f.get_run_value(src, 'firmwareVersion')
assert isinstance(val, np.int32)
assert f.get_run_value(src, 'firmwareVersion.value') == val
with pytest.raises(SourceNameError):
f.get_run_value(src + '_NONEXIST', 'firmwareVersion')
with pytest.raises(PropertyNameError):
f.get_run_value(src, 'non.existant')
def test_iterate_trains_require_all(mock_sa3_control_data):
with H5File(mock_sa3_control_data) as f:
trains_iter = f.trains(devices=[('*/CAM/BEAMVIEW:daqOutput',
'data.image.dims')],
require_all=True)
tids = [t for (t, _) in trains_iter]
assert tids == []
trains_iter = f.trains(devices=[('*/CAM/BEAMVIEW:daqOutput',
'data.image.dims')],
require_all=False)
tids = [t for (t, _) in trains_iter]
assert tids != []
def test_file_get_array_control_roi(mock_sa3_control_data):
with H5File(mock_sa3_control_data) as f:
sel = f.select_trains(by_index[:6])
arr = sel.get_array(
'SA3_XTD10_VAC/DCTRL/D6_APERT_IN_OK',
'interlock.a1.AActCommand.value',
roi=by_index[:25],
)
assert isinstance(arr, DataArray)
assert arr.shape == (6, 25)
assert arr.coords['trainId'][0] == 10000
def test_select_require_all(mock_sa3_control_data, select_str):
# De-select two sources in this example set, which have no trains
# at all, to allow matching trains across all sources with the same
# result.
run = H5File(mock_sa3_control_data) \
.deselect([('SA3_XTD10_MCP/ADC/1:*', '*'),
('SA3_XTD10_IMGFEL/CAM/BEAMVIEW:*', '*')])
subrun = run.select(select_str, require_all=True)
np.testing.assert_array_equal(subrun.train_ids, run.train_ids[1::2])
# The train IDs are held by ndarrays during this operation, make
# sure it's a list of np.uint64 again.
assert isinstance(subrun.train_ids, list)
assert all([isinstance(x, np.uint64) for x in subrun.train_ids])
def test_single_value(mock_sa3_control_data, monkeypatch):
f = H5File(mock_sa3_control_data)
imager = f['SA3_XTD10_IMGFEL/CAM/BEAMVIEW:daqOutput', 'data.image.pixels']
flux = f['SA3_XTD10_XGM/XGM/DOOCS', 'pulseEnergy.photonFlux']
# Try without data for a source and key.
with pytest.raises(NoDataError):
imager.as_single_value() # FEL imager with no data.
with pytest.raises(NoDataError):
flux[:0].as_single_value() # No data through selection.
# Monkeypatch some actual data into the KeyData object
data = np.arange(flux.shape[0])
monkeypatch.setattr(flux, 'ndarray', lambda: data)
# Try some tolerances that have to fail.
with pytest.raises(ValueError):
flux.as_single_value()
with pytest.raises(ValueError):
flux.as_single_value(atol=1)
with pytest.raises(ValueError):
flux.as_single_value(rtol=0.1)
# Try with large enough tolerances.
assert flux.as_single_value(atol=len(data) / 2) == np.median(data)
assert flux.as_single_value(rtol=0.5,
atol=len(data) / 4) == np.median(data)
assert flux.as_single_value(rtol=1) == np.median(data)
# Other reduction options
assert flux.as_single_value(rtol=1, reduce_by='mean') == np.mean(data)
assert flux.as_single_value(rtol=1, reduce_by=np.mean) == np.mean(data)
assert flux.as_single_value(atol=len(data) - 1, reduce_by='first') == 0
# Try vector data.
intensity = f['SA3_XTD10_XGM/XGM/DOOCS:output', 'data.intensityTD']
data = np.repeat(data, intensity.shape[1]).reshape(-1, intensity.shape[-1])
monkeypatch.setattr(intensity, 'ndarray', lambda: data)
with pytest.raises(ValueError):
intensity.as_single_value()
np.testing.assert_equal(intensity.as_single_value(rtol=1), np.median(data))
def test_drop_empty_trains(mock_sa3_control_data):
f = H5File(mock_sa3_control_data)
beamview = f['SA3_XTD10_IMGFEL/CAM/BEAMVIEW2:daqOutput', 'data.image.dims']
assert len(beamview.train_ids) == 500
a1 = beamview.ndarray()
assert a1.shape == (250, 2)
frame_counts = beamview.data_counts(labelled=False)
assert frame_counts.shape == (500, )
assert frame_counts.min() == 0
beamview_w_data = beamview.drop_empty_trains()
assert len(beamview_w_data.train_ids) == 250
np.testing.assert_array_equal(beamview_w_data.ndarray(), a1)
frame_counts = beamview_w_data.data_counts(labelled=False)
assert frame_counts.shape == (250, )
assert frame_counts.min() == 1
def test_iterate_trains_select_keys(mock_fxe_control_data):
sel = {
'SA1_XTD2_XGM/DOOCS/MAIN': {
'beamPosition.ixPos.value',
'beamPosition.ixPos.timestamp',
}
}
with H5File(mock_fxe_control_data) as f:
for train_id, data in islice(f.trains(devices=sel), 10):
assert train_id in range(10000, 10400)
assert 'SA1_XTD2_XGM/DOOCS/MAIN' in data.keys()
assert 'beamPosition.ixPos.value' in data[
'SA1_XTD2_XGM/DOOCS/MAIN']
assert 'beamPosition.ixPos.timestamp' in data[
'SA1_XTD2_XGM/DOOCS/MAIN']
assert 'beamPosition.iyPos.value' not in data[
'SA1_XTD2_XGM/DOOCS/MAIN']
assert 'SA3_XTD10_VAC/TSENS/S30160K' not in data
def test_write_selected(mock_fxe_raw_run):
with TemporaryDirectory() as td:
new_file = osp.join(td, 'test.h5')
with RunDirectory(mock_fxe_raw_run) as run:
run.select('SPB_XTD9_XGM/*').write(new_file)
assert_isfile(new_file)
with H5File(new_file) as f:
assert f.control_sources == {'SPB_XTD9_XGM/DOOCS/MAIN'}
assert f.instrument_sources == {'SPB_XTD9_XGM/DOOCS/MAIN:output'}
s = f.get_series('SPB_XTD9_XGM/DOOCS/MAIN',
'beamPosition.ixPos.value')
# This should have concatenated the two sequence files (400 + 80)
assert len(s) == 480
a = f.get_array('SPB_XTD9_XGM/DOOCS/MAIN:output',
'data.intensityTD')
assert a.shape == (480, 1000)
def test_file_get_series_instrument(mock_agipd_data):
with H5File(mock_agipd_data) as f:
s = f.get_series('SPB_DET_AGIPD1M-1/DET/7CH0:xtdf', 'header.linkId')
assert isinstance(s, pd.Series)
assert len(s) == 250
assert s.index[0] == 10000
# Multiple readings per train
s2 = f.get_series('SPB_DET_AGIPD1M-1/DET/7CH0:xtdf', 'image.status')
assert isinstance(s2, pd.Series)
assert isinstance(s2.index, pd.MultiIndex)
assert len(s2) == 16000
assert len(s2.loc[10000:10004]) == 5 * 64
sel = f.select_trains(by_index[5:10])
s3 = sel.get_series('SPB_DET_AGIPD1M-1/DET/7CH0:xtdf', 'image.status')
assert isinstance(s3, pd.Series)
assert isinstance(s3.index, pd.MultiIndex)
assert len(s3) == 5 * 64
np.testing.assert_array_equal(s3.index.get_level_values(0),
np.arange(10005, 10010).repeat(64))
def test_file_get_series_instrument(mock_spb_proc_run):
agipd_file = os.path.join(mock_spb_proc_run,
'CORR-R0238-AGIPD07-S00000.h5')
with H5File(agipd_file) as f:
s = f.get_series('SPB_DET_AGIPD1M-1/DET/7CH0:xtdf', 'header.linkId')
assert isinstance(s, pd.Series)
assert len(s) == 64
assert s.index[0] == 10000
# Multiple readings per train
s2 = f.get_series('SPB_DET_AGIPD1M-1/DET/7CH0:xtdf', 'image.pulseId')
assert isinstance(s2, pd.Series)
assert not s2.index.is_unique
assert len(s2) == 64 * 64
assert len(s2.loc[10000:10004]) == 5 * 64
sel = f.select_trains(by_index[5:10])
s3 = sel.get_series('SPB_DET_AGIPD1M-1/DET/7CH0:xtdf', 'image.pulseId')
assert isinstance(s3, pd.Series)
assert not s3.index.is_unique
assert len(s3) == 5 * 64
np.testing.assert_array_equal(s3.index.values,
np.arange(10005, 10010).repeat(64))
def test_iterate_datacollection(agipd_file_tid_low):
f = H5File(agipd_file_tid_low, inc_suspect_trains=False)
tids = [t for (t, _) in f.trains()]
assert len(tids) == 249
assert 9000 not in tids
def test_array_dup(agipd_file_tid_high):
f = H5File(agipd_file_tid_high, inc_suspect_trains=False)
arr = f['SPB_DET_AGIPD1M-1/DET/0CH0:xtdf', 'image.pulseId'].xarray()
assert arr.shape == (485 * 64, 1)
assert list(arr.coords['trainId'].values[(9*64):(11*64):64]) == [10009, 10011]
def test_file_get_series_control(mock_fxe_control_data):
with H5File(mock_fxe_control_data) as f:
s = f.get_series('SA1_XTD2_XGM/DOOCS/MAIN', "beamPosition.iyPos.value")
assert isinstance(s, pd.Series)
assert len(s) == 400
assert s.index[0] == 10000
def test_iterate_trains_flat_keys(mock_agipd_data):
with H5File(mock_agipd_data) as f:
for train_id, data in islice(f.trains(flat_keys=True), 10):
assert train_id in range(10000, 10250)
assert ('SPB_DET_AGIPD1M-1/DET/7CH0:xtdf', 'image.data') in data
def test_empty_file_info(mock_empty_file, capsys):
f = H5File(mock_empty_file)
f.info() # smoke test
def test_train_info(mock_lpd_data, capsys):
with H5File(mock_lpd_data) as f:
f.train_info(10004)
out, err = capsys.readouterr()
assert "Devices" in out
assert "FXE_DET_LPD1M-1/DET/0CH0:xtdf" in out
def test_detector_info(mock_lpd_data):
with H5File(mock_lpd_data) as f:
di = f.detector_info('FXE_DET_LPD1M-1/DET/0CH0:xtdf')
assert di['dims'] == (256, 256)
assert di['frames_per_train'] == 128
assert di['total_frames'] == 128 * 480
def test_components_array(agipd_file_flag0):
f = H5File(agipd_file_flag0, inc_suspect_trains=False)
agipd = AGIPD1M(f, modules=[0])
arr = agipd.get_array('image.data', pulses=np.s_[:1])
assert arr.shape == (1, 485, 1, 2, 512, 128)
assert arr.dims == ('module', 'train', 'pulse', 'data_gain', 'slow_scan', 'fast_scan')
def test_get_run_values(mock_fxe_control_data):
f = H5File(mock_fxe_control_data)
src = 'FXE_XAD_GEC/CAM/CAMERA'
d = f.get_run_values(src, )
assert isinstance(d['firmwareVersion.value'], np.int32)
assert isinstance(d['enableShutter.value'], np.uint8)
def test_detector_info_oldfmt(mock_agipd_data):
with H5File(mock_agipd_data) as f:
di = f.detector_info('SPB_DET_AGIPD1M-1/DET/7CH0:xtdf')
assert di['dims'] == (512, 128)
assert di['frames_per_train'] == 64
assert di['total_frames'] == 16000
def test_open_file_format_1_0(mock_sa3_control_data_fmt_1_0):
f = H5File(mock_sa3_control_data_fmt_1_0)
file_access = f.files[0]
assert file_access.format_version == '1.0'
assert 'SA3_XTD10_VAC/TSENS/S30180K' in f.control_sources