def __init__(self, filename, source_ind=0): if not karabo_installed: raise IOError('karabo_data is required to load karabo h5 files') self.f = H5File(filename) self.series_max = len(self.f.train_ids) self.sources = [ s for s in self.f.instrument_sources if "daqOutput" in s ] self.current_source = self.sources[source_ind]
def test_open_file(mock_sa3_control_data): f = H5File(mock_sa3_control_data) file_access = f.files[0] assert file_access.format_version in ('0.5', '1.0') assert 'SA3_XTD10_VAC/TSENS/S30180K' in f.control_sources if file_access.format_version == '0.5': assert 'METADATA/dataSourceId' in file_access.file else: assert 'METADATA/dataSources/dataSourceId' in file_access.file
def test_file_get_array(mock_fxe_control_data): with H5File(mock_fxe_control_data) as f: arr = f.get_array('FXE_XAD_GEC/CAM/CAMERA:daqOutput', 'data.image.pixels') assert isinstance(arr, DataArray) assert arr.dims == ('trainId', 'dim_0', 'dim_1') assert arr.shape == (400, 255, 1024) assert arr.coords['trainId'][0] == 10000
def test_select_require_all(mock_sa3_control_data, select_str): # De-select two sources in this example set, which have no trains # at all, to allow matching trains across all sources with the same # result. run = H5File(mock_sa3_control_data) \ .deselect([('SA3_XTD10_MCP/ADC/1:*', '*'), ('SA3_XTD10_IMGFEL/CAM/BEAMVIEW:*', '*')]) subrun = run.select(select_str, require_all=True) np.testing.assert_array_equal(subrun.train_ids, run.train_ids[1::2])
def test_still_valid_elsewhere(agipd_file_tid_very_high, mock_sa3_control_data): dc = H5File( agipd_file_tid_very_high, inc_suspect_trains=False ).union(H5File(mock_sa3_control_data)) assert dc.train_ids == list(range(10000, 10500)) agipd_src = 'SPB_DET_AGIPD1M-1/DET/7CH0:xtdf' tsens_src = 'SA3_XTD10_VAC/TSENS/S30250K' sel = dc.select({ agipd_src: {'image.pulseId'}, tsens_src: {'value.value'} }) assert sel.all_sources == {agipd_src, tsens_src} _, t1 = sel.train_from_id(10200, flat_keys=True) assert set(t1) >= {(agipd_src, 'image.pulseId'), (tsens_src, 'value.value')} _, t2 = sel.train_from_id(10400, flat_keys=True) assert (agipd_src, 'image.pulseId') not in t2 assert (tsens_src, 'value.value') in t2 tids_from_iter, data_from_iter = [], [] for tid, d in sel.trains(flat_keys=True): if tid in (10200, 10400): tids_from_iter.append(tid) data_from_iter.append(d) assert tids_from_iter == [10200, 10400] assert [set(d) for d in data_from_iter] == [set(t1), set(t2)] # Check that select with require_all respects the valid train filtering: sel2 = dc.select(agipd_src, require_all=True) assert len(sel2.train_ids) == 249 dc_inc = H5File(agipd_file_tid_very_high, inc_suspect_trains=True)\ .union(H5File(mock_sa3_control_data)) sel_inc = dc_inc.select(sel) _, t2_inc = sel_inc.train_from_id(10400, flat_keys=True) assert set(t2_inc) == set(t1) sel2_inc = dc_inc.select(agipd_src, require_all=True) assert len(sel2_inc.train_ids) == 250
def test_file_get_array_missing_trains(mock_sa3_control_data): with H5File(mock_sa3_control_data) as f: sel = f.select_trains(by_index[:6]) arr = sel.get_array('SA3_XTD10_IMGFEL/CAM/BEAMVIEW2:daqOutput', 'data.image.dims') assert isinstance(arr, DataArray) assert arr.dims == ('trainId', 'dim_0') assert arr.shape == (3, 2) np.testing.assert_array_less(arr.coords['trainId'], 10006) np.testing.assert_array_less(10000, arr.coords['trainId'])
def test_iterate_trains_fxe(mock_fxe_control_data): with H5File(mock_fxe_control_data) as f: for train_id, data in islice(f.trains(), 10): assert train_id in range(10000, 10400) assert 'SA1_XTD2_XGM/DOOCS/MAIN' in data.keys() assert 'beamPosition.ixPos.value' in data[ 'SA1_XTD2_XGM/DOOCS/MAIN'] assert 'data.image.pixels' in data[ 'FXE_XAD_GEC/CAM/CAMERA:daqOutput'] assert 'data.image.pixels' not in data[ 'FXE_XAD_GEC/CAM/CAMERA_NODATA:daqOutput']
def test_get_train_bad_device_name(mock_spb_control_data_badname): # Check that we can handle devices which don't have the standard Karabo # name structure A/B/C. with H5File(mock_spb_control_data_badname) as f: train_id, data = f.train_from_id(10004) assert train_id == 10004 device = 'SPB_IRU_SIDEMIC_CAM:daqOutput' assert device in data assert 'data.image.dims' in data[device] dims = data[device]['data.image.dims'] assert list(dims) == [1000, 1000]
def test_get_run_value(mock_fxe_control_data): f = H5File(mock_fxe_control_data) src = 'FXE_XAD_GEC/CAM/CAMERA' val = f.get_run_value(src, 'firmwareVersion') assert isinstance(val, np.int32) assert f.get_run_value(src, 'firmwareVersion.value') == val with pytest.raises(SourceNameError): f.get_run_value(src + '_NONEXIST', 'firmwareVersion') with pytest.raises(PropertyNameError): f.get_run_value(src, 'non.existant')
def test_iterate_trains_require_all(mock_sa3_control_data): with H5File(mock_sa3_control_data) as f: trains_iter = f.trains(devices=[('*/CAM/BEAMVIEW:daqOutput', 'data.image.dims')], require_all=True) tids = [t for (t, _) in trains_iter] assert tids == [] trains_iter = f.trains(devices=[('*/CAM/BEAMVIEW:daqOutput', 'data.image.dims')], require_all=False) tids = [t for (t, _) in trains_iter] assert tids != []
def test_file_get_array_control_roi(mock_sa3_control_data): with H5File(mock_sa3_control_data) as f: sel = f.select_trains(by_index[:6]) arr = sel.get_array( 'SA3_XTD10_VAC/DCTRL/D6_APERT_IN_OK', 'interlock.a1.AActCommand.value', roi=by_index[:25], ) assert isinstance(arr, DataArray) assert arr.shape == (6, 25) assert arr.coords['trainId'][0] == 10000
def test_select_require_all(mock_sa3_control_data, select_str): # De-select two sources in this example set, which have no trains # at all, to allow matching trains across all sources with the same # result. run = H5File(mock_sa3_control_data) \ .deselect([('SA3_XTD10_MCP/ADC/1:*', '*'), ('SA3_XTD10_IMGFEL/CAM/BEAMVIEW:*', '*')]) subrun = run.select(select_str, require_all=True) np.testing.assert_array_equal(subrun.train_ids, run.train_ids[1::2]) # The train IDs are held by ndarrays during this operation, make # sure it's a list of np.uint64 again. assert isinstance(subrun.train_ids, list) assert all([isinstance(x, np.uint64) for x in subrun.train_ids])
def test_single_value(mock_sa3_control_data, monkeypatch): f = H5File(mock_sa3_control_data) imager = f['SA3_XTD10_IMGFEL/CAM/BEAMVIEW:daqOutput', 'data.image.pixels'] flux = f['SA3_XTD10_XGM/XGM/DOOCS', 'pulseEnergy.photonFlux'] # Try without data for a source and key. with pytest.raises(NoDataError): imager.as_single_value() # FEL imager with no data. with pytest.raises(NoDataError): flux[:0].as_single_value() # No data through selection. # Monkeypatch some actual data into the KeyData object data = np.arange(flux.shape[0]) monkeypatch.setattr(flux, 'ndarray', lambda: data) # Try some tolerances that have to fail. with pytest.raises(ValueError): flux.as_single_value() with pytest.raises(ValueError): flux.as_single_value(atol=1) with pytest.raises(ValueError): flux.as_single_value(rtol=0.1) # Try with large enough tolerances. assert flux.as_single_value(atol=len(data) / 2) == np.median(data) assert flux.as_single_value(rtol=0.5, atol=len(data) / 4) == np.median(data) assert flux.as_single_value(rtol=1) == np.median(data) # Other reduction options assert flux.as_single_value(rtol=1, reduce_by='mean') == np.mean(data) assert flux.as_single_value(rtol=1, reduce_by=np.mean) == np.mean(data) assert flux.as_single_value(atol=len(data) - 1, reduce_by='first') == 0 # Try vector data. intensity = f['SA3_XTD10_XGM/XGM/DOOCS:output', 'data.intensityTD'] data = np.repeat(data, intensity.shape[1]).reshape(-1, intensity.shape[-1]) monkeypatch.setattr(intensity, 'ndarray', lambda: data) with pytest.raises(ValueError): intensity.as_single_value() np.testing.assert_equal(intensity.as_single_value(rtol=1), np.median(data))
def test_drop_empty_trains(mock_sa3_control_data): f = H5File(mock_sa3_control_data) beamview = f['SA3_XTD10_IMGFEL/CAM/BEAMVIEW2:daqOutput', 'data.image.dims'] assert len(beamview.train_ids) == 500 a1 = beamview.ndarray() assert a1.shape == (250, 2) frame_counts = beamview.data_counts(labelled=False) assert frame_counts.shape == (500, ) assert frame_counts.min() == 0 beamview_w_data = beamview.drop_empty_trains() assert len(beamview_w_data.train_ids) == 250 np.testing.assert_array_equal(beamview_w_data.ndarray(), a1) frame_counts = beamview_w_data.data_counts(labelled=False) assert frame_counts.shape == (250, ) assert frame_counts.min() == 1
def test_iterate_trains_select_keys(mock_fxe_control_data): sel = { 'SA1_XTD2_XGM/DOOCS/MAIN': { 'beamPosition.ixPos.value', 'beamPosition.ixPos.timestamp', } } with H5File(mock_fxe_control_data) as f: for train_id, data in islice(f.trains(devices=sel), 10): assert train_id in range(10000, 10400) assert 'SA1_XTD2_XGM/DOOCS/MAIN' in data.keys() assert 'beamPosition.ixPos.value' in data[ 'SA1_XTD2_XGM/DOOCS/MAIN'] assert 'beamPosition.ixPos.timestamp' in data[ 'SA1_XTD2_XGM/DOOCS/MAIN'] assert 'beamPosition.iyPos.value' not in data[ 'SA1_XTD2_XGM/DOOCS/MAIN'] assert 'SA3_XTD10_VAC/TSENS/S30160K' not in data
def test_write_selected(mock_fxe_raw_run): with TemporaryDirectory() as td: new_file = osp.join(td, 'test.h5') with RunDirectory(mock_fxe_raw_run) as run: run.select('SPB_XTD9_XGM/*').write(new_file) assert_isfile(new_file) with H5File(new_file) as f: assert f.control_sources == {'SPB_XTD9_XGM/DOOCS/MAIN'} assert f.instrument_sources == {'SPB_XTD9_XGM/DOOCS/MAIN:output'} s = f.get_series('SPB_XTD9_XGM/DOOCS/MAIN', 'beamPosition.ixPos.value') # This should have concatenated the two sequence files (400 + 80) assert len(s) == 480 a = f.get_array('SPB_XTD9_XGM/DOOCS/MAIN:output', 'data.intensityTD') assert a.shape == (480, 1000)
def test_file_get_series_instrument(mock_agipd_data): with H5File(mock_agipd_data) as f: s = f.get_series('SPB_DET_AGIPD1M-1/DET/7CH0:xtdf', 'header.linkId') assert isinstance(s, pd.Series) assert len(s) == 250 assert s.index[0] == 10000 # Multiple readings per train s2 = f.get_series('SPB_DET_AGIPD1M-1/DET/7CH0:xtdf', 'image.status') assert isinstance(s2, pd.Series) assert isinstance(s2.index, pd.MultiIndex) assert len(s2) == 16000 assert len(s2.loc[10000:10004]) == 5 * 64 sel = f.select_trains(by_index[5:10]) s3 = sel.get_series('SPB_DET_AGIPD1M-1/DET/7CH0:xtdf', 'image.status') assert isinstance(s3, pd.Series) assert isinstance(s3.index, pd.MultiIndex) assert len(s3) == 5 * 64 np.testing.assert_array_equal(s3.index.get_level_values(0), np.arange(10005, 10010).repeat(64))
def test_file_get_series_instrument(mock_spb_proc_run): agipd_file = os.path.join(mock_spb_proc_run, 'CORR-R0238-AGIPD07-S00000.h5') with H5File(agipd_file) as f: s = f.get_series('SPB_DET_AGIPD1M-1/DET/7CH0:xtdf', 'header.linkId') assert isinstance(s, pd.Series) assert len(s) == 64 assert s.index[0] == 10000 # Multiple readings per train s2 = f.get_series('SPB_DET_AGIPD1M-1/DET/7CH0:xtdf', 'image.pulseId') assert isinstance(s2, pd.Series) assert not s2.index.is_unique assert len(s2) == 64 * 64 assert len(s2.loc[10000:10004]) == 5 * 64 sel = f.select_trains(by_index[5:10]) s3 = sel.get_series('SPB_DET_AGIPD1M-1/DET/7CH0:xtdf', 'image.pulseId') assert isinstance(s3, pd.Series) assert not s3.index.is_unique assert len(s3) == 5 * 64 np.testing.assert_array_equal(s3.index.values, np.arange(10005, 10010).repeat(64))
def test_iterate_datacollection(agipd_file_tid_low): f = H5File(agipd_file_tid_low, inc_suspect_trains=False) tids = [t for (t, _) in f.trains()] assert len(tids) == 249 assert 9000 not in tids
def test_array_dup(agipd_file_tid_high): f = H5File(agipd_file_tid_high, inc_suspect_trains=False) arr = f['SPB_DET_AGIPD1M-1/DET/0CH0:xtdf', 'image.pulseId'].xarray() assert arr.shape == (485 * 64, 1) assert list(arr.coords['trainId'].values[(9*64):(11*64):64]) == [10009, 10011]
def test_file_get_series_control(mock_fxe_control_data): with H5File(mock_fxe_control_data) as f: s = f.get_series('SA1_XTD2_XGM/DOOCS/MAIN', "beamPosition.iyPos.value") assert isinstance(s, pd.Series) assert len(s) == 400 assert s.index[0] == 10000
def test_iterate_trains_flat_keys(mock_agipd_data): with H5File(mock_agipd_data) as f: for train_id, data in islice(f.trains(flat_keys=True), 10): assert train_id in range(10000, 10250) assert ('SPB_DET_AGIPD1M-1/DET/7CH0:xtdf', 'image.data') in data
def test_empty_file_info(mock_empty_file, capsys): f = H5File(mock_empty_file) f.info() # smoke test
def test_train_info(mock_lpd_data, capsys): with H5File(mock_lpd_data) as f: f.train_info(10004) out, err = capsys.readouterr() assert "Devices" in out assert "FXE_DET_LPD1M-1/DET/0CH0:xtdf" in out
def test_detector_info(mock_lpd_data): with H5File(mock_lpd_data) as f: di = f.detector_info('FXE_DET_LPD1M-1/DET/0CH0:xtdf') assert di['dims'] == (256, 256) assert di['frames_per_train'] == 128 assert di['total_frames'] == 128 * 480
def test_components_array(agipd_file_flag0): f = H5File(agipd_file_flag0, inc_suspect_trains=False) agipd = AGIPD1M(f, modules=[0]) arr = agipd.get_array('image.data', pulses=np.s_[:1]) assert arr.shape == (1, 485, 1, 2, 512, 128) assert arr.dims == ('module', 'train', 'pulse', 'data_gain', 'slow_scan', 'fast_scan')
def test_get_run_values(mock_fxe_control_data): f = H5File(mock_fxe_control_data) src = 'FXE_XAD_GEC/CAM/CAMERA' d = f.get_run_values(src, ) assert isinstance(d['firmwareVersion.value'], np.int32) assert isinstance(d['enableShutter.value'], np.uint8)
def test_detector_info_oldfmt(mock_agipd_data): with H5File(mock_agipd_data) as f: di = f.detector_info('SPB_DET_AGIPD1M-1/DET/7CH0:xtdf') assert di['dims'] == (512, 128) assert di['frames_per_train'] == 64 assert di['total_frames'] == 16000
def test_open_file_format_1_0(mock_sa3_control_data_fmt_1_0): f = H5File(mock_sa3_control_data_fmt_1_0) file_access = f.files[0] assert file_access.format_version == '1.0' assert 'SA3_XTD10_VAC/TSENS/S30180K' in f.control_sources