def test_merge_detector(mock_fxe_raw_run, mock_fxe_control_data,
mock_spb_proc_run):
with RunDirectory(mock_fxe_raw_run) as run:
for tid, data in _iter_trains(run, merge_detector=True):
assert 'FXE_DET_LPD1M-1/DET/APPEND' in data
assert 'FXE_DET_LPD1M-1/DET/0CH0:xtdf' not in data
shape = data['FXE_DET_LPD1M-1/DET/APPEND']['image.data'].shape
assert shape == (128, 1, 16, 256, 256)
break
for tid, data in _iter_trains(run):
assert 'FXE_DET_LPD1M-1/DET/0CH0:xtdf' in data
shape = data['FXE_DET_LPD1M-1/DET/0CH0:xtdf']['image.data'].shape
assert shape == (128, 1, 256, 256)
break
with H5File(mock_fxe_control_data) as run:
for tid, data in _iter_trains(run, merge_detector=True):
assert frozenset(data) == run.select_trains(by_id[[tid
]]).all_sources
break
with RunDirectory(mock_spb_proc_run) as run:
for tid, data in _iter_trains(run, merge_detector=True):
shape = data['SPB_DET_AGIPD1M-1/DET/APPEND']['image.data'].shape
assert shape == (64, 16, 512, 128)
shape = data['SPB_DET_AGIPD1M-1/DET/APPEND']['image.gain'].shape
assert shape == (64, 16, 512, 128)
shape = data['SPB_DET_AGIPD1M-1/DET/APPEND']['image.mask'].shape
assert shape == (64, 16, 512, 128)
break
def test_union_raw_proc(mock_spb_raw_run, mock_spb_proc_run):
raw_run = RunDirectory(mock_spb_raw_run)
proc_run = RunDirectory(mock_spb_proc_run)
run = raw_run.deselect('*AGIPD1M*').union(proc_run)
assert run.all_sources == (raw_run.all_sources | proc_run.all_sources)
if raw_run.run_metadata()['dataFormatVersion'] != '0.5':
assert run.is_single_run
def test_read_fxe_raw_run_selective(mock_fxe_raw_run):
run = RunDirectory(mock_fxe_raw_run, include='*DA*')
assert run.train_ids == list(range(10000, 10480))
assert 'SA1_XTD2_XGM/DOOCS/MAIN' in run.control_sources
assert 'FXE_DET_LPD1M-1/DET/0CH0:xtdf' not in run.detector_sources
run = RunDirectory(mock_fxe_raw_run, include='*LPD*')
assert run.train_ids == list(range(10000, 10480))
assert 'SA1_XTD2_XGM/DOOCS/MAIN' not in run.control_sources
assert 'FXE_DET_LPD1M-1/DET/0CH0:xtdf' in run.detector_sources
def test_iterate_select_trains(mock_fxe_raw_run):
run = RunDirectory(mock_fxe_raw_run)
tids = [tid for (tid, _) in run.trains(train_range=by_id[10004:10006])]
assert tids == [10004, 10005]
tids = [tid for (tid, _) in run.trains(train_range=by_id[:10003])]
assert tids == [10000, 10001, 10002]
# Overlap with start of run
tids = [tid for (tid, _) in run.trains(train_range=by_id[9000:10003])]
assert tids == [10000, 10001, 10002]
# Overlap with end of run
tids = [tid for (tid, _) in run.trains(train_range=by_id[10478:10500])]
assert tids == [10478, 10479]
# Not overlapping
with pytest.raises(ValueError) as excinfo:
list(run.trains(train_range=by_id[9000:9050]))
assert 'before' in str(excinfo.value)
with pytest.raises(ValueError) as excinfo:
list(run.trains(train_range=by_id[10500:10550]))
assert 'after' in str(excinfo.value)
tids = [tid for (tid, _) in run.trains(train_range=by_index[4:6])]
assert tids == [10004, 10005]
def test_select_trains(mock_fxe_raw_run):
run = RunDirectory(mock_fxe_raw_run)
assert len(run.train_ids) == 480
sel = run.select_trains(by_id[10200:10220])
assert sel.train_ids == list(range(10200, 10220))
sel = run.select_trains(by_index[:10])
assert sel.train_ids == list(range(10000, 10010))
with pytest.raises(ValueError):
run.select_trains(by_id[9000:9100]) # Before data
with pytest.raises(ValueError):
run.select_trains(by_id[12000:12500]) # After data
# Select a list of train IDs
sel = run.select_trains(by_id[[9950, 10000, 10101, 10500]])
assert sel.train_ids == [10000, 10101]
with pytest.raises(ValueError):
run.select_trains(by_id[[9900, 10600]])
# Select a list of indexes
sel = run.select_trains(by_index[[5, 25]])
assert sel.train_ids == [10005, 10025]
with pytest.raises(IndexError):
run.select_trains(by_index[[480]])
def test_permission():
d = mkdtemp()
os.chmod(d, not stat.S_IRUSR)
with pytest.raises(PermissionError) as excinfo:
run = RunDirectory(d)
assert "Permission denied" in str(excinfo.value)
assert d in str(excinfo.value)
def test_select_by(mock_spb_raw_run):
run = RunDirectory(mock_spb_raw_run)
am0 = run['SPB_DET_AGIPD1M-1/DET/0CH0:xtdf', 'image.data']
subrun = run.select(am0)
assert subrun.all_sources == {am0.source}
assert subrun.keys_for_source(am0.source) == {am0.key}
def test_run_get_series_select_trains(mock_fxe_raw_run):
run = RunDirectory(mock_fxe_raw_run)
sel = run.select_trains(by_id[10100:10150])
s = sel.get_series('SA1_XTD2_XGM/DOOCS/MAIN', "beamPosition.iyPos.value")
assert isinstance(s, pd.Series)
assert len(s) == 50
assert list(s.index) == list(range(10100, 10150))
def test_stack_detector_data_stackview(mock_fxe_raw_run):
test_run = RunDirectory(mock_fxe_raw_run)
tid, data = test_run.train_from_id(10000, devices=[('*/DET/*', 'image.data')])
# Three variants of missing data:
# 1. Source missing
del data['FXE_DET_LPD1M-1/DET/3CH0:xtdf']
# 2. Key missing
del data['FXE_DET_LPD1M-1/DET/7CH0:xtdf']['image.data']
# 3. Empty array
missing = ['FXE_DET_LPD1M-1/DET/{}CH0:xtdf'.format(m) for m in (1, 5, 9, 15)]
for module in missing:
data[module]['image.data'] = np.zeros((0, 1, 256, 256), dtype=np.uint16)
comb = stack_detector_data(data, 'image.data', fillvalue=22, real_array=False)
assert comb.shape == (128, 1, 16, 256, 256)
assert not (comb[:, :, 0] == 22).any() # Control
assert (comb[:, :, 3] == 22).all() # Source missing
assert (comb[:, :, 7] == 22).all() # Key missing
assert (comb[:, :, 5] == 22).all() # Empty array
# Slice across all modules
pulse = comb[0, 0]
assert pulse.shape == (16, 256, 256)
assert not (pulse[0] == 22).any()
assert (pulse[3] == 22).all()
assert (pulse[7] == 22).all()
assert (pulse[5] == 22).all()
pulse_arr = pulse.asarray()
assert pulse_arr.shape == (16, 256, 256)
assert pulse_arr.max() == 22
assert pulse_arr.min() == 0
def test_iterate_select_trains(mock_fxe_raw_run):
run = RunDirectory(mock_fxe_raw_run)
tids = [tid for (tid, _) in run.trains(train_range=by_id[10004:10006])]
assert tids == [10004, 10005]
tids = [tid for (tid, _) in run.trains(train_range=by_id[:10003])]
assert tids == [10000, 10001, 10002]
# Overlap with start of run
tids = [tid for (tid, _) in run.trains(train_range=by_id[9000:10003])]
assert tids == [10000, 10001, 10002]
# Overlap with end of run
tids = [tid for (tid, _) in run.trains(train_range=by_id[10478:10500])]
assert tids == [10478, 10479]
# Not overlapping
with catch_warnings(record=True) as w:
tids = [tid for (tid, _) in run.trains(train_range=by_id[9000:9050])]
assert tids == []
assert 'before' in str(w[0].message)
with catch_warnings(record=True) as w:
tids = [tid for (tid, _) in run.trains(train_range=by_id[10500:10550])]
assert tids == []
assert 'after' in str(w[0].message)
tids = [tid for (tid, _) in run.trains(train_range=by_index[4:6])]
assert tids == [10004, 10005]
def test_write_virtual(mock_fxe_raw_run):
with TemporaryDirectory() as td:
new_file = osp.join(td, 'test.h5')
with RunDirectory(mock_fxe_raw_run) as run:
run.write_virtual(new_file)
assert_isfile(new_file)
with h5py.File(new_file, 'r') as f:
ds = f['CONTROL/SPB_XTD9_XGM/DOOCS/MAIN/beamPosition/ixPos/value']
assert ds.is_virtual
with H5File(new_file) as f:
np.testing.assert_array_equal(
f.train_ids, np.arange(10000, 10480, dtype=np.uint64))
assert 'SPB_XTD9_XGM/DOOCS/MAIN' in f.control_sources
assert 'SPB_XTD9_XGM/DOOCS/MAIN:output' in f.instrument_sources
s = f.get_series('SPB_XTD9_XGM/DOOCS/MAIN',
'beamPosition.ixPos.value')
# This should have concatenated the two sequence files (400 + 80)
assert len(s) == 480
a = f.get_array('SPB_XTD9_XGM/DOOCS/MAIN:output',
'data.intensityTD')
assert a.shape == (480, 1000)
def test_serve_files(mock_fxe_raw_run):
args = [
'karabo-bridge-serve-files', '-z', 'PUSH',
str(mock_fxe_raw_run),
str(44444)
]
interface = None
with Popen(args,
stdin=PIPE,
stdout=PIPE,
stderr=PIPE,
env=dict(os.environ, PYTHONUNBUFFERED='1')) as p:
for line in p.stdout:
line = line.decode('utf-8')
if line.startswith('Streamer started on:'):
interface = line.partition(':')[2].strip()
break
print('interface:', interface)
assert interface is not None, p.stderr.read().decode()
with Client(interface, sock='PULL', timeout=5) as c:
data, meta = c.next()
tid = next(m['timestamp.tid'] for m in meta.values())
sources = RunDirectory(mock_fxe_raw_run).select_trains(
by_id[[tid]]).all_sources
assert frozenset(data) == sources
p.kill()
rc = p.wait(timeout=2)
assert rc == -9 # process terminated by kill signal
def test_stack_detector_data_missing(mock_fxe_raw_run):
test_run = RunDirectory(mock_fxe_raw_run)
tid, data = test_run.train_from_id(10000,
devices=[('*/DET/*', 'image.data')])
# Three variants of missing data:
# 1. Source missing
del data['FXE_DET_LPD1M-1/DET/3CH0:xtdf']
# 2. Key missing
del data['FXE_DET_LPD1M-1/DET/7CH0:xtdf']['image.data']
# 3. Empty array
missing = [
'FXE_DET_LPD1M-1/DET/{}CH0:xtdf'.format(m) for m in (1, 5, 9, 15)
]
for module in missing:
data[module]['image.data'] = np.zeros((0, 1, 256, 256),
dtype=np.uint16)
comb = stack_detector_data(data, 'image.data', fillvalue=22)
assert comb.shape == (128, 1, 16, 256, 256)
assert not (comb[:, :, 0] == 22).any() # Control
assert (comb[:, :, 3] == 22).all() # Source missing
assert (comb[:, :, 7] == 22).all() # Key missing
assert (comb[:, :, 5] == 22).all() # Empty array
# default fillvalue for int is 0
comb = stack_detector_data(data, 'image.data')
assert (comb[:, :, 3] == 0).all()
with pytest.raises(ValueError):
comb = stack_detector_data(data, 'image.data', fillvalue=np.nan)
def agipd_frame(mock_run):
run = RunDirectory(mock_run)
print(run.instrument_sources)
print(run.keys_for_source('SPB_DET_AGIPD1M-1/DET/10CH0:xtdf'))
tid, train_data = run.select('*/DET/*', 'image.data').train_from_index(0)
ary = stack_detector_data(train_data, 'image.data')
return ary[0]
def test_save_load_map(run_with_extra_file, tmp_path):
run_dir, extra_file = run_with_extra_file
run_map_path = str(tmp_path / 'kd_test_run_map.json')
class TestRunFilesMap(run_files_map.RunFilesMap):
def map_paths_for_run(self, directory):
return [run_map_path]
rfm = TestRunFilesMap(run_dir)
assert rfm.files_data == {}
with RunDirectory(run_dir) as run:
rfm.save(run.files)
rfm2 = TestRunFilesMap(run_dir)
assert rfm2.cache_file == run_map_path
file_info = rfm2.get(extra_file)
assert isinstance(file_info['train_ids'], np.ndarray)
assert isinstance(file_info['control_sources'], frozenset)
assert isinstance(file_info['instrument_sources'], frozenset)
# Modify a file; this should make the cache invalid
with h5py.File(extra_file, 'r+') as f:
f.attrs['test_save_load_map'] = 1
rfm3 = TestRunFilesMap(run_dir)
assert rfm3.cache_file == run_map_path
assert rfm3.get(extra_file) is None
def test_train_from_index_fxe_run(mock_fxe_raw_run):
run = RunDirectory(mock_fxe_raw_run)
_, data = run.train_from_index(479)
assert 'FXE_DET_LPD1M-1/DET/15CH0:xtdf' in data
assert 'image.data' in data['FXE_DET_LPD1M-1/DET/15CH0:xtdf']
assert 'FXE_XAD_GEC/CAM/CAMERA' in data
assert 'firmwareVersion.value' in data['FXE_XAD_GEC/CAM/CAMERA']
def test_stack_data(mock_fxe_raw_run):
test_run = RunDirectory(mock_fxe_raw_run)
tid, data = test_run.train_from_id(10000,
devices=[('*/DET/*', 'image.data')])
comb = stack_data(data, 'image.data')
assert comb.shape == (128, 1, 16, 256, 256)
def gather_sources(path):
"""Gather slow sources from a run
Parameters:
-----------
path: str
Path to HDF5 run folder
Return:
-------
Slow sources: Frozenset
Set of slow sources available. Empty if none found.
Run info: string
Information about the run. Empty string if no info found
"""
DETECTOR_SOURCE_RE = re.compile(r'(.+)/DET/(.+):(.+)')
sources = frozenset()
info = ""
if osp.isdir(path):
try:
# This will work in case users are using data stored
# in /gpfs/exfel/exp/<INSTRUMENT>/cycle/proposal/proc/runumber
# or they have raw folder with same path instead of 'proc'
# in it in the end.
run = RunDirectory(path.replace('/proc/', '/raw/'))
sources = frozenset(
src
for src in run.control_sources.union(run.instrument_sources)
if not DETECTOR_SOURCE_RE.match(src))
info = get_info(run)
except Exception as ex:
# Will be raised if no folder with 'raw' exist or no files
# found in raw folder.
logger.info("Failed to find 'raw' folder! "
"Looking for slow data in current folder")
# Fallback to corrected datapath. Will return empty
# frozenset if no control source found.
try:
run = RunDirectory(path)
sources = frozenset(src for src in run.control_sources.union(
run.instrument_sources)
if not DETECTOR_SOURCE_RE.match(src))
info = get_info(run)
except Exception as ex:
logger.error(repr(ex))
return sources, info
def test_iterate_spb_raw_run(mock_spb_raw_run):
run = RunDirectory(mock_spb_raw_run)
trains_iter = run.trains()
tid, data = next(trains_iter)
assert tid == 10000
device = 'SPB_IRU_CAM/CAM/SIDEMIC:daqOutput'
assert device in data
assert data[device]['data.image.pixels'].shape == (1024, 768)
def test_iter_trains(mock_spb_raw_run):
run = RunDirectory(mock_spb_raw_run)
xgm_beam_x = run['SPB_XTD9_XGM/DOOCS/MAIN', 'beamPosition.ixPos.value']
assert [t for (t, _) in xgm_beam_x.trains()] == list(range(10000, 10064))
for _, v in xgm_beam_x.trains():
assert isinstance(v, np.float32)
break
def test_run_immutable_sources(mock_fxe_raw_run):
test_run = RunDirectory(mock_fxe_raw_run)
before = len(test_run.all_sources)
with pytest.raises(AttributeError):
test_run.all_sources.pop()
assert len(test_run.all_sources) == before
def test_run_get_array_error(mock_fxe_raw_run):
run = RunDirectory(mock_fxe_raw_run)
with pytest.raises(SourceNameError):
run.get_array('bad_name', 'data.intensityTD')
with pytest.raises(PropertyNameError):
run.get_array('SA1_XTD2_XGM/DOOCS/MAIN:output', 'bad_name')
def test_run_get_array_empty(mock_fxe_raw_run):
run = RunDirectory(mock_fxe_raw_run)
arr = run.get_array('FXE_XAD_GEC/CAM/CAMERA_NODATA:daqOutput',
'data.image.pixels')
assert isinstance(arr, DataArray)
assert arr.dims[0] == 'trainId'
assert arr.shape == (0, 255, 1024)
def test_iterate_run_glob_devices(mock_fxe_raw_run):
run = RunDirectory(mock_fxe_raw_run)
trains_iter = run.trains([("*/DET/*", "image.data")])
tid, data = next(trains_iter)
assert tid == 10000
assert 'FXE_DET_LPD1M-1/DET/15CH0:xtdf' in data
assert 'image.data' in data['FXE_DET_LPD1M-1/DET/15CH0:xtdf']
assert 'detector.data' not in data['FXE_DET_LPD1M-1/DET/15CH0:xtdf']
assert 'FXE_XAD_GEC/CAM/CAMERA' not in data
def main(argv=None):
ap = argparse.ArgumentParser('karabo-data-make-virtual-cxi')
ap.add_argument('run_dir', help="Path to an EuXFEL run directory")
# Specifying a proposal directory & a run number is the older interface.
# If the run_number argument is passed, run_dir is used as proposal.
ap.add_argument('run_number', nargs="?", help=argparse.SUPPRESS)
ap.add_argument(
'-o',
'--output',
help="Filename or path for the CXI output file. "
"By default, it is written in the proposal's scratch directory.")
ap.add_argument(
'--min-modules',
type=int,
default=9,
metavar='N',
help="Include trains where at least N modules have data (default 9)")
args = ap.parse_args(argv)
out_file = args.output
logging.basicConfig(level=logging.INFO)
if args.run_number:
# proposal directory, run number
run = 'r%04d' % int(args.run_number)
proposal = args.run_dir
run_dir = osp.join(args.run_dir, 'proc', run)
if out_file is None:
out_file = osp.join(proposal, 'scratch',
'{}_detectors_virt.cxi'.format(run))
else:
# run directory
run_dir = os.path.abspath(args.run_dir)
if out_file is None:
m = re.search(r'/(raw|proc)/(r\d{4})/?$', run_dir)
if not m:
sys.exit("ERROR: '-o outfile' option needed when "
"input directory doesn't look like .../proc/r0123")
proposal = run_dir[:m.start()]
fname = '{}_{}_detectors_virt.cxi'.format(*m.group(2, 1))
out_file = osp.join(proposal, 'scratch', fname)
out_dir = osp.dirname(osp.abspath(out_file))
if not os.access(run_dir, os.R_OK):
sys.exit("ERROR: Don't have read access to {}".format(run_dir))
if not os.access(out_dir, os.W_OK):
sys.exit("ERROR: Don't have write access to {}".format(out_dir))
log.info("Reading run directory %s", run_dir)
run = RunDirectory(run_dir)
det = _get_detector(run, args.min_modules)
if det is None:
sys.exit("No AGIPD or LPD sources found in {!r}".format(run_dir))
det.write_virtual_cxi(out_file)
def test_use_voview(mock_spb_raw_run, tmp_path):
new_run_dir = tmp_path / 'r0238'
copytree(mock_spb_raw_run, new_run_dir)
voview_file = new_run_dir / 'overview.h5'
run_orig = RunDirectory(str(new_run_dir), _use_voview=False)
assert len(run_orig.files) > 1
assert voview.find_file_write(new_run_dir) == str(voview_file)
vofw = voview.VirtualOverviewFileWriter(voview_file, run_orig)
vofw.write()
run = RunDirectory(str(new_run_dir))
assert [f.filename for f in run.files] == [str(voview_file)]
assert len(run.train_ids) == 64
assert 'SPB_DET_AGIPD1M-1/DET/0CH0:xtdf' in run.instrument_sources
assert 'SA1_XTD2_XGM/DOOCS/MAIN' in run.control_sources
def test_get_sourcedata(mock_spb_raw_run):
run = RunDirectory(mock_spb_raw_run)
am0 = run['SPB_DET_AGIPD1M-1/DET/0CH0:xtdf']
assert len(am0.files) == 1
assert am0.section == 'INSTRUMENT'
xgm = run['SPB_XTD9_XGM/DOOCS/MAIN']
assert len(xgm.files) == 2
assert xgm.section == 'CONTROL'
def test_iterate_fxe_run(mock_fxe_raw_run):
run = RunDirectory(mock_fxe_raw_run)
trains_iter = run.trains()
tid, data = next(trains_iter)
assert tid == 10000
assert 'FXE_DET_LPD1M-1/DET/15CH0:xtdf' in data
assert 'image.data' in data['FXE_DET_LPD1M-1/DET/15CH0:xtdf']
assert 'FXE_XAD_GEC/CAM/CAMERA' in data
assert 'firmwareVersion.value' in data['FXE_XAD_GEC/CAM/CAMERA']
def test_stack_detector_data_jungfrau(mock_jungfrau_run):
run = RunDirectory(mock_jungfrau_run)
_, data = run.select('*JF4M/DET/*', 'data.adc').train_from_index(0)
comb = stack_detector_data(data,
'data.adc',
modules=8,
pattern=r'/DET/JNGFR(\d+)',
starts_at=1)
assert comb.shape == (16, 8, 512, 1024)
def test_run_get_array_multiple_per_train(mock_fxe_raw_run):
run = RunDirectory(mock_fxe_raw_run)
sel = run.select_trains(np.s_[:2])
arr = sel.get_array('FXE_DET_LPD1M-1/DET/6CH0:xtdf',
'image.data',
roi=np.s_[:, 10:20, 20:40])
assert isinstance(arr, DataArray)
assert arr.shape == (256, 1, 10, 20)
np.testing.assert_array_equal(arr.coords['trainId'],
np.repeat([10000, 10001], 128))
