def test_pattern(tmp_path: Path, dl1_file, dl1_proton_file):
from ctapipe.tools.dl1_merge import MergeTool
# touch a random file to test that the pattern does not use it
open(dl1_file.parent / "foo.h5", "w").close()
# copy to make sure we don't have other files in the dl1 dir disturb this
for f in (dl1_file, dl1_proton_file):
shutil.copy(f, tmp_path)
output = tmp_path / "merged_pattern.dl1.h5"
ret = run_tool(
tool=MergeTool(),
argv=[
"-i",
str(tmp_path),
"-p",
"*.dl1.h5",
f"--output={output}",
"--overwrite",
],
cwd=tmp_path,
)
assert ret == 0
run_stage1(output, cwd=tmp_path)
def test_simple(tmp_path, dl1_file, dl1_proton_file):
from ctapipe.tools.dl1_merge import MergeTool
output = tmp_path / "merged_simple.dl1.h5"
ret = run_tool(
MergeTool(),
argv=[str(dl1_file), str(dl1_proton_file), f"--output={output}", "--overwrite"],
cwd=tmp_path,
)
assert ret == 0
run_stage1(output, cwd=tmp_path)
def test_skip_images(tmp_path, gamma_dl1_path, proton_dl1_path):
from ctapipe.tools.dl1_merge import MergeTool
# create a second file so we can test the patterns
output = tmp_path / "merged_no_images.dl1.h5"
ret = run_tool(
MergeTool(),
argv=[
str(gamma_dl1_path),
str(proton_dl1_path),
f"--output={output}",
"--skip-images",
"--overwrite",
],
cwd=tmp_path,
)
assert ret == 0
def test_pattern(tmp_path: Path, gamma_dl1_path, proton_dl1_path):
from ctapipe.tools.dl1_merge import MergeTool
# touch a random file to test that the pattern does not use it
open(tmp_path / "foo.h5", "w").close()
output = tmp_path / "merged_pattern.dl1.h5"
ret = run_tool(
MergeTool(),
argv=[
"-i",
str(tmp_path),
"-p",
"*.dl1.h5",
f"--output={output}",
"--overwrite",
],
cwd=tmp_path,
)
assert ret == 0
run_stage1(output, cwd=tmp_path)
def test_allowed_tels(tmp_path, dl1_file, dl1_proton_file):
from ctapipe.tools.dl1_merge import MergeTool
from ctapipe.instrument import SubarrayDescription
# create file to test 'allowed-tels' option
output = tmp_path / "merged_allowed_tels.dl1.h5"
ret = run_tool(
MergeTool(),
argv=[
str(dl1_file),
str(dl1_proton_file),
f"--output={output}",
"--allowed-tels=[1,2]",
"--overwrite",
],
cwd=tmp_path,
)
assert ret == 0
s = SubarrayDescription.from_hdf(output)
assert s.tel.keys() == {1, 2}
def test_skip_images(tmp_path, dl1_file, dl1_proton_file):
from ctapipe.tools.dl1_merge import MergeTool
# create a second file so we can test the patterns
output = tmp_path / "merged_no_images.dl1.h5"
ret = run_tool(
MergeTool(),
argv=[
str(dl1_file),
str(dl1_proton_file),
f"--output={output}",
"--skip-images",
"--overwrite",
],
cwd=tmp_path,
)
with tables.open_file(output, "r") as f:
assert "images" not in f.root.dl1.event.telescope
assert "parameters" in f.root.dl1.event.telescope
assert ret == 0
def test_merge(tmpdir):
from ctapipe.tools.dl1_merge import MergeTool
from ctapipe.tools.stage1 import Stage1Tool
config = Path("./examples/stage1_config.json").absolute()
tmp_dir = tempfile.TemporaryDirectory()
in_1 = tmp_dir.name + "/test_file_1.hdf5"
in_2 = tmp_dir.name + "/test_file_2.hdf5"
out_all = tmp_dir.name + "/merged_file_all.hdf5"
out_skip_images = tmp_dir.name + "/merged_file_images.hdf5"
out_skip_parameters = tmp_dir.name + "/merged_file_parameters.hdf5"
out_tels_dir_pattern = tmp_dir.name + "/merged_file_tels_dir_pattern.hdf5"
assert (
run_tool(
Stage1Tool(),
argv=[
f"--config={config}",
f"--input={GAMMA_TEST_LARGE}",
f"--output={in_1}",
"--write-parameters",
"--write-images",
"--overwrite",
],
cwd=tmpdir,
)
== 0
)
assert (
run_tool(
Stage1Tool(),
argv=[
f"--config={config}",
f"--input={GAMMA_TEST_LARGE}",
f"--output={in_2}",
"--write-parameters",
"--write-images",
"--overwrite",
],
cwd=tmpdir,
)
== 0
)
assert (
run_tool(
MergeTool(),
argv=[
f"--i={tmp_dir.name}",
"--p='test_file_*.hdf5'",
f"--o={out_tels_dir_pattern}",
"--overwrite",
"--t=[2, 3]",
],
cwd=tmpdir,
)
== 0
)
assert (
run_tool(
MergeTool(), argv=[in_1, in_2, f"--o={out_all}", "--overwrite"], cwd=tmpdir
)
== 0
)
assert (
run_tool(
MergeTool(),
argv=[in_1, in_2, f"--o={out_skip_images}", "--overwrite", "--skip-images"],
cwd=tmpdir,
)
== 0
)
assert (
run_tool(
MergeTool(),
argv=[
in_1,
in_2,
f"--o={out_skip_parameters}",
"--overwrite",
"--skip-parameters",
],
cwd=tmpdir,
)
== 0
)
out_files_list = [
out_all,
out_skip_images,
out_skip_parameters,
out_tels_dir_pattern,
]
for out_file in out_files_list:
with tables.open_file(out_file, mode="r") as out_f, tables.open_file(
in_1, mode="r"
) as in_f:
# Check expanded tables
assert len(out_f.root.simulation.service.shower_distribution) == 2
assert len(out_f.root.simulation.event.subarray.shower) == 220
assert len(out_f.root.configuration.simulation.run) == 2
assert len(out_f.root.dl1.monitoring.subarray.pointing) == 2
assert len(out_f.root.dl1.event.subarray.trigger) == 220
assert len(out_f.root.dl1.event.telescope.trigger) == 918
assert len(out_f.root.simulation.service.shower_distribution) == 2
# Check subarray and service meta
assert out_f.root.dl1.service["image_statistics.__table_column_meta__"]
assert out_f.root.configuration.instrument.subarray.layout
assert out_f.root.configuration.instrument.telescope.optics
assert out_f.root.configuration.instrument.telescope.camera.geometry_LSTCam
assert out_f.root.configuration.instrument.telescope.camera.readout_LSTCam
# Check image statistics
table_in = in_f.root["/dl1/service/image_statistics"]
table_out = out_f.root["/dl1/service/image_statistics"]
for row in range(len(table_in)):
assert table_out.cols.counts[row] == np.multiply(
table_in.cols.counts[row], 2
)
assert table_out.cols.cumulative_counts[row] == np.multiply(
table_in.cols.cumulative_counts[row], 2
)
# Check telescope tables
if out_file == out_tels_dir_pattern:
telescope_nodes = {
"/dl1/monitoring/telescope/pointing",
"/dl1/event/telescope/images",
"/dl1/event/telescope/parameters",
}
for node in telescope_nodes:
assert len(out_f.list_nodes(node)) == 2
for tel_name in {"tel_002", "tel_003"}:
assert len(out_f.root[node + "/" + tel_name]) == np.multiply(
len(in_f.root[node + "/" + tel_name]), 2
)
continue
for tel in in_f.root.dl1.monitoring.telescope.pointing:
assert len(
out_f.root.dl1.monitoring.telescope.pointing[tel.name]
) == np.multiply(
len(in_f.root.dl1.monitoring.telescope.pointing[tel.name]), 2
)
if out_file != out_skip_images:
for tel in in_f.root.dl1.event.telescope.images:
assert len(
out_f.root.dl1.event.telescope.images[tel.name]
) == np.multiply(
len(in_f.root.dl1.event.telescope.images[tel.name]), 2
)
if out_file != out_skip_parameters:
for tel in in_f.root.dl1.event.telescope.parameters:
assert len(
out_f.root.dl1.event.telescope.parameters[tel.name]
) == np.multiply(
len(in_f.root.dl1.event.telescope.parameters[tel.name]), 2
)
config = Path("./examples/stage1_config.json").absolute()
dl1b_file = tmp_dir.name + "/dl1b_from_simtel.dl1.h5"
assert (
run_tool(
Stage1Tool(),
argv=[
f"--config={config}",
f"--input={GAMMA_TEST_LARGE}",
f"--output={dl1b_file}",
"--write-parameters",
"--overwrite",
],
cwd=tmpdir,
)
== 0
)
# check tables were written
with tables.open_file(dl1b_file, mode="r") as tf:
assert tf.root.dl1
assert tf.root.dl1.event.telescope
assert tf.root.dl1.event.subarray
assert tf.root.configuration.instrument.subarray.layout
assert tf.root.configuration.instrument.telescope.optics
assert tf.root.configuration.instrument.telescope.camera.geometry_LSTCam
assert tf.root.configuration.instrument.telescope.camera.readout_LSTCam
assert tf.root.dl1.monitoring.subarray.pointing.dtype.names == (
"time",
"array_azimuth",
"array_altitude",
"array_ra",
"array_dec",
)
# check we can read telescope parameters
dl1_features = pd.read_hdf(dl1b_file, "/dl1/event/telescope/parameters/tel_001")
features = (
"obs_id",
"event_id",
"tel_id",
"hillas_intensity",
"concentration_cog",
"leakage_pixels_width_1",
)
for feature in features:
assert feature in dl1_features.columns
dl1a_file = tmp_dir.name + "/dl1a_from_simtel.dl1.h5"
assert (
run_tool(
Stage1Tool(),
argv=[
f"--config={config}",
f"--input={GAMMA_TEST_LARGE}",
f"--output={dl1a_file}",
"--write-images",
"--overwrite",
],
cwd=tmpdir,
)
== 0
)
with tables.open_file(dl1a_file, mode="r") as tf:
assert tf.root.dl1
assert tf.root.dl1.event.telescope
assert tf.root.dl1.event.subarray
assert tf.root.configuration.instrument.subarray.layout
assert tf.root.configuration.instrument.telescope.optics
assert tf.root.configuration.instrument.telescope.camera.geometry_LSTCam
assert tf.root.configuration.instrument.telescope.camera.readout_LSTCam
assert tf.root.dl1.event.telescope.images.tel_001
dl1_image = tf.root.dl1.event.telescope.images.tel_001
assert "image_mask" in dl1_image.dtype.names
assert "image" in dl1_image.dtype.names
assert "peak_time" in dl1_image.dtype.names
def test_merge(tmpdir):
from ctapipe.tools.dl1_merge import MergeTool
from ctapipe.tools.stage1 import Stage1Tool
config = Path("./examples/stage1_config.json").absolute()
with tempfile.NamedTemporaryFile(
suffix=".hdf5") as f1, tempfile.NamedTemporaryFile(
suffix=".hdf5") as f2, tempfile.NamedTemporaryFile(
suffix=".hdf5") as out_all, tempfile.NamedTemporaryFile(
suffix=".hdf5"
) as out_skip_images, tempfile.NamedTemporaryFile(
suffix=".hdf5") as out_skip_parameters:
assert (run_tool(
Stage1Tool(),
argv=[
f"--config={config}",
f"--input={GAMMA_TEST_LARGE}",
f"--output={f1.name}",
"--write-parameters",
"--write-images",
"--overwrite",
],
cwd=tmpdir,
) == 0)
assert (run_tool(
Stage1Tool(),
argv=[
f"--config={config}",
f"--input={GAMMA_TEST_LARGE}",
f"--output={f2.name}",
"--write-parameters",
"--write-images",
"--overwrite",
],
cwd=tmpdir,
) == 0)
assert (run_tool(
MergeTool(),
argv=[
f"{f1.name}", f"{f2.name}", f"--o={out_all.name}",
"--overwrite"
],
cwd=tmpdir,
) == 0)
assert (run_tool(
MergeTool(),
argv=[
f"{f1.name}",
f"{f2.name}",
f"--o={out_skip_images.name}",
"--overwrite",
"--skip-images",
],
cwd=tmpdir,
) == 0)
assert (run_tool(
MergeTool(),
argv=[
f"{f1.name}",
f"{f2.name}",
f"--o={out_skip_parameters.name}",
"--overwrite",
"--skip-parameters",
],
cwd=tmpdir,
) == 0)
out_files_list = [
out_all.name, out_skip_images.name, out_skip_parameters.name
]
for out_file in out_files_list:
with tables.open_file(out_file,
mode="r") as out_f, tables.open_file(
f1.name, mode="r") as in_f:
# Check expanded tables
assert len(
out_f.root.simulation.service.shower_distribution) == 2
assert len(out_f.root.simulation.event.subarray.shower) == 220
assert len(out_f.root.configuration.simulation.run) == 2
assert len(out_f.root.dl1.monitoring.subarray.pointing) == 2
assert len(out_f.root.dl1.event.subarray.trigger) == 220
assert len(out_f.root.dl1.event.telescope.trigger) == 918
assert len(
out_f.root.simulation.service.shower_distribution) == 2
# Check subarray and service meta
assert out_f.root.dl1.service[
"image_statistics.__table_column_meta__"]
assert out_f.root.configuration.instrument.subarray.layout
assert out_f.root.configuration.instrument.telescope.optics
assert (out_f.root.configuration.instrument.telescope.camera.
geometry_LSTCam)
assert (out_f.root.configuration.instrument.telescope.camera.
readout_LSTCam)
# Check image statistics
table_in = in_f.root["/dl1/service/image_statistics"]
table_out = out_f.root["/dl1/service/image_statistics"]
for row in range(len(table_in)):
assert table_out.cols.counts[row] == np.multiply(
table_in.cols.counts[row], 2)
assert table_out.cols.cumulative_counts[
row] == np.multiply(
table_in.cols.cumulative_counts[row], 2)
# Check telescope tables
for tel in in_f.root.dl1.monitoring.telescope.pointing:
assert len(out_f.root.dl1.monitoring.telescope.pointing[
tel.name]) == np.multiply(
len(in_f.root.dl1.monitoring.telescope.pointing[
tel.name]), 2)
if out_file != out_skip_images.name:
for tel in in_f.root.dl1.event.telescope.images:
assert len(out_f.root.dl1.event.telescope.images[
tel.name]) == np.multiply(
len(in_f.root.dl1.event.telescope.images[
tel.name]), 2)
if out_file != out_skip_parameters.name:
for tel in in_f.root.dl1.event.telescope.parameters:
assert len(out_f.root.dl1.event.telescope.parameters[
tel.name]) == np.multiply(
len(in_f.root.dl1.event.telescope.parameters[
tel.name]), 2)