def test_stage1_datalevels(tmpdir):
"""test the dl1 tool on a file not providing r1 or dl0"""
from ctapipe.io import EventSource
from ctapipe.tools.stage1 import Stage1Tool
class DummyEventSource(EventSource):
@classmethod
def is_compatible(cls, path):
with open(path, "rb") as f:
dummy = f.read(5)
return dummy == b"dummy"
@property
def datalevels(self):
return (DataLevel.R0, )
@property
def is_simulation(self):
return True
@property
def obs_ids(self):
return [1]
@property
def subarray(self):
return None
def _generator(self):
return None
with tempfile.NamedTemporaryFile(mode="wb", suffix=".dummy") as f:
with tempfile.NamedTemporaryFile(mode="wb", suffix=".h5") as out:
f.write(b"dummy")
f.flush()
config = Path("./examples/stage1_config.json").absolute()
tool = Stage1Tool()
assert (run_tool(
tool,
argv=[
f"--config={config}",
f"--input={f.name}",
f"--output={out.name}",
"--write-images",
"--overwrite",
],
cwd=tmpdir,
) == 1)
# make sure the dummy event source was really used
assert isinstance(tool.event_source, DummyEventSource)
# we need to "touch" the output file again, otherwise tempfile will
# complain it no longer exists as the tool removed it
open(out.name, mode="a").close()
def test_stage1_datalevels(tmpdir):
"""test the dl1 tool on a file not providing r1, dl0 or dl1a"""
from ctapipe.io import EventSource
from ctapipe.tools.stage1 import Stage1Tool
class DummyEventSource(EventSource):
@classmethod
def is_compatible(cls, path):
with open(path, "rb") as f:
dummy = f.read(5)
return dummy == b"dummy"
@property
def datalevels(self):
return (DataLevel.R0,)
@property
def is_simulation(self):
return True
@property
def obs_ids(self):
return [1]
@property
def subarray(self):
return None
def _generator(self):
return None
dummy_file = tmp_dir.name + "/datalevels_dummy.h5"
out_file = tmp_dir.name + "/datalevels_dummy_stage1_output.h5"
with open(dummy_file, "wb") as f:
f.write(b"dummy")
f.flush()
config = Path("./examples/stage1_config.json").absolute()
tool = Stage1Tool()
assert (
run_tool(
tool,
argv=[
f"--config={config}",
f"--input={dummy_file}",
f"--output={out_file}",
"--write-images",
"--overwrite",
],
cwd=tmpdir,
)
== 1
)
# make sure the dummy event source was really used
assert isinstance(tool.event_source, DummyEventSource)
def test_export_config_to_yaml():
""" test that we can export a Tool's config to YAML"""
import yaml
from ctapipe.tools.stage1 import Stage1Tool
tool = Stage1Tool()
tool.progress_bar = True
yaml_string = export_tool_config_to_commented_yaml(tool)
# check round-trip back from yaml:
config_dict = yaml.load(yaml_string, Loader=yaml.SafeLoader)
assert config_dict["Stage1Tool"]["progress_bar"] is True
def run_stage1(input_path, cwd, output_path=None):
config = Path("./examples/stage1_config.json").absolute()
if output_path is None:
output_path = Path(
tempfile.NamedTemporaryFile(suffix=".dl1.h5",
dir=cwd).name).absolute()
ret = run_tool(
Stage1Tool(),
argv=[
f"--config={config}",
f"--input={input_path}",
f"--output={output_path}",
"--write-parameters",
"--write-images",
"--overwrite",
],
cwd=cwd,
)
assert ret == 0, "Running stage1 failed"
return output_path
def test_no_ff_tagging(tmpdir):
"""Test the ctapipe stage1 tool can read in LST real data using the event source"""
from ctapipe.tools.stage1 import Stage1Tool
from ctapipe.core.tool import run_tool
tmpdir = Path(tmpdir)
config_path = tmpdir / 'config.json'
config = {
'LSTEventSource': {
"use_flatfield_heuristic": False,
'LSTR0Corrections': {
'drs4_pedestal_path': str(test_drs4_pedestal_path),
'drs4_time_calibration_path': str(test_time_calib_path),
'calibration_path': str(test_calib_path),
},
'PointingSource': {
'drive_report_path': str(test_drive_report)
},
'EventTimeCalculator': {
'run_summary_path': str(test_run_summary),
},
},
"CameraCalibrator": {
"image_extractor_type": "LocalPeakWindowSum",
"LocalPeakWindowSum": {
"window_shift": 4,
"window_width": 8,
"apply_integration_correction": False,
}
},
"TailcutsImageCleaner": {
"picture_threshold_pe": 6,
"boundary_threshold_pe": 3,
"keep_isolated_pixels": False,
"min_picture_neighbors": 1,
}
}
with config_path.open('w') as f:
json.dump(config, f)
tool = Stage1Tool()
output = tmpdir / "test_dl1.h5"
ret = run_tool(tool,
argv=[
f'--input={test_r0_path}',
f'--output={output}',
f'--config={config_path}',
])
assert ret == 0
# test our custom default works
assert tool.event_source.r0_r1_calibrator.gain_selector.threshold == 3500
parameters = read_table(output, '/dl1/event/telescope/parameters/tel_001')
assert len(parameters) == 200
trigger = read_table(output, '/dl1/event/subarray/trigger')
# test regression of event time calculation
first_event_time = Time(59101.95035244, format='mjd', scale='tai')
assert np.all((trigger['time'] - first_event_time).to_value(u.s) < 10)
event_type_counts = np.bincount(trigger['event_type'])
# one pedestal and flat field expected each, rest should be physics data
# without ff heuristic, the ff event will have type SUBARRAY
assert event_type_counts.sum() == 200
assert event_type_counts[EventType.FLATFIELD.value] == 0
assert event_type_counts[EventType.SKY_PEDESTAL.value] == 1
assert event_type_counts[EventType.SUBARRAY.value] == 199
def test_merge(tmpdir):
from ctapipe.tools.dl1_merge import MergeTool
from ctapipe.tools.stage1 import Stage1Tool
config = Path("./examples/stage1_config.json").absolute()
tmp_dir = tempfile.TemporaryDirectory()
in_1 = tmp_dir.name + "/test_file_1.hdf5"
in_2 = tmp_dir.name + "/test_file_2.hdf5"
out_all = tmp_dir.name + "/merged_file_all.hdf5"
out_skip_images = tmp_dir.name + "/merged_file_images.hdf5"
out_skip_parameters = tmp_dir.name + "/merged_file_parameters.hdf5"
out_tels_dir_pattern = tmp_dir.name + "/merged_file_tels_dir_pattern.hdf5"
assert (
run_tool(
Stage1Tool(),
argv=[
f"--config={config}",
f"--input={GAMMA_TEST_LARGE}",
f"--output={in_1}",
"--write-parameters",
"--write-images",
"--overwrite",
],
cwd=tmpdir,
)
== 0
)
assert (
run_tool(
Stage1Tool(),
argv=[
f"--config={config}",
f"--input={GAMMA_TEST_LARGE}",
f"--output={in_2}",
"--write-parameters",
"--write-images",
"--overwrite",
],
cwd=tmpdir,
)
== 0
)
assert (
run_tool(
MergeTool(),
argv=[
f"--i={tmp_dir.name}",
"--p='test_file_*.hdf5'",
f"--o={out_tels_dir_pattern}",
"--overwrite",
"--t=[2, 3]",
],
cwd=tmpdir,
)
== 0
)
assert (
run_tool(
MergeTool(), argv=[in_1, in_2, f"--o={out_all}", "--overwrite"], cwd=tmpdir
)
== 0
)
assert (
run_tool(
MergeTool(),
argv=[in_1, in_2, f"--o={out_skip_images}", "--overwrite", "--skip-images"],
cwd=tmpdir,
)
== 0
)
assert (
run_tool(
MergeTool(),
argv=[
in_1,
in_2,
f"--o={out_skip_parameters}",
"--overwrite",
"--skip-parameters",
],
cwd=tmpdir,
)
== 0
)
out_files_list = [
out_all,
out_skip_images,
out_skip_parameters,
out_tels_dir_pattern,
]
for out_file in out_files_list:
with tables.open_file(out_file, mode="r") as out_f, tables.open_file(
in_1, mode="r"
) as in_f:
# Check expanded tables
assert len(out_f.root.simulation.service.shower_distribution) == 2
assert len(out_f.root.simulation.event.subarray.shower) == 220
assert len(out_f.root.configuration.simulation.run) == 2
assert len(out_f.root.dl1.monitoring.subarray.pointing) == 2
assert len(out_f.root.dl1.event.subarray.trigger) == 220
assert len(out_f.root.dl1.event.telescope.trigger) == 918
assert len(out_f.root.simulation.service.shower_distribution) == 2
# Check subarray and service meta
assert out_f.root.dl1.service["image_statistics.__table_column_meta__"]
assert out_f.root.configuration.instrument.subarray.layout
assert out_f.root.configuration.instrument.telescope.optics
assert out_f.root.configuration.instrument.telescope.camera.geometry_LSTCam
assert out_f.root.configuration.instrument.telescope.camera.readout_LSTCam
# Check image statistics
table_in = in_f.root["/dl1/service/image_statistics"]
table_out = out_f.root["/dl1/service/image_statistics"]
for row in range(len(table_in)):
assert table_out.cols.counts[row] == np.multiply(
table_in.cols.counts[row], 2
)
assert table_out.cols.cumulative_counts[row] == np.multiply(
table_in.cols.cumulative_counts[row], 2
)
# Check telescope tables
if out_file == out_tels_dir_pattern:
telescope_nodes = {
"/dl1/monitoring/telescope/pointing",
"/dl1/event/telescope/images",
"/dl1/event/telescope/parameters",
}
for node in telescope_nodes:
assert len(out_f.list_nodes(node)) == 2
for tel_name in {"tel_002", "tel_003"}:
assert len(out_f.root[node + "/" + tel_name]) == np.multiply(
len(in_f.root[node + "/" + tel_name]), 2
)
continue
for tel in in_f.root.dl1.monitoring.telescope.pointing:
assert len(
out_f.root.dl1.monitoring.telescope.pointing[tel.name]
) == np.multiply(
len(in_f.root.dl1.monitoring.telescope.pointing[tel.name]), 2
)
if out_file != out_skip_images:
for tel in in_f.root.dl1.event.telescope.images:
assert len(
out_f.root.dl1.event.telescope.images[tel.name]
) == np.multiply(
len(in_f.root.dl1.event.telescope.images[tel.name]), 2
)
if out_file != out_skip_parameters:
for tel in in_f.root.dl1.event.telescope.parameters:
assert len(
out_f.root.dl1.event.telescope.parameters[tel.name]
) == np.multiply(
len(in_f.root.dl1.event.telescope.parameters[tel.name]), 2
)
config = Path("./examples/stage1_config.json").absolute()
dl1b_file = tmp_dir.name + "/dl1b_from_simtel.dl1.h5"
assert (
run_tool(
Stage1Tool(),
argv=[
f"--config={config}",
f"--input={GAMMA_TEST_LARGE}",
f"--output={dl1b_file}",
"--write-parameters",
"--overwrite",
],
cwd=tmpdir,
)
== 0
)
# check tables were written
with tables.open_file(dl1b_file, mode="r") as tf:
assert tf.root.dl1
assert tf.root.dl1.event.telescope
assert tf.root.dl1.event.subarray
assert tf.root.configuration.instrument.subarray.layout
assert tf.root.configuration.instrument.telescope.optics
assert tf.root.configuration.instrument.telescope.camera.geometry_LSTCam
assert tf.root.configuration.instrument.telescope.camera.readout_LSTCam
assert tf.root.dl1.monitoring.subarray.pointing.dtype.names == (
"time",
"array_azimuth",
"array_altitude",
"array_ra",
"array_dec",
)
# check we can read telescope parameters
dl1_features = pd.read_hdf(dl1b_file, "/dl1/event/telescope/parameters/tel_001")
features = (
"obs_id",
"event_id",
"tel_id",
"hillas_intensity",
"concentration_cog",
"leakage_pixels_width_1",
)
for feature in features:
assert feature in dl1_features.columns
dl1a_file = tmp_dir.name + "/dl1a_from_simtel.dl1.h5"
assert (
run_tool(
Stage1Tool(),
argv=[
f"--config={config}",
f"--input={GAMMA_TEST_LARGE}",
f"--output={dl1a_file}",
"--write-images",
"--overwrite",
],
cwd=tmpdir,
)
== 0
)
with tables.open_file(dl1a_file, mode="r") as tf:
assert tf.root.dl1
assert tf.root.dl1.event.telescope
assert tf.root.dl1.event.subarray
assert tf.root.configuration.instrument.subarray.layout
assert tf.root.configuration.instrument.telescope.optics
assert tf.root.configuration.instrument.telescope.camera.geometry_LSTCam
assert tf.root.configuration.instrument.telescope.camera.readout_LSTCam
assert tf.root.dl1.event.telescope.images.tel_001
dl1_image = tf.root.dl1.event.telescope.images.tel_001
assert "image_mask" in dl1_image.dtype.names
assert "image" in dl1_image.dtype.names
assert "peak_time" in dl1_image.dtype.names
def test_stage_1_dl1(tmpdir, dl1_image_file, dl1_parameters_file):
from ctapipe.tools.stage1 import Stage1Tool
config = Path("./examples/stage1_config.json").absolute()
# DL1A file as input
dl1b_from_dl1a_file = tmp_dir.name + "/dl1b_from dl1a.dl1.h5"
assert (
run_tool(
Stage1Tool(),
argv=[
f"--config={config}",
f"--input={dl1_image_file}",
f"--output={dl1b_from_dl1a_file}",
"--write-parameters",
"--overwrite",
],
cwd=tmpdir,
)
== 0
)
# check tables were written
with tables.open_file(dl1b_from_dl1a_file, mode="r") as tf:
assert tf.root.dl1
assert tf.root.dl1.event.telescope
assert tf.root.dl1.event.subarray
assert tf.root.configuration.instrument.subarray.layout
assert tf.root.configuration.instrument.telescope.optics
assert tf.root.configuration.instrument.telescope.camera.geometry_LSTCam
assert tf.root.configuration.instrument.telescope.camera.readout_LSTCam
assert tf.root.dl1.monitoring.subarray.pointing.dtype.names == (
"time",
"array_azimuth",
"array_altitude",
"array_ra",
"array_dec",
)
# check we can read telescope parameters
dl1_features = pd.read_hdf(
dl1b_from_dl1a_file, "/dl1/event/telescope/parameters/tel_001"
)
features = (
"obs_id",
"event_id",
"tel_id",
"hillas_intensity",
"concentration_cog",
"leakage_pixels_width_1",
)
for feature in features:
assert feature in dl1_features.columns
# DL1B file as input
assert (
run_tool(
Stage1Tool(),
argv=[
f"--config={config}",
f"--input={dl1_parameters_file}",
f"--output={tmp_dir.name + '/dl1b_from_dl1b.dl1.h5'}",
"--write-parameters",
"--overwrite",
],
cwd=tmpdir,
)
== 1
)
def test_merge(tmpdir):
from ctapipe.tools.dl1_merge import MergeTool
from ctapipe.tools.stage1 import Stage1Tool
config = Path("./examples/stage1_config.json").absolute()
with tempfile.NamedTemporaryFile(
suffix=".hdf5") as f1, tempfile.NamedTemporaryFile(
suffix=".hdf5") as f2, tempfile.NamedTemporaryFile(
suffix=".hdf5") as out_all, tempfile.NamedTemporaryFile(
suffix=".hdf5"
) as out_skip_images, tempfile.NamedTemporaryFile(
suffix=".hdf5") as out_skip_parameters:
assert (run_tool(
Stage1Tool(),
argv=[
f"--config={config}",
f"--input={GAMMA_TEST_LARGE}",
f"--output={f1.name}",
"--write-parameters",
"--write-images",
"--overwrite",
],
cwd=tmpdir,
) == 0)
assert (run_tool(
Stage1Tool(),
argv=[
f"--config={config}",
f"--input={GAMMA_TEST_LARGE}",
f"--output={f2.name}",
"--write-parameters",
"--write-images",
"--overwrite",
],
cwd=tmpdir,
) == 0)
assert (run_tool(
MergeTool(),
argv=[
f"{f1.name}", f"{f2.name}", f"--o={out_all.name}",
"--overwrite"
],
cwd=tmpdir,
) == 0)
assert (run_tool(
MergeTool(),
argv=[
f"{f1.name}",
f"{f2.name}",
f"--o={out_skip_images.name}",
"--overwrite",
"--skip-images",
],
cwd=tmpdir,
) == 0)
assert (run_tool(
MergeTool(),
argv=[
f"{f1.name}",
f"{f2.name}",
f"--o={out_skip_parameters.name}",
"--overwrite",
"--skip-parameters",
],
cwd=tmpdir,
) == 0)
out_files_list = [
out_all.name, out_skip_images.name, out_skip_parameters.name
]
for out_file in out_files_list:
with tables.open_file(out_file,
mode="r") as out_f, tables.open_file(
f1.name, mode="r") as in_f:
# Check expanded tables
assert len(
out_f.root.simulation.service.shower_distribution) == 2
assert len(out_f.root.simulation.event.subarray.shower) == 220
assert len(out_f.root.configuration.simulation.run) == 2
assert len(out_f.root.dl1.monitoring.subarray.pointing) == 2
assert len(out_f.root.dl1.event.subarray.trigger) == 220
assert len(out_f.root.dl1.event.telescope.trigger) == 918
assert len(
out_f.root.simulation.service.shower_distribution) == 2
# Check subarray and service meta
assert out_f.root.dl1.service[
"image_statistics.__table_column_meta__"]
assert out_f.root.configuration.instrument.subarray.layout
assert out_f.root.configuration.instrument.telescope.optics
assert (out_f.root.configuration.instrument.telescope.camera.
geometry_LSTCam)
assert (out_f.root.configuration.instrument.telescope.camera.
readout_LSTCam)
# Check image statistics
table_in = in_f.root["/dl1/service/image_statistics"]
table_out = out_f.root["/dl1/service/image_statistics"]
for row in range(len(table_in)):
assert table_out.cols.counts[row] == np.multiply(
table_in.cols.counts[row], 2)
assert table_out.cols.cumulative_counts[
row] == np.multiply(
table_in.cols.cumulative_counts[row], 2)
# Check telescope tables
for tel in in_f.root.dl1.monitoring.telescope.pointing:
assert len(out_f.root.dl1.monitoring.telescope.pointing[
tel.name]) == np.multiply(
len(in_f.root.dl1.monitoring.telescope.pointing[
tel.name]), 2)
if out_file != out_skip_images.name:
for tel in in_f.root.dl1.event.telescope.images:
assert len(out_f.root.dl1.event.telescope.images[
tel.name]) == np.multiply(
len(in_f.root.dl1.event.telescope.images[
tel.name]), 2)
if out_file != out_skip_parameters.name:
for tel in in_f.root.dl1.event.telescope.parameters:
assert len(out_f.root.dl1.event.telescope.parameters[
tel.name]) == np.multiply(
len(in_f.root.dl1.event.telescope.parameters[
tel.name]), 2)
def test_stage_1(tmpdir):
from ctapipe.tools.stage1 import Stage1Tool
config = Path("./examples/stage1_config.json").absolute()
with tempfile.NamedTemporaryFile(suffix=".hdf5") as f:
assert (run_tool(
Stage1Tool(),
argv=[
f"--config={config}",
f"--input={GAMMA_TEST_LARGE}",
f"--output={f.name}",
"--write-parameters",
"--overwrite",
],
cwd=tmpdir,
) == 0)
# check tables were written
with tables.open_file(f.name, mode="r") as tf:
assert tf.root.dl1
assert tf.root.dl1.event.telescope
assert tf.root.dl1.event.subarray
assert tf.root.configuration.instrument.subarray.layout
assert tf.root.configuration.instrument.telescope.optics
assert tf.root.configuration.instrument.telescope.camera.geometry_LSTCam
assert tf.root.configuration.instrument.telescope.camera.readout_LSTCam
assert tf.root.dl1.monitoring.subarray.pointing.dtype.names == (
"time",
"array_azimuth",
"array_altitude",
"array_ra",
"array_dec",
)
# check we can read telescope parametrs
dl1_features = pd.read_hdf(f.name,
"/dl1/event/telescope/parameters/tel_001")
features = (
"obs_id",
"event_id",
"tel_id",
"hillas_intensity",
"concentration_cog",
"leakage_pixels_width_1",
)
for feature in features:
assert feature in dl1_features.columns
with tempfile.NamedTemporaryFile(suffix=".hdf5") as f:
assert (run_tool(
Stage1Tool(),
argv=[
f"--config={config}",
f"--input={GAMMA_TEST_LARGE}",
f"--output={f.name}",
"--write-images",
"--overwrite",
],
cwd=tmpdir,
) == 0)
with tables.open_file(f.name, mode="r") as tf:
assert tf.root.dl1
assert tf.root.dl1.event.telescope
assert tf.root.dl1.event.subarray
assert tf.root.configuration.instrument.subarray.layout
assert tf.root.configuration.instrument.telescope.optics
assert tf.root.configuration.instrument.telescope.camera.geometry_LSTCam
assert tf.root.configuration.instrument.telescope.camera.readout_LSTCam
assert tf.root.dl1.event.telescope.images.tel_001
dl1_image = tf.root.dl1.event.telescope.images.tel_001
assert "image_mask" in dl1_image.dtype.names
assert "image" in dl1_image.dtype.names
assert "peak_time" in dl1_image.dtype.names
