def dl1_muon_file(dl1_tmp_path):
"""
DL1 file containing only images from a muon simulation set.
"""
from ctapipe.core import run_tool
from ctapipe.tools.process import ProcessorTool
output = dl1_tmp_path / "muons.dl1.h5"
# prevent running process multiple times in case of parallel tests
with FileLock(output.with_suffix(output.suffix + ".lock")):
if output.is_file():
return output
infile = get_dataset_path("lst_muons.simtel.zst")
argv = [
f"--input={infile}",
f"--output={output}",
"--write-images",
"--DataWriter.write_parameters=False",
"--DataWriter.Contact.name=αℓℓ the äüöß",
"--SimTelEventSource.focal_length_choice=nominal",
]
assert run_tool(ProcessorTool(), argv=argv, cwd=dl1_tmp_path) == 0
return output
def test_training_from_simtel(tmp_path):
""" check we can write both dl1 and dl2 info (e.g. for training input) """
config = Path("./examples/training_config.json").absolute()
output = tmp_path / "test_training.DL1DL2.h5"
assert (
run_tool(
ProcessorTool(),
argv=[
f"--config={config}",
f"--input={GAMMA_TEST_LARGE}",
f"--output={output}",
"--max-events=5",
"--overwrite",
],
cwd=tmp_path,
)
== 0
)
# check tables were written
with tables.open_file(output, mode="r") as testfile:
assert testfile.root.dl1.event.telescope.parameters.tel_002
assert testfile.root.dl2.event.subarray.geometry.HillasReconstructor
def test_image_modifications(tmp_path, dl1_image_file):
"""
Test that running ctapipe-process with an ImageModifier set
produces a file with different images.
"""
unmodified_images = read_table(dl1_image_file,
"/dl1/event/telescope/images/tel_025")
noise_config = resource_file("image_modification_config.json")
dl1_modified = tmp_path / "dl1_modified.dl1.h5"
assert (run_tool(
ProcessorTool(),
argv=[
f"--config={noise_config}",
f"--input={dl1_image_file}",
f"--output={dl1_modified}",
"--write-parameters",
"--overwrite",
],
cwd=tmp_path,
) == 0)
modified_images = read_table(dl1_modified,
"/dl1/event/telescope/images/tel_025")
# Test that significantly more light is recorded (bias in dim pixels)
assert modified_images["image"].sum() / unmodified_images["image"].sum(
) > 1.5
def test_stage1():
"""Test the ctapipe stage1 tool can read in LST real data using the event source"""
from ctapipe.tools.process import ProcessorTool
from ctapipe.core import run_tool
tool = ProcessorTool()
output = str(test_cal_path).replace(".root", ".h5")
ret = run_tool(tool, argv=[
f'--input={test_cal_path}',
f'--output={output}',
f'--config={str(config)}',
"--camera-frame",
])
assert ret == 0
parameters = read_table(output, '/dl1/event/telescope/parameters/tel_001')
assert len(parameters) == 458
trigger = read_table(output, '/dl1/event/subarray/trigger')
event_type_counts = np.bincount(trigger['event_type'])
# no pedestals expected, should be only physics data
assert event_type_counts.sum() == 458
assert event_type_counts[EventType.FLATFIELD.value] == 0
assert event_type_counts[EventType.SKY_PEDESTAL.value] == 0
assert event_type_counts[EventType.SUBARRAY.value] == 458
def test_stage1_datalevels(tmp_path):
"""test the dl1 tool on a file not providing r1, dl0 or dl1a"""
class DummyEventSource(EventSource):
""" for testing """
@staticmethod
def is_compatible(file_path):
with open(file_path, "rb") as infile:
dummy = infile.read(5)
return dummy == b"dummy"
@property
def datalevels(self):
return (DataLevel.R0,)
@property
def is_simulation(self):
return True
@property
def obs_ids(self):
return [1]
@property
def subarray(self):
return None
def _generator(self):
return None
dummy_file = tmp_path / "datalevels_dummy.h5"
out_file = tmp_path / "datalevels_dummy_stage1_output.h5"
with open(dummy_file, "wb") as infile:
infile.write(b"dummy")
infile.flush()
config = Path("./examples/stage1_config.json").absolute()
tool = ProcessorTool()
assert (
run_tool(
tool,
argv=[
f"--config={config}",
f"--input={dummy_file}",
f"--output={out_file}",
"--write-images",
"--overwrite",
],
cwd=tmp_path,
)
== 1
)
# make sure the dummy event source was really used
assert isinstance(tool.event_source, DummyEventSource)
def test_export_config_to_yaml():
""" test that we can export a Tool's config to YAML"""
import yaml
from ctapipe.tools.process import ProcessorTool
tool = ProcessorTool()
tool.progress_bar = True
yaml_string = export_tool_config_to_commented_yaml(tool)
# check round-trip back from yaml:
config_dict = yaml.load(yaml_string, Loader=yaml.SafeLoader)
assert config_dict["ProcessorTool"]["progress_bar"] is True
def test_read_yaml_toml_json_config(dl1_image_file, config_files):
"""check that we can read multiple formats of config file"""
tool = ProcessorTool()
for config_base in config_files:
config = resource_file(config_base)
tool.load_config_file(config)
tool.config.EventSource.input_url = dl1_image_file
tool.config.DataWriter.overwrite = True
tool.setup()
assert (tool.get_current_config()["ProcessorTool"]["DataWriter"]
["contact_info"].name == "YOUR-NAME-HERE")
def test_multiple_configs(dl1_image_file):
"""ensure a config file loaded later overwrites keys from an earlier one"""
tool = ProcessorTool()
tool.load_config_file(resource_file("base_config.yaml"))
tool.load_config_file(resource_file("stage2_config.yaml"))
tool.config.EventSource.input_url = dl1_image_file
tool.config.DataWriter.overwrite = True
tool.setup()
# ensure the overwriting works (base config has this option disabled)
assert (tool.get_current_config()["ProcessorTool"]["DataWriter"]
["write_showers"] is True)
def test_stage_2_from_dl1_images(tmp_path, dl1_image_file):
"""check we can go to DL2 geometry from DL1 images"""
config = resource_file("stage2_config.json")
output = tmp_path / "test_stage2_from_dl1image.DL2.h5"
assert (run_tool(
ProcessorTool(),
argv=[
f"--config={config}",
f"--input={dl1_image_file}",
f"--output={output}",
"--overwrite",
],
cwd=tmp_path,
) == 0)
# check tables were written
with tables.open_file(output, mode="r") as testfile:
assert testfile.root.dl2.event.subarray.geometry.HillasReconstructor
def dl1_proton_file(dl1_tmp_path, prod5_proton_simtel_path):
"""
DL1 file containing images and parameters for a prod5 proton run
"""
from ctapipe.tools.process import ProcessorTool
from ctapipe.core import run_tool
output = dl1_tmp_path / "proton.dl1.h5"
with FileLock(output.with_suffix(output.suffix + ".lock")):
if output.is_file():
return output
argv = [
f"--input={prod5_proton_simtel_path}",
f"--output={output}",
"--write-images",
]
assert run_tool(ProcessorTool(), argv=argv, cwd=dl1_tmp_path) == 0
return output
def test_stage_2_from_simtel(tmp_path):
""" check we can go to DL2 geometry from simtel file """
config = files("ctapipe.tools.tests.resources").joinpath(
"stage2_config.json")
output = tmp_path / "test_stage2_from_simtel.DL2.h5"
assert (run_tool(
ProcessorTool(),
argv=[
f"--config={config}",
f"--input={GAMMA_TEST_LARGE}",
f"--output={output}",
"--max-events=5",
"--overwrite",
],
cwd=tmp_path,
) == 0)
# check tables were written
with tables.open_file(output, mode="r") as testfile:
assert testfile.root.dl2.event.subarray.geometry.HillasReconstructor
def dl1_parameters_file(dl1_tmp_path, prod5_gamma_simtel_path):
"""
DL1 File containing only parameters (DL1B) from a gamma simulation set.
"""
from ctapipe.core import run_tool
from ctapipe.tools.process import ProcessorTool
output = dl1_tmp_path / "gamma_parameters.dl1.h5"
# prevent running process multiple times in case of parallel tests
with FileLock(output.with_suffix(output.suffix + ".lock")):
if output.is_file():
return output
argv = [
f"--input={prod5_gamma_simtel_path}",
f"--output={output}",
"--write-parameters",
"--max-events=20",
]
assert run_tool(ProcessorTool(), argv=argv, cwd=dl1_tmp_path) == 0
return output
def run_stage1(input_path, cwd, output_path=None):
config = Path("./examples/stage1_config.json").absolute()
if output_path is None:
output_path = Path(
tempfile.NamedTemporaryFile(suffix=".dl1.h5", dir=cwd).name
).absolute()
ret = run_tool(
ProcessorTool(),
argv=[
f"--config={config}",
f"--input={input_path}",
f"--output={output_path}",
"--write-parameters",
"--write-images",
"--overwrite",
"--max-events=5",
],
cwd=cwd,
)
assert ret == 0, "Running stage1 failed"
def dl1_by_type_file(dl1_tmp_path, prod5_gamma_simtel_path):
"""
DL1 file containing both images and parameters from a gamma simulation set.
"""
from ctapipe.tools.process import ProcessorTool
from ctapipe.core import run_tool
output = dl1_tmp_path / "gamma_by_type.dl1.h5"
# prevent running stage1 multiple times in case of parallel tests
with FileLock(output.with_suffix(output.suffix + ".lock")):
if output.is_file():
return output
argv = [
f"--input={prod5_gamma_simtel_path}",
f"--output={output}",
"--write-images",
"--max-events=20",
"--DataWriter.split_datasets_by=tel_type",
]
assert run_tool(ProcessorTool(), argv=argv, cwd=dl1_tmp_path) == 0
return output
def dl2_proton_geometry_file(dl2_tmp_path, prod5_proton_simtel_path):
"""
File containing both parameters and shower geometry from a gamma simulation set.
"""
from ctapipe.core import run_tool
from ctapipe.tools.process import ProcessorTool
output = dl2_tmp_path / "proton.training.h5"
# prevent running process multiple times in case of parallel tests
with FileLock(output.with_suffix(output.suffix + ".lock")):
if output.is_file():
return output
argv = [
f"--input={prod5_proton_simtel_path}",
f"--output={output}",
"--write-images",
"--write-stereo-shower",
"--max-events=20",
]
assert run_tool(ProcessorTool(), argv=argv, cwd=dl2_tmp_path) == 0
return output
def test_stage_1_dl1(tmp_path, dl1_image_file, dl1_parameters_file):
""" check simtel to DL1 conversion """
config = Path("./examples/stage1_config.json").absolute()
# DL1A file as input
dl1b_from_dl1a_file = tmp_path / "dl1b_fromdl1a.dl1.h5"
assert (
run_tool(
ProcessorTool(),
argv=[
f"--config={config}",
f"--input={dl1_image_file}",
f"--output={dl1b_from_dl1a_file}",
"--write-parameters",
"--overwrite",
],
cwd=tmp_path,
)
== 0
)
# check tables were written
with tables.open_file(dl1b_from_dl1a_file, mode="r") as testfile:
assert testfile.root.dl1
assert testfile.root.dl1.event.telescope
assert testfile.root.dl1.event.subarray
assert testfile.root.configuration.instrument.subarray.layout
assert testfile.root.configuration.instrument.telescope.optics
assert testfile.root.configuration.instrument.telescope.camera.geometry_0
assert testfile.root.configuration.instrument.telescope.camera.readout_0
assert testfile.root.dl1.monitoring.subarray.pointing.dtype.names == (
"time",
"array_azimuth",
"array_altitude",
"array_ra",
"array_dec",
)
# check we can read telescope parameters
dl1_features = pd.read_hdf(
dl1b_from_dl1a_file, "/dl1/event/telescope/parameters/tel_025"
)
features = (
"obs_id",
"event_id",
"tel_id",
"hillas_intensity",
"concentration_cog",
"leakage_pixels_width_1",
)
for feature in features:
assert feature in dl1_features.columns
# DL1B file as input
assert (
run_tool(
ProcessorTool(),
argv=[
f"--config={config}",
f"--input={dl1_parameters_file}",
f"--output={tmp_path}/dl1b_from_dl1b.dl1.h5",
"--write-parameters",
"--overwrite",
],
cwd=tmp_path,
)
== 1
)