def test_dl2(tmp_path, dl2_shower_geometry_file, dl2_proton_geometry_file):
from ctapipe.tools.merge import MergeTool
output = tmp_path / "merged.dl2.h5"
ret = run_tool(
MergeTool(),
argv=[
f"--output={output}",
str(dl2_shower_geometry_file),
str(dl2_proton_geometry_file),
],
)
assert ret == 0, f"Running merge for dl2 files failed with exit code {ret}"
table1 = read_table(dl2_shower_geometry_file,
"/dl2/event/subarray/geometry/HillasReconstructor")
table2 = read_table(dl2_proton_geometry_file,
"/dl2/event/subarray/geometry/HillasReconstructor")
table_merged = read_table(
output, "/dl2/event/subarray/geometry/HillasReconstructor")
diff = StringIO()
identical = report_diff_values(vstack([table1, table2]),
table_merged,
fileobj=diff)
assert (
identical
), f"Merged table not equal to individual tables. Diff:\n {diff.getvalue()}"
def test_read_table(tmp_path):
# write a simple hdf5 file using
container = ReconstructedEnergyContainer()
filename = tmp_path / "test_astropy_table.h5"
with HDF5TableWriter(filename) as writer:
for energy in np.logspace(1, 2, 10) * u.TeV:
container.energy = energy
writer.write("events", container)
# try opening the result
table = read_table(filename, "/events")
assert "energy" in table.columns
assert table["energy"].unit == u.TeV
assert "CTAPIPE_VERSION" in table.meta
assert table["energy"].description is not None
# test using a string
table = read_table(str(filename), "/events")
# test write the table back out to some other format:
table.write(tmp_path / "test_output.ecsv")
table.write(tmp_path / "test_output.fits.gz")
# test using a file handle
with tables.open_file(filename) as handle:
table = read_table(handle, "/events")
# test a bad input
with pytest.raises(ValueError):
table = read_table(12345, "/events")
def test_file_closed(tmp_path):
"""Test read_table closes the file even when an exception happens during read"""
path = tmp_path / "empty.hdf5"
with tables.open_file(path, "w"):
pass
with pytest.raises(tables.NoSuchNodeError):
read_table(path, "/foo")
# test we can open the file for writing, fails if read_table did not close
# the file
with tables.open_file(path, "w"):
pass
def test_write_table(tmp_path):
"""Test write_table"""
from ctapipe.io.astropy_helpers import write_table, read_table
table = Table({
"a": [1, 2, 3],
"b": np.array([1, 2, 3], dtype=np.uint16),
"speed": [2.0, 3.0, 4.2] * (u.m / u.s),
"time": Time([58e3, 59e3, 60e3], format="mjd"),
"name": ["a", "bb", "ccc"],
})
table.meta["FOO"] = "bar"
table["speed"].description = "Speed of stuff"
output_path = tmp_path = tmp_path / "table.h5"
table_path = "/foo/bar"
write_table(table, output_path, table_path)
read = read_table(output_path, table_path)
for name, column in table.columns.items():
assert name in read.colnames, f"Column {name} not found in output file"
assert (read.dtype[name] == table.dtype[name]
), f"Column {name} dtype different in output file"
# time conversion is not lossless
if name == "time":
assert np.allclose(column.tai.mjd, read[name].tai.mjd)
else:
assert np.all(
column == read[name]), f"Column {name} differs after reading"
assert "FOO" in read.meta
assert read.meta["FOO"] == "bar"
assert read["speed"].description == "Speed of stuff"
# test error for already existing table
with pytest.raises(IOError):
write_table(table, output_path, table_path)
# test we can append
write_table(table, output_path, table_path, append=True)
read = read_table(output_path, table_path)
assert len(read) == 2 * len(table)
# test we can overwrite
write_table(table, output_path, table_path, overwrite=True)
assert len(read_table(output_path, table_path)) == len(table)
def test_skip_images(tmp_path, dl1_file, dl1_proton_file):
from ctapipe.tools.merge import MergeTool
# create a second file so we can test the patterns
output = tmp_path / "merged_no_images.dl1.h5"
ret = run_tool(
MergeTool(),
argv=[
str(dl1_file),
str(dl1_proton_file),
f"--output={output}",
"--skip-images",
"--overwrite",
],
cwd=tmp_path,
)
assert ret == 0
with tables.open_file(output, "r") as f:
assert "images" not in f.root.dl1.event.telescope
assert "images" in f.root.simulation.event.telescope
assert "parameters" in f.root.dl1.event.telescope
t = read_table(output, "/simulation/event/telescope/images/tel_001")
assert "true_image" not in t.colnames
assert "true_image_sum" in t.colnames
def test_read_table_slicing(tmp_path):
filename = tmp_path / "test_slicing.h5"
# write a simple hdf5 file using
class Data(Container):
index = Field(0)
value = Field(0.0)
rng = np.random.default_rng(0)
values = rng.normal(size=100)
index = np.arange(len(values))
with HDF5TableWriter(filename) as writer:
for i, value in zip(index, values):
container = Data(index=i, value=value)
writer.write("events", container)
# try opening the result
table = read_table(filename, "/events", start=50)
assert len(table) == 50
assert np.all(table["index"] == index[50:])
assert np.all(table["value"] == values[50:])
table = read_table(filename, "/events", stop=50)
assert len(table) == 50
assert np.all(table["index"] == index[:50])
assert np.all(table["value"] == values[:50])
table = read_table(filename, "/events", start=10, stop=30)
assert len(table) == 20
assert np.all(table["index"] == index[10:30])
assert np.all(table["value"] == values[10:30])
table = read_table(filename, "/events", step=5)
assert len(table) == 20
assert np.all(table["index"] == index[::5])
assert np.all(table["value"] == values[::5])
def test_transforms(tmp_path):
path = tmp_path / "test_trans.hdf5"
data = np.array([100, 110], dtype="int16").view([("waveform", "int16")])
print(data)
with tables.open_file(path, "w") as f:
f.create_table("/data", "test", obj=data, createparents=True)
f.root.data.test.attrs["waveform_TRANSFORM_SCALE"] = 100.0
f.root.data.test.attrs["waveform_TRANSFORM_OFFSET"] = 200
f.root.data.test.attrs["waveform_TRANSFORM_DTYPE"] = "float64"
table = read_table(path, "/data/test")
assert np.all(table["waveform"] == [-1.0, -0.9])
def test_read_table_time(tmp_path):
t0 = Time("2020-01-01T20:00:00.0")
times = t0 + np.arange(10) * u.s
# use table writer to write test file
filename = tmp_path / "test_astropy_table.h5"
with HDF5TableWriter(filename) as writer:
for t in times:
container = TelescopeTriggerContainer(time=t, n_trigger_pixels=10)
writer.write("events", container)
# check reading in the times works as expected
table = read_table(filename, "/events")
assert isinstance(table["time"], Time)
assert np.allclose(times.tai.mjd, table["time"].tai.mjd)
def test_condition(tmp_path):
# write a simple hdf5 file using
container = ReconstructedEnergyContainer()
filename = tmp_path / "test_astropy_table.h5"
with HDF5TableWriter(filename) as writer:
for energy in [np.nan, 100, np.nan, 50, -1.0] * u.TeV:
container.energy = energy
writer.write("events", container)
# try opening the result
table = read_table(filename, "/events", condition="energy > 0")
assert len(table) == 2
assert np.all(table["energy"] == [100, 50] * u.TeV)