def test_name(self):
tab = self.arr.view(Table)
assert tab.name == DEFAULT_NAME
tab = Table(self.arr)
assert tab.name == DEFAULT_NAME
tab = Table(self.arr, name="foo")
assert tab.name == "foo"
def extract_blob(self):
try:
length = unpack("<i", self.file.read(4))[0]
except struct.error:
raise StopIteration
blob = Blob()
blob["PacketInfo"] = Table(
{
"data_type": b"".join(unpack("cccc",
self.file.read(4))).decode(),
"run": unpack(">i", self.file.read(4))[0],
"udp_sequence": unpack(">i", self.file.read(4))[0],
"timestamp": unpack(">I", self.file.read(4))[0],
"ns_ticks": unpack(">I", self.file.read(4))[0],
"dom_id": unpack(">i", self.file.read(4))[0],
"dom_status": unpack(">I", self.file.read(4))[0],
},
h5loc="/packet_info",
split_h5=True,
name="UDP Packet Info",
)
remaining_length = length - 7 * 4
pmt_data = []
count = remaining_length // self.pmt_dt.itemsize
pmt_data = np.fromfile(self.file, dtype=self.pmt_dt, count=count)
blob["Hits"] = Table(pmt_data, h5loc="/hits", split_h5=True)
return blob
def test_h5loc(self):
tab = self.arr.view(Table)
assert tab.h5loc == DEFAULT_H5LOC
tab = Table(self.arr)
assert tab.h5loc == DEFAULT_H5LOC
tab = Table(self.arr, h5loc="/foo")
assert tab.h5loc == "/foo"
def test_add_two_tables_with_different_lengths_and_columns(self):
tab1 = Table({"a": [1, 2], "b": [100, 200]})
tab2 = Table({"a": [3, 4, 5], "b": [300, 400, 500]})
added_tab = tab1 + tab2
assert 5 == len(added_tab)
self.assertListEqual([1, 2, 3, 4, 5], list(added_tab.a))
self.assertListEqual([100, 200, 300, 400, 500], list(added_tab.b))
def test_reorder_dtypes_with_differing_names_raises(self):
dtype = np.dtype([("a", "<i8"), ("c", "<i8"), ("b", "<f8")])
dtype_reordered = np.dtype([("b", "<f8"), ("a", "<i8")])
tab = Table({"a": 1, "b": 2.5, "c": 3}, dtype=dtype)
with self.assertRaises(ValueError):
tab2 = Table(tab, dtype=dtype_reordered)
def test_split(self):
tab = self.arr.view(Table)
assert tab.split_h5 is False
tab = Table(self.arr)
assert tab.split_h5 is False
tab = Table(self.arr, split_h5=True)
assert tab.split_h5
def test_merge_different_columns_with_no_nan_compatible_dtype_even_if_fillna(
self):
tab1 = Table({"a": [1]}, h5loc="/a", h5singleton=True)
tab2 = Table({"b": [2]})
tab3 = Table({"c": [3]})
with self.assertRaises(ValueError):
merged_tab = Table.merge([tab1, tab2, tab3], fillna=True)
def test_from_record(self):
t = Table({"a": [0, 1, 2], "b": [10, 20, 30]})
t2 = Table(t[1])
assert 1 == t2[0].a
assert 1 == t2.a[0]
assert 20 == t2[0].b
assert 20 == t2.b[0]
def test_cut4d(self):
point4d = Table({"pos_x": [0], "pos_y": [3], "pos_z": [0], "t": [20]})
items = Table({
"pos_x": [0, 10, 0, 20, 0],
"pos_y": [10, 0, 0, 0, 30],
"pos_z": [0, 0, 10, 0, 0],
"time": [60, 15, 40, 20, 100],
})
tmin = -50.0
tmax = 10.0
rmin = 3.0
rmax = 80.0
selected_items = cut4d(point4d, tmin, tmax, rmin, rmax, items)
assert len(selected_items) == 3
self.assertListEqual(
list(np.array([0, 10, 0, 60])),
list(selected_items.T[0]),
)
self.assertListEqual(
list(np.array([0, 0, 10, 40])),
list(selected_items.T[1]),
)
self.assertListEqual(
list(np.array([0, 30, 0, 100])),
list(selected_items.T[2]),
)
def process(self, blob):
self.count += 1
tab = Table({"a": self.count * 10, "b": 1}, h5loc="tab")
tab2 = Table({"a": np.arange(self.count)}, h5loc="tab2")
blob["Tab"] = tab
blob["Tab2"] = tab2
return blob
def process(self, blob):
blob["Tab"] = Table({"a": self.i}, h5loc="/tab")
blob["SplitTab"] = Table(
{"b": self.i}, h5loc="/split_tab", split_h5=True
)
blob["Arr"] = NDArray(np.arange(self.i + 1), h5loc="/arr")
self.i += 1
return blob
def test_merge(self):
tab1 = Table({"a": [1]}, h5loc="/a", h5singleton=True)
tab2 = Table({"a": [2]})
tab3 = Table({"a": [3]})
merged_tab = Table.merge([tab1, tab2, tab3])
assert 3 == len(merged_tab)
self.assertListEqual([1, 2, 3], list(merged_tab.a))
assert "/a" == merged_tab.h5loc
assert merged_tab.h5singleton
def test_flat_raises(self):
with pytest.raises(ValueError):
t = Table([1, 2, 3], dtype=int).dtype
with pytest.raises(ValueError):
t = Table([1, 2, 3], dtype=float).dtype
with pytest.raises(ValueError):
t = Table([1, 2, 3], dtype=None).dtype
with pytest.raises(ValueError):
t = Table([1, 2, 3]).dtype
with pytest.raises(ValueError):
t = Table([1, 2, 3], colnames=["a", "b", "c"]) # noqa
def test_cherenkov_from_Table(self):
arr = cherenkov(Table(self.calib_hits), Table(self.track))
self.assertAlmostEqual(arr["d_photon_closest"][0], 24.049593557846112)
self.assertAlmostEqual(arr["d_photon"][0], 35.80244420413484)
self.assertAlmostEqual(arr["d_track"][0], 45.88106599210481)
self.assertAlmostEqual(arr["t_photon"][0], 70311759.26448613)
self.assertAlmostEqual(arr["cos_photon_PMT"][0], -0.98123942583677)
self.assertAlmostEqual(arr["dir_x_photon"][0], 0.45964884122649263)
self.assertAlmostEqual(arr["dir_y_photon"][0], -0.8001372907490844)
self.assertAlmostEqual(arr["dir_z_photon"][0], -0.3853612055096594)
def test_adding_preserves_metadata(self):
tab1 = Table({"a": [1, 2]},
h5loc="/a",
h5singleton=True,
split_h5=True,
name="FooTable")
tab2 = Table({"a": [3, 4, 5]})
added_tab = tab1 + tab2
assert "/a" == tab1.h5loc
assert added_tab.h5singleton
assert added_tab.split_h5
assert "FooTable" == added_tab.name
def test_from_2d(self):
l2d = [(0, 1), (2, 3), (4, 5), (6, 7), (8, 9)]
names = ["a", "origin", "pmt_id", "time", "group_id"]
dta = inflate_dtype(l2d, names)
with pytest.raises(ValueError):
t = Table(l2d)
with pytest.raises(ValueError):
t = Table(l2d, dtype=None)
with pytest.raises(ValueError):
t = Table(l2d, colnames=names)
with pytest.raises(ValueError):
t = Table(l2d, dtype=dta)
with pytest.raises(ValueError):
t = Table(l2d, dtype=dta, colnames=["a", "b", "c", "d"]) # noqa
def extract_event(self, event_number):
blob = self._current_blob
r = self.event_reader
hits = r.events.snapshot_hits[event_number]
trg_hits = r.events.triggered_hits[event_number]
raw_event_info = r.events.headers[event_number]
trigger_mask = self._get_trigger_mask(hits, trg_hits)
hit_series = Table(
{
"channel_id": hits.channel_id,
"dom_id": hits.dom_id,
"time": hits.time,
"tot": hits.tot,
"triggered": trigger_mask,
},
name="Hits",
h5loc="/hits",
split_h5=True,
)
event_info = Table(
{
"det_id": raw_event_info["detector_id"],
"frame_index": raw_event_info["frame_index"],
"livetime_sec": 0,
"mc_id": 0,
"mc_t": 0,
"n_events_gen": 0,
"n_files_gen": 0,
"overlays": raw_event_info["overlays"],
"trigger_counter": raw_event_info["trigger_counter"],
"trigger_mask": raw_event_info["trigger_mask"],
"utc_nanoseconds":
raw_event_info["UTC_16nanosecondcycles"] * 16.0,
"utc_seconds": raw_event_info["UTC_seconds"],
"weight_w1": np.nan,
"weight_w2": np.nan,
"weight_w3": np.nan,
"run_id": raw_event_info["run"],
},
name="EventInfo",
h5loc="/event_info",
)
self.event_index += 1
blob["EventInfo"] = event_info
blob["Hits"] = hit_series
return blob
def test_reorder_dtypes(self):
dtype = np.dtype([("a", "<i8"), ("c", "<i8"), ("b", "<f8")])
dtype_reordered = np.dtype([("b", "<f8"), ("c", "<i8"), ("a", "<i8")])
tab = Table({"a": 1, "b": 2.5, "c": 3}, dtype=dtype)
assert tab.dtype == dtype
assert 1 == tab.a[0]
assert 2.5 == tab.b[0]
assert 3 == tab.c[0]
tab_reordered = Table(tab, dtype=dtype_reordered)
assert tab_reordered.dtype == dtype_reordered
assert 1 == tab_reordered.a[0]
assert 2.5 == tab_reordered.b[0]
assert 3 == tab_reordered.c[0]
def test_merge_other_different_columns_fills_nan_when_fillna(self):
tab1 = Table({"a": [1.1, 1.2], "b": [10.1, 10.2]})
tab2 = Table({"a": [2.1, 2.2], "c": [100.1, 100.2]})
merged_tab = Table.merge([tab1, tab2], fillna=True)
assert 4 == len(merged_tab)
self.assertListEqual([1.1, 1.2, 2.1, 2.2], list(merged_tab.a))
self.assertListEqual([10.1, 10.2], list(merged_tab.b[:2]))
self.assertListEqual([100.1, 100.2], list(merged_tab.c[2:]))
assert np.isnan(merged_tab.c[0])
assert np.isnan(merged_tab.c[1])
assert np.isnan(merged_tab.b[2])
assert np.isnan(merged_tab.b[3])
def setUp(self):
self.arr_bare = Table({
"a": [1, 2, 3],
"b": [3, 4, 5],
})
self.arr_wpos = Table({
"a": [1, 2, 3],
"b": [3, 4, 5],
"pos_x": [10, 20, 30],
"pos_y": [40, 50, 60],
"pos_z": [70, 80, 90],
"dir_x": [10.0, 20.0, 30.0],
"dir_y": [40.0, 50.0, 60.0],
"dir_z": [70.0, 80.0, 90.0],
})
def process(self, blob):
self.index += 1
mc_hits = Table({"pmt_id": [1, 2, 1], "time": [10.1, 11.2, 12.3]})
hits = Table(
{
"dom_id": [2, 3, 3],
"channel_id": [0, 1, 2],
"time": [10.1, 11.2, 12.3],
"tot": [0, 10, 255],
}
)
blob["Hits"] = hits
blob["McHits"] = mc_hits
return blob
def process_online_reco(self, data, blob):
data_io = BytesIO(data)
preamble = DAQPreamble(file_obj=data_io) # noqa
_data = unpack("<iiiQI", data_io.read(4 + 4 + 4 + 8 + 4))
det_id, run_id, frame_index, trigger_counter, utc_seconds = _data
shower_reco = unpack("9d", data_io.read(9 * 8))
shower_meta = unpack("3i", data_io.read(12))
track_reco = unpack("9d", data_io.read(9 * 8))
track_meta = unpack("3i", data_io.read(12))
print("Shower: x/y/z/dx/dy/dz/E/Q/t (type/status/ndf): ", shower_reco,
shower_meta)
print("Track: x/y/z/dx/dy/dz/E/Q/t (type/status/ndf): ", track_reco,
track_meta)
blob["ReconstructionInfo"] = Table(
{
"det_id": det_id,
"run_id": run_id,
"frame_index": frame_index,
"trigger_counter": trigger_counter,
"utc_seconds": utc_seconds,
},
h5loc="reco",
split_h5=True,
name="Reconstructions",
)
args = track_reco + track_meta
blob["RecoTrack"] = RecoTrack(*args)
args = shower_reco + shower_meta
blob["RecoShower"] = RecoShower(*args)
def test_correct_slewing(self):
hits = Table(
{
"dom_id": [2, 3, 3],
"channel_id": [0, 1, 2],
"time": [10.1, 11.2, 12.3],
"tot": [0, 10, 255],
}
)
tester = self
class HitCalibrator(Module):
def process(self, blob):
self.services["correct_slewing"](hits)
a_hit = hits[0]
tester.assertAlmostEqual(10.1 - slew(a_hit.tot), a_hit.time)
a_hit = hits[1]
tester.assertAlmostEqual(11.2 - slew(a_hit.tot), a_hit.time)
return blob
pipe = Pipeline()
pipe.attach(CalibrationService, filename=data_path("detx/detx_v1.detx"))
pipe.attach(HitCalibrator)
pipe.drain(1)
def test_assert_apply_adds_pmt_id_to_hits(self):
calib = Calibration(filename=data_path("detx/detx_v1.detx"))
hits = Table(
{"dom_id": [2, 3, 3], "channel_id": [0, 1, 2], "time": [10.1, 11.2, 12.3]}
)
chits = calib.apply(hits, correct_slewing=False)
self.assertListEqual([4, 8, 9], list(chits.pmt_id))
def test_apply_to_hits_with_dom_id_and_channel_id_with_wrong_calib_raises(self):
calib = Calibration(filename=data_path("detx/detx_v1.detx"))
hits = Table({"dom_id": [999], "channel_id": [0], "time": [10.1]})
with self.assertRaises(KeyError):
calib.apply(hits, correct_slewing=False)
def test_triggered_keeps_attrs(self):
n = 5
channel_ids = np.arange(n)
dom_ids = np.arange(n)
times = np.arange(n)
tots = np.arange(n)
triggereds = np.array([0, 1, 1, 0, 1])
hits = Table(
{
"channel_id": channel_ids,
"dom_id": dom_ids,
"time": times,
"tot": tots,
"triggered": triggereds,
"group_id": 0, # event_id
},
name="hits",
h5loc="/foo",
split_h5=True,
)
triggered_hits = hits.triggered_rows
assert len(triggered_hits) == 3
assert triggered_hits.split_h5
assert triggered_hits.name == "hits"
assert triggered_hits.h5loc == "/foo"
def test_fromcolumns(self):
n = 5
dlist = [
np.ones(n, dtype=int),
np.zeros(n, dtype=float),
0,
]
dt = np.dtype([("a", float), ("b", float), ("c", float)])
with pytest.raises(ValueError):
tab = Table(dlist, dtype=dt)
tab = Table.from_columns(dlist, dtype=dt)
print(tab.dtype)
print(tab.shape)
print(tab)
assert tab.h5loc == DEFAULT_H5LOC
assert isinstance(tab, Table)
tab = Table.from_columns(dlist, dtype=dt, h5loc="/foo")
print(tab.dtype)
print(tab.shape)
print(tab)
assert tab.h5loc == "/foo"
assert isinstance(tab, Table)
bad_dt = [("a", float), ("b", float), ("c", float), ("d", int)]
with pytest.raises(ValueError):
tab = Table.from_columns(dlist, dtype=bad_dt)
print(tab.dtype)
print(tab.shape)
print(tab)
def process(self, blob):
blob["Arr"] = NDArray(np.arange(self.index + 1), h5loc="/arr")
blob["Tab"] = Table(
{"a": np.arange(self.index + 1), "i": self.index}, h5loc="/tab"
)
self.index += 1
return blob
def test_fromrows(self):
dlist = [
[1, 2, 3],
[4, 5, 6],
]
dt = np.dtype([("a", float), ("b", float), ("c", float)])
with pytest.raises(ValueError):
tab = Table(dlist, dtype=dt)
tab = Table.from_rows(dlist, dtype=dt)
print(tab.dtype)
print(tab.shape)
print(tab)
assert tab.h5loc == DEFAULT_H5LOC
assert isinstance(tab, Table)
tab = Table.from_rows(dlist, dtype=dt, h5loc="/foo")
print(tab.dtype)
print(tab.shape)
print(tab)
assert tab.h5loc == "/foo"
assert isinstance(tab, Table)
bad_dt = [("a", float), ("b", float), ("c", float), ("d", int)]
with pytest.raises(ValueError):
tab = Table.from_rows(dlist, dtype=bad_dt)
print(tab.dtype)
print(tab.shape)
print(tab)
def test_append_columns(self):
tab = Table(self.arr)
print(tab)
with pytest.raises(ValueError):
tab = tab.append_columns("new", [1, 2, 3, 4])
tab = tab.append_columns("new", [1, 2, 3])
print(tab)
assert tab.new[0] == 1
assert tab.new[-1] == 3
tab = tab.append_columns("bar", 0)
print(tab)
assert tab.bar[0] == 0
assert tab.bar[-1] == 0
tab = tab.append_columns("lala", [1])
print(tab)
assert tab.lala[0] == 1
assert tab.lala[-1] == 1
with pytest.raises(ValueError):
tab = tab.append_columns(["m", "n"], [1, 2])
with pytest.raises(ValueError):
tab = tab.append_columns(["m", "n"], [[1], [2]])
tab = tab.append_columns(["m", "n"], [[1, 1, 2], [2, 4, 5]])
print(tab)
assert tab.m[0] == 1
assert tab.m[-1] == 2
assert tab.n[0] == 2
assert tab.n[-1] == 5