def __init__(self, path, max_events=None):
"""
Reads simtelarray files utilising the SimTelEventSource from ctapipe
Parameters
----------
path : str
Path to the simtel file
max_events : int
Maximum number of events to read from the file
"""
super().__init__(path, max_events)
try:
from ctapipe.io import SimTelEventSource, EventSeeker
except ModuleNotFoundError:
msg = "Cannot find ctapipe installation"
raise ModuleNotFoundError(msg)
try:
from target_calib import CameraConfiguration
except ModuleNotFoundError:
msg = ("Cannot find TARGET libraries, please follow installation "
"instructions from https://forge.in2p3.fr/projects/gct/"
"wiki/Installing_CHEC_Software")
raise ModuleNotFoundError(msg)
self.path = path
reader = SimTelEventSource(input_url=path,
max_events=max_events,
back_seekable=True)
self.seeker = EventSeeker(reader)
first_event = self.seeker[0]
tels = list(first_event.r0.tels_with_data)
self.tel = tels[0]
shape = first_event.r0.tel[self.tel].waveform.shape
_, self.n_pixels, self.n_samples = shape
self.n_modules = self.n_pixels // 64
self.index = 0
n_modules = 32
camera_version = "1.1.0"
self._camera_config = CameraConfiguration(camera_version)
tc_mapping = self._camera_config.GetMapping(n_modules == 1)
self.mapping = get_clp_mapping_from_tc_mapping(tc_mapping)
pix_x = first_event.inst.subarray.tel[tels[0]].camera.pix_x.value
pix_y = first_event.inst.subarray.tel[tels[0]].camera.pix_y.value
self.mapping['xpix'] = pix_x
self.mapping['ypix'] = pix_y
self.reference_pulse_path = self._camera_config.GetReferencePulsePath()
self.camera_version = self._camera_config.GetVersion()
self.gps_time = None
self.mc_true = None
self.mc = None
self.pointing = None
self.mcheader = None
def plot_tm():
"""
Make a camera image plot for values that are per superpixel
"""
from target_calib import CameraConfiguration
c = CameraConfiguration("1.1.0")
m = c.GetMapping()
df = get_tm_mapping(get_clp_mapping_from_tc_mapping(m))
camera = CameraImage.from_mapping(df)
image = np.zeros(m.GetNModules())
image[::2] = 1
camera.image = image
plt.show()
def from_tc_mapping(cls, tc_mapping, **kwargs):
"""
Generate the class using the TargetCalib Mapping Class
Parameters
----------
tc_mapping : `target_calib.Mapping`
kwargs
Arguments passed to `CHECLabPy.plottong.setup.Plotter`
Returns
-------
`CameraImage`
"""
mapping = get_clp_mapping_from_tc_mapping(tc_mapping)
return cls.from_mapping(mapping, **kwargs)
def __init__(self,
ip,
port,
sampleinterval=0.1,
timewindow=10.0,
size=(600, 350)):
# PyQtGraph stuff
self._interval = int(sampleinterval * 1000)
self.app = QtGui.QApplication([])
self.win = pg.GraphicsWindow()
self.win.setWindowTitle("Triggerpattern viewer")
# data
self.data = defaultdict(
list) # {'hv_current':list(),'hv_voltage':list()}
self.time = list()
self.plts = {}
self.curve_names = list()
self.plot_time_window = 10
# self.win.addItem(pg.TextItem(text='HEJHEJHEJ', color=(200, 200, 200), html=None, anchor=(0, 0), border=None, fill=None, angle=0, rotateAxis=None),row=1 )
self.lastframeindicator = pg.LabelItem(text="")
self.lastframe = datetime.now()
self.win.addItem(self.lastframeindicator)
self.gotdataindicator = pg.LabelItem(
text='<font color="{}">{}</font>'.format("green", "connected"))
self.win.addItem(self.gotdataindicator)
self.win.nextRow()
self.countersindicator = pg.LabelItem(text="")
self.win.addItem(self.countersindicator)
self.win.nextRow()
self.last_uc_ev = 0
self.missed_counter = 0
self.readout_counter = 0
self._add_plot(
"Trigger rate",
("Rate Hz", ""),
("Time", "min"),
["Nominal triggers"], # , "Busy triggers"],
)
# self._add_plot(
# "Slow signal amplitude",
# ("Amplitude", "mV"),
# ("Time", "min"),
# ["total amplitude", "max amplitude X 10"],
# )
self.win.nextRow()
self.img = pg.ImageItem(levels=(0, 400))
self.p = self.win.addPlot()
self.p.addItem(self.img)
self.c = CameraConfiguration("1.1.0")
self.m = self.c.GetMapping()
from CHECLabPy.plotting.camera import CameraImage, CameraImageImshow
from CHECLabPy.utils.mapping import (
get_clp_mapping_from_tc_mapping,
get_superpixel_mapping,
get_tm_mapping,
)
self.sp_mapping = get_superpixel_mapping(
get_clp_mapping_from_tc_mapping(self.m))
self.xpix = np.array(self.sp_mapping.xpix)
self.ypix = np.array(self.sp_mapping.ypix)
self.xmap = np.array(self.sp_mapping.row)
self.ymap = np.array(self.sp_mapping.col)
print(self.xmap.shape, self.ymap.shape)
self.map = np.zeros((24, 24), dtype=np.uint64)
for i in range(512):
self.map[self.xmap[i], self.ymap[i]] = i
self.map = self.map.flatten()
from ssdaq.data._dataimpl.trigger_format import (
get_SP2bptrigg_mapping,
get_bptrigg2SP_mapping,
)
self.bptmap = get_bptrigg2SP_mapping()
self.timer = QtCore.QTimer()
self.timer.timeout.connect(self.updateplots)
self.timer.start(self._interval)
self.tr_listener = subscribers.BasicTriggerSubscriber(port=port, ip=ip)
self.tr_listener.start()
def main():
description = ('Reduce a *_r1.tio file into a *_dl1.hdf5 file containing '
'various parameters extracted from the waveforms')
parser = argparse.ArgumentParser(description=description,
formatter_class=Formatter)
parser.add_argument('-f',
'--files',
dest='input_paths',
nargs='+',
help='path to the TIO r1 run files')
parser.add_argument('-m',
'--monitor',
dest='monitor',
action='store',
help='path to the monitor file (OPTIONAL)')
parser.add_argument('-o',
'--output',
dest='output_path',
action='store',
help='path to store the output HDF5 dl1 file '
'(OPTIONAL, will be automatically set if '
'not specified)')
parser.add_argument('-n',
'--maxevents',
dest='max_events',
action='store',
help='Number of events to process',
type=int)
parser.add_argument('-r',
'--reducer',
dest='reducer',
action='store',
default='AverageWF',
choices=WaveformReducerFactory.subclass_names,
help='WaveformReducer to use')
parser.add_argument('-c',
'--config',
dest='configuration',
help="""Configuration to pass to the waveform reducer
(Usage: '{"window_shift":6, "window_size":6}') """)
parser.add_argument('-p',
'--plot',
dest='plot',
action='store_true',
help="Plot stages for waveform reducers")
args = parser.parse_args()
if args.configuration:
config = json.loads(args.configuration)
config_string = args.configuration
else:
config = {}
config_string = ""
input_paths = args.input_paths
n_files = len(input_paths)
for i_path, input_path in enumerate(input_paths):
print("PROGRESS: Reducing file {}/{}".format(i_path + 1, n_files))
kwargs = dict(input_url=input_path, max_events=args.max_events)
reader = HESSIOEventSource(**kwargs)
seeker = EventSeeker(reader)
n_events = len(seeker)
first_event = seeker[0]
tels = list(first_event.r0.tels_with_data)
_, n_pixels, n_samples = first_event.r0.tel[tels[0]].waveform.shape
n_modules = 32
n_cells = 1
pixel_array = np.arange(n_pixels)
camera_version = "1.1.0"
camera_config = CameraConfiguration(camera_version)
tc_mapping = camera_config.GetMapping(n_modules == 1)
mapping = get_clp_mapping_from_tc_mapping(tc_mapping)
if 'reference_pulse_path' not in config:
reference_pulse_path = camera_config.GetReferencePulsePath()
config['reference_pulse_path'] = reference_pulse_path
kwargs = dict(n_pixels=n_pixels,
n_samples=n_samples,
plot=args.plot,
mapping=mapping,
**config)
reducer = WaveformReducerFactory.produce(args.reducer, **kwargs)
baseline_subtractor = BaselineSubtractor(seeker)
input_path = reader.input_url
output_path = args.output_path
if not output_path:
output_path = input_path.replace(".simtel.gz", "_dl1.h5")
output_path = output_path.replace("run", "Run")
r1 = HESSIOR1Calibrator()
with DL1Writer(output_path, n_events * n_pixels,
args.monitor) as writer:
t_cpu = 0
start_time = 0
desc = "Processing events"
for event in tqdm(seeker, total=n_events, desc=desc):
iev = event.count
r1.calibrate(event)
waveforms = event.r1.tel[tels[0]].waveform[0]
mc_true = event.mc.tel[tels[0]].photo_electron_image
t_cpu = pd.to_datetime(event.trig.gps_time.value, unit='s')
if not start_time:
start_time = t_cpu
waveforms_bs = baseline_subtractor.subtract(waveforms)
bs = baseline_subtractor.baseline
params = reducer.process(waveforms_bs)
df_ev = pd.DataFrame(
dict(iev=iev,
pixel=pixel_array,
first_cell_id=0,
t_cpu=t_cpu,
t_tack=0,
baseline_subtracted=bs,
**params,
mc_true=mc_true))
writer.append_event(df_ev)
sn_dict = {}
for tm in range(n_modules):
sn_dict['TM{:02d}_SN'.format(tm)] = "NaN"
metadata = dict(source="CHECLabPy",
date_generated=pd.datetime.now(),
input_path=input_path,
n_events=n_events,
n_modules=n_modules,
n_pixels=n_pixels,
n_samples=n_samples,
n_cells=n_cells,
start_time=start_time,
end_time=t_cpu,
camera_version=camera_version,
reducer=reducer.__class__.__name__,
configuration=config_string,
**sn_dict)
writer.add_metadata(**metadata)
writer.add_mapping(mapping)
def mapping(self):
return get_clp_mapping_from_tc_mapping(self.tc_mapping)
def __init__(self, path, max_events=None):
"""
Reads simtelarray files utilising the SimTelEventSource from ctapipe
Parameters
----------
path : str
Path to the simtel file
max_events : int
Maximum number of events to read from the file
"""
super().__init__(path, max_events)
try:
from ctapipe.io import SimTelEventSource, EventSeeker
from ctapipe.coordinates import EngineeringCameraFrame
except ModuleNotFoundError:
msg = "Cannot find ctapipe installation"
raise ModuleNotFoundError(msg)
try:
from target_calib import CameraConfiguration
except ModuleNotFoundError:
msg = ("Cannot find TARGET libraries, please follow installation "
"instructions from https://forge.in2p3.fr/projects/gct/"
"wiki/Installing_CHEC_Software")
raise ModuleNotFoundError(msg)
self.path = path
reader = SimTelEventSource(input_url=path,
max_events=max_events,
back_seekable=True)
self.seeker = EventSeeker(reader)
first_event = self.seeker[0]
tels = list(first_event.r0.tels_with_data)
self.tel = tels[0]
shape = first_event.r0.tel[self.tel].waveform.shape
_, self.n_pixels, self.n_samples = shape
self.n_modules = self.n_pixels // 64
n_modules = 32
camera_version = "1.1.0"
self._camera_config = CameraConfiguration(camera_version)
tc_mapping = self._camera_config.GetMapping(n_modules == 1)
self.mapping = get_clp_mapping_from_tc_mapping(tc_mapping)
n_rows = self.mapping.metadata['n_rows']
n_columns = self.mapping.metadata['n_columns']
camera_geom = first_event.inst.subarray.tel[tels[0]].camera
engineering_frame = EngineeringCameraFrame(n_mirrors=2)
engineering_geom = camera_geom.transform_to(engineering_frame)
pix_x = engineering_geom.pix_x.value
pix_y = engineering_geom.pix_y.value
row, col = get_row_column(pix_x, pix_y)
camera_2d = np.zeros((n_rows, n_columns), dtype=np.int)
camera_2d[row, col] = np.arange(self.n_pixels, dtype=np.int)
self.pixel_order = camera_2d[self.mapping['row'], self.mapping['col']]
self.reference_pulse_path = self._camera_config.GetReferencePulsePath()
self.camera_version = self._camera_config.GetVersion()
self._iev = None
self._t_cpu = None
self.mc = None
self.pointing = None
self.mcheader = None
def trigger_pattern_diagnostics(self, files, save, entry=False):
run_name = files.run
if files["trigfile"] is not None:
filename = files["trigfile"][0]
else:
return None
reader = DataReader(filename)
dbentry = DBEntry(run_name, self.short_name)
print(reader)
triggs = reader[:]
mean_rate = len(triggs) / (triggs[-1].TACK - triggs[0].TACK) * 1e9
print("Mean trigger rate {}{}Hz".format(*get_si_prefix(mean_rate)))
print("Total number of trigger patterns", len(triggs))
tacks = []
nSP = []
ntriggSP = []
heatmap = []
trigg_phase = []
from ssdaq.data._dataimpl.trigger_format import get_bptrigg2SP_mapping
m1 = get_bptrigg2SP_mapping()
last_uc_ev = 0
missed_counter = []
for t in triggs:
if last_uc_ev != 0 and last_uc_ev + 1 != t.uc_ev:
missed_counter.append(t.uc_ev - last_uc_ev)
else:
missed_counter.append(0)
last_uc_ev = t.uc_ev
trigg = t.trigg
tacks.append(t.TACK)
nSP.append(np.sum(t.trigg_union))
ntriggSP.append(np.sum(trigg))
trigg_phase.append(t.trigg_phase)
# if np.sum(t.trigg_union) < 200:
heatmap.append(trigg[m1])
heatmap = np.array(heatmap)
t0 = tacks[0]
run_length = tacks[-1] - tacks[0]
ntriggSP = np.array(ntriggSP)
nSP = np.array(nSP)
tacks = np.array(tacks, dtype=np.uint64)
timeedges = np.linspace(0, run_length, int(run_length / 1e9) + 1)
nSPedges = np.logspace(0, 2.6, 100)
tack_dt = np.diff(tacks)
triggdt = dashi.histogram.hist1d(
np.logspace(np.log10(np.min(tack_dt)), np.log10(np.max(tack_dt)),
100))
triggdt.fill(tack_dt)
triggTotSP = dashi.histogram.hist1d(nSPedges)
triggTotSP.fill(np.array(nSP))
triggSP = dashi.histogram.hist1d(nSPedges)
triggSP.fill(np.array(ntriggSP))
triggRate = dashi.histogram.hist1d(timeedges)
triggRate.fill(tacks - t0)
triggRate.binedges[:] /= 1e9
missedPackets = dashi.histogram.hist1d(timeedges)
missedPackets.fill(tacks - t0, np.array(missed_counter))
missedPackets.binedges[:] /= 1e9
t_phases = np.log2(np.array(trigg_phase))
triggerphase = dashi.histogram.hist1d(
np.linspace(
np.min(t_phases),
np.max(t_phases) + 1,
np.max(t_phases) - np.min(t_phases) + 2,
) - 0.5)
triggerphase.fill(t_phases)
fig = plt.figure(figsize=(11, 7))
fig.suptitle("{} mean rate {}{}Hz".format(run_name,
*get_si_prefix(mean_rate)))
ax1 = fig.add_subplot(2, 2, 1)
triggdt.line()
plt.xscale("log")
plt.yscale("log")
plt.xlabel("TACK dt (ns)")
plt.ylabel("number of triggers")
ax2 = fig.add_subplot(2, 2, 2)
triggTotSP.line(label="Total number of SPs (union)")
triggSP.line(label="Triggered SPs")
plt.xscale("log")
plt.yscale("log")
plt.xlabel("number of SPs")
plt.ylabel("number of triggers")
plt.legend(fontsize=6)
ax3 = fig.add_subplot(2, 2, 3)
color = "tab:red"
triggRate.line(color=color)
plt.xlabel("Time since run start (s)")
plt.ylabel("Instantious trigger rate (Hz)", color=color)
ax3twin = ax3.twinx(
) # instantiate a second axes that shares the same x-axis
color = "tab:blue"
ax3twin.set_ylabel(
"number of missed packets",
color=color) # we already handled the x-label with ax1
missedPackets.line(color=color)
# ax3twin.plot(, data2, color=color)
ax3twin.tick_params(axis="y", labelcolor=color)
ax3 = fig.add_subplot(2, 2, 4)
plt.errorbar(
triggerphase.bincenters,
triggerphase.bincontent,
yerr=triggerphase.binerror,
xerr=triggerphase.binwidths / 2,
elinewidth=1,
linestyle="",
fmt="o",
)
plt.xlabel("Trigger phase")
plt.ylabel("Number of triggers")
fig.tight_layout(rect=[0, 0.03, 1, 0.95])
c = CameraConfiguration("1.1.0")
m = c.GetMapping()
p_image = ImagePlotter(get_clp_mapping_from_tc_mapping(m))
p_image.set_image(
"{} Trigger heat map No Flasher correction".format(
run_name, *get_si_prefix(mean_rate)),
np.sum(heatmap, axis=0) / len(triggs),
)
p_image2 = ImagePlotter(get_clp_mapping_from_tc_mapping(m))
p_image2.set_image(
"{} Trigger heat map with Flasher correction".format(
run_name, *get_si_prefix(mean_rate)),
np.sum(heatmap[nSP < 200], axis=0) / len(triggs),
)
p_image3 = ImagePlotter(get_clp_mapping_from_tc_mapping(m))
p_image3.set_image(
"{} Triggered SPs".format(run_name, *get_si_prefix(mean_rate)),
np.sum(heatmap, axis=0) > 0,
)
if save:
fig.savefig(f"{run_name}_trigpat_diag.png")
print(f"Figure saved to: {run_name}_trigpat_diag.png")
p_image.save(f"{run_name}_trig_heatmap_noflashcorr.png")
p_image2.save(f"{run_name}_trig_heatmap_withflashcorr.png")
p_image3.save(f"{run_name}_trig_sps.png")
elif entry:
start_time = reader.timestamp
date = start_time.date()
dt = start_time - datetime.datetime(date.year, date.month,
date.day)
if dt.seconds < 3600 * 6:
date = date.replace(day=date.day - 1)
dbentry.add_modstats({
"obsdate":
date,
"run_start":
start_time.time(),
"run_start_timestamp":
reader.timestamp,
"ntriggs":
reader.n_entries + int(np.sum(missedPackets.bincontent)),
"run_length":
datetime.timedelta(seconds=run_length * 1e-9),
"rate":
mean_rate,
})
dbentry.add_stats({
"number of triggers":
make_field(
"number of triggers",
int(np.sum(missedPackets.bincontent)) + reader.n_entries,
),
"mean rate":
SVal(mean_rate, "Hz"),
"lost trigger packets":
int(np.sum(missedPackets.bincontent)),
"max rate":
SVal(np.max(triggRate.bincontent), "Hz"),
"min rate":
SVal(np.min(triggRate.bincontent), "Hz"),
"number of triggering SPs":
int(np.sum(p_image3.ci_trigger.image)),
"number of nontriggering SPs":
512 - int(np.sum(p_image3.ci_trigger.image)),
"fraction of triggering SPs":
"{0:.2f}%".format(
np.sum(p_image3.ci_trigger.image) / 512.0 * 100),
})
dbentry.add_fig({
"trigpat_diag":
savefig_to_buffer(fig),
"trig_heatmap_noflashcorr":
savefig_to_buffer(p_image.fig),
"trig_heatmap_withflashcorr":
savefig_to_buffer(p_image2.fig),
"trig_sps":
savefig_to_buffer(p_image3.fig),
})
dbentry.set_title("Trigger patterns")
plt.close('all')
return dbentry
