class ADC2PEPlots(Tool):
name = "ADC2PEPlots"
description = "Create plots related to adc2pe"
aliases = Dict(dict(max_events='TargetioFileReader.max_events'))
classes = List([
TargetioFileReader,
TargetioR1Calibrator,
])
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.reader = None
self.reader_pe = None
self.dl0 = None
self.dl1 = None
self.fitter = None
self.fitter_pe = None
self.dead = None
self.n_pixels = None
self.n_samples = None
self.spe_path = "/Volumes/gct-jason/data/170314/spe/Run00073_r0/extract_spe/spe.npy"
self.gm_path = "/Volumes/gct-jason/data/170310/hv/gain_matching_coeff.npz"
self.p_pixelspe = None
self.p_tmspe = None
self.p_tmspe_pe = None
self.p_adc2pe = None
self.p_adc2pe_1100tm = None
self.p_adc2pe_1100tm_stats = None
self.p_eped = None
self.p_eped_sigma = None
self.p_spe = None
self.p_spe_sigma = None
self.p_lambda = None
self.p_enf = None
def setup(self):
self.log_format = "%(levelname)s: %(message)s [%(name)s.%(funcName)s]"
kwargs = dict(config=self.config, tool=self)
filepath = '/Volumes/gct-jason/data/170314/spe/Run00073_r1_adc.tio'
self.reader = TargetioFileReader(input_path=filepath, **kwargs)
filepath = '/Volumes/gct-jason/data/170314/spe/Run00073_r1.tio'
self.reader_pe = TargetioFileReader(input_path=filepath, **kwargs)
cleaner = CHECMWaveformCleanerAverage(**kwargs)
extractor = AverageWfPeakIntegrator(**kwargs)
self.dl0 = CameraDL0Reducer(**kwargs)
self.dl1 = CameraDL1Calibrator(extractor=extractor,
cleaner=cleaner,
**kwargs)
self.fitter = CHECMSPEFitter(**kwargs)
self.fitter.range = [-30, 160]
self.fitter_pe = CHECMSPEFitter(**kwargs)
self.fitter_pe.range = [-1, 6]
self.fitter_pe.initial = dict(norm=None,
eped=0,
eped_sigma=0.2,
spe=1,
spe_sigma=0.5,
lambda_=0.2)
self.dead = Dead()
script = "checm_paper_adc2pe"
self.p_pixelspe = PixelSPEFitPlotter(**kwargs,
script=script,
figure_name="spe_fit_pixel1559")
self.p_tmspe = TMSPEFitPlotter(**kwargs,
script=script,
figure_name="spe_fit_tm24")
self.p_tmspe_pe = TMSPEFitPlotter(**kwargs,
script=script,
figure_name="spe_fit_tm24_pe")
self.p_adc2pe = ADC2PEPlotter(**kwargs,
script=script,
figure_name="adc2pe",
shape='square')
self.p_adc2pe_1100tm = ADC2PE1100VTMPlotter(
**kwargs,
script=script,
figure_name="adc2pe_1100V_tms",
shape='wide')
self.p_adc2pe_1100tm_stats = ADC2PE1100VTMStatsPlotter(
**kwargs,
script=script,
figure_name="adc2pe_1100V_tms_stats",
shape='wide')
self.p_eped = Hist(**kwargs,
script=script,
figure_name="f_eped",
shape='square')
self.p_eped_sigma = Hist(**kwargs,
script=script,
figure_name="f_eped_sigma",
shape='square')
self.p_spe = Hist(**kwargs,
script=script,
figure_name="f_spe",
shape='square')
self.p_spe_sigma = Hist(**kwargs,
script=script,
figure_name="f_spe_sigma",
shape='square')
self.p_lambda = Hist(**kwargs,
script=script,
figure_name="f_lambda",
shape='square')
self.p_enf = Hist(**kwargs,
script=script,
figure_name="f_enf",
shape='square')
def start(self):
n_events = self.reader.num_events
first_event = self.reader.get_event(0)
telid = list(first_event.r0.tels_with_data)[0]
n_pixels, n_samples = first_event.r1.tel[telid].pe_samples[0].shape
### SPE values from fit _______________________________________________
# Prepare storage array
dl1 = np.zeros((n_events, n_pixels))
dl1_pe = np.zeros((n_events, n_pixels))
hist_pix1559 = None
edges_pix1559 = None
between_pix1559 = None
fit_pix1559 = None
fitx_pix1559 = None
hist_tm24 = np.zeros((64, self.fitter.nbins))
edges_tm24 = np.zeros((64, self.fitter.nbins + 1))
between_tm24 = np.zeros((64, self.fitter.nbins))
hist_tm24_pe = np.zeros((64, self.fitter.nbins))
edges_tm24_pe = np.zeros((64, self.fitter.nbins + 1))
between_tm24_pe = np.zeros((64, self.fitter.nbins))
f_eped = np.zeros(n_pixels)
f_eped_sigma = np.zeros(n_pixels)
f_spe = np.zeros(n_pixels)
f_spe_sigma = np.zeros(n_pixels)
f_lambda = np.zeros(n_pixels)
source = self.reader.read()
desc = "Looping through file"
for event in tqdm(source, total=n_events, desc=desc):
index = event.count
self.dl0.reduce(event)
self.dl1.calibrate(event)
dl1[index] = event.dl1.tel[telid].image
source = self.reader_pe.read()
desc = "Looping through file (pe)"
for event in tqdm(source, total=n_events, desc=desc):
index = event.count
self.dl0.reduce(event)
self.dl1.calibrate(event)
dl1_pe[index] = event.dl1.tel[telid].image
desc = "Fitting pixels"
for pix in trange(n_pixels, desc=desc):
tm = pix // 64
tmpix = pix % 64
if not self.fitter.apply(dl1[:, pix]):
self.log.warning("Pixel {} couldn't be fit".format(pix))
continue
if pix == 1559:
hist_pix1559 = self.fitter.hist
edges_pix1559 = self.fitter.edges
between_pix1559 = self.fitter.between
fit_pix1559 = self.fitter.fit
fitx_pix1559 = self.fitter.fit_x
if tm == 24:
hist_tm24[tmpix] = self.fitter.hist
edges_tm24[tmpix] = self.fitter.edges
between_tm24[tmpix] = self.fitter.between
f_eped[pix] = self.fitter.coeff['eped']
f_eped_sigma[pix] = self.fitter.coeff['eped_sigma']
f_spe[pix] = self.fitter.coeff['spe']
f_spe_sigma[pix] = self.fitter.coeff['spe_sigma']
f_lambda[pix] = self.fitter.coeff['lambda_']
f_eped = checm_dac_to_volts(f_eped)
f_eped_sigma = checm_dac_to_volts(f_eped_sigma)
f_spe = checm_dac_to_volts(f_spe)
f_spe_sigma = checm_dac_to_volts(f_spe_sigma)
f_eped = self.dead.mask1d(f_eped).compressed()
f_eped_sigma = self.dead.mask1d(f_eped_sigma).compressed()
f_spe = self.dead.mask1d(f_spe).compressed()
f_spe_sigma = self.dead.mask1d(f_spe_sigma).compressed()
f_lambda = self.dead.mask1d(f_lambda).compressed()
edges_pix1559 = checm_dac_to_volts(edges_pix1559)
between_pix1559 = checm_dac_to_volts(between_pix1559)
fitx_pix1559 = checm_dac_to_volts(fitx_pix1559)
edges_tm24 = checm_dac_to_volts(edges_tm24)
between_tm24 = checm_dac_to_volts(edges_tm24)
desc = "Fitting pixels (pe)"
for pix in trange(n_pixels, desc=desc):
tm = pix // 64
tmpix = pix % 64
if tm != 24:
continue
if not self.fitter_pe.apply(dl1_pe[:, pix]):
self.log.warning("Pixel {} couldn't be fit".format(pix))
continue
hist_tm24_pe[tmpix] = self.fitter_pe.hist
edges_tm24_pe[tmpix] = self.fitter_pe.edges
between_tm24_pe[tmpix] = self.fitter_pe.between
### SPE values for each hv setting ____________________________________
kwargs = dict(config=self.config,
tool=self,
spe_path=self.spe_path,
gain_matching_path=self.gm_path)
a2p = TargetioADC2PECalibrator(**kwargs)
hv_dict = dict()
hv_dict['800'] = [800] * 2048
hv_dict['900'] = [900] * 2048
hv_dict['1000'] = [1000] * 2048
hv_dict['1100'] = [1100] * 2048
hv_dict['800gm'] = [a2p.gm800[i // 64] for i in range(2048)]
hv_dict['900gm'] = [a2p.gm900[i // 64] for i in range(2048)]
hv_dict['1000gm'] = [a2p.gm1000[i // 64] for i in range(2048)]
df_list = []
for key, l in hv_dict.items():
hv = int(key.replace("gm", ""))
gm = 'gm' in key
gm_t = 'Gain-matched' if 'gm' in key else 'Non-gain-matched'
for pix in range(n_pixels):
if pix in self.dead.dead_pixels:
continue
adc2pe = a2p.get_adc2pe_at_hv(l[pix], pix)
df_list.append(
dict(key=key,
hv=hv,
gm=gm,
gm_t=gm_t,
pixel=pix,
tm=pix // 64,
spe=1 / adc2pe))
df = pd.DataFrame(df_list)
df = df.sort_values(by='gm', ascending=True)
df = df.assign(spe_mv=checm_dac_to_volts(df['spe']))
# Create figures
self.p_pixelspe.create(hist_pix1559, edges_pix1559, between_pix1559,
fit_pix1559, fitx_pix1559)
self.p_tmspe.create(hist_tm24, edges_tm24, between_tm24, "V ns")
self.p_tmspe_pe.create(hist_tm24_pe, edges_tm24_pe, between_tm24_pe,
"p.e.", 1)
self.p_adc2pe.create(df)
self.p_adc2pe_1100tm.create(df)
self.p_adc2pe_1100tm_stats.create(df)
self.p_eped.create(f_eped, "Pedestal (V ns)")
self.p_eped_sigma.create(f_eped_sigma, "Pedestal Sigma (V ns)")
self.p_spe.create(f_spe, "SPE (V ns)")
self.p_spe_sigma.create(f_spe_sigma, "SPE Sigma (V ns)")
self.p_lambda.create(f_lambda, "Illumination (Photoelectrons)")
enf = np.sqrt(f_spe_sigma**2 - f_eped_sigma**2) / f_spe
self.p_enf.create(enf, "Relative SPE Width")
def finish(self):
# Save figures
self.p_pixelspe.save()
self.p_tmspe.save()
self.p_tmspe_pe.save()
self.p_adc2pe.save()
self.p_adc2pe_1100tm.save()
self.p_adc2pe_1100tm_stats.save()
self.p_eped.save()
self.p_eped_sigma.save()
self.p_spe.save()
self.p_spe_sigma.save()
self.p_lambda.save()
self.p_enf.save()
class FWInvestigator(Tool):
name = "FWInvestigator"
description = "Investigate the FW"
aliases = Dict(dict())
classes = List([])
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.reader = None
self.dl0 = None
self.dl1 = None
self.fitter = None
self.dead = None
self.fw_calibrator = None
directory = join(realpath(dirname(__file__)), "../targetpipe/io")
self.fw_txt_path = join(directory, "FW.txt")
self.fw_storage_path = join(directory, "FW.h5")
self.fw_storage_path_spe = join(directory, "FW_spe_LS62.h5")
self.spe_fw = 1210
self.p_attenuation = None
self.p_pe = None
def setup(self):
self.log_format = "%(levelname)s: %(message)s [%(name)s.%(funcName)s]"
kwargs = dict(config=self.config, tool=self)
filepath = '/Volumes/gct-jason/data/170314/spe/Run00073_r1_adc.tio'
self.reader = TargetioFileReader(input_path=filepath, **kwargs)
cleaner = CHECMWaveformCleanerAverage(**kwargs)
extractor = AverageWfPeakIntegrator(**kwargs)
self.dl0 = CameraDL0Reducer(**kwargs)
self.dl1 = CameraDL1Calibrator(extractor=extractor,
cleaner=cleaner,
**kwargs)
self.fitter = CHECMSPEFitter(**kwargs)
self.fitter.range = [-30, 160]
self.dead = Dead()
self.fw_calibrator = FWCalibrator(**kwargs)
script = "filter_wheel"
self.p_attenuation = Scatter(**kwargs, script=script, figure_name="attenuation")
self.p_pe = Scatter(**kwargs, script=script, figure_name="pe")
# self.p_tmspe = TMSPEFitPlotter(**kwargs, script=script, figure_name="spe_fit_tm24")
# self.p_tmspe_pe = TMSPEFitPlotter(**kwargs, script=script, figure_name="spe_fit_tm24_pe")
# self.p_adc2pe = ADC2PEPlotter(**kwargs, script=script, figure_name="adc2pe", shape='wide')
# self.p_adc2pe_1100tm = ADC2PE1100VTMPlotter(**kwargs, script=script, figure_name="adc2pe_1100V_tms", shape='wide')
# self.p_adc2pe_1100tm_stats = ADC2PE1100VTMStatsPlotter(**kwargs, script=script, figure_name="adc2pe_1100V_tms_stats", shape='wide')
def start(self):
n_events = self.reader.num_events
first_event = self.reader.get_event(0)
telid = list(first_event.r0.tels_with_data)[0]
n_pixels, n_samples = first_event.r1.tel[telid].pe_samples[0].shape
dl1 = np.zeros((n_events, n_pixels))
lambda_ = np.zeros(n_pixels)
source = self.reader.read()
desc = "Looping through file"
for event in tqdm(source, total=n_events, desc=desc):
index = event.count
self.dl0.reduce(event)
self.dl1.calibrate(event)
dl1[index] = event.dl1.tel[telid].image
desc = "Fitting pixels"
for pix in trange(n_pixels, desc=desc):
if not self.fitter.apply(dl1[:, pix]):
self.log.warning("Pixel {} couldn't be fit".format(pix))
continue
lambda_[pix] = self.fitter.coeff['lambda_']
lambda_ = self.dead.mask1d(lambda_)
avg_lamda = np.mean(lambda_)
columns = ['position', 'attenuation_mean', 'attenuation_rms']
df = pd.read_table(self.fw_txt_path, sep=' ', names=columns, usecols=[0, 1, 2], skiprows=1)
df = df.groupby('position').apply(np.mean)
self.fw_calibrator.df = df
self.fw_calibrator.save(self.fw_storage_path)
self.fw_calibrator.set_calibration(self.spe_fw, avg_lamda)
df = self.fw_calibrator.df
x = df['position']
y = df['attenuation_mean']
y_err = df['attenuation_rms']
self.p_attenuation.create(x, y, y_err, '', "Postion", "Attenuation", "Filter Wheel Attenuation")
x = df['position']
y = df['pe']
y_err = df['pe_err']
self.p_pe.create(x, y, y_err, '', "Postion", "Illumination (p.e.)", "Filter Wheel Calibrated")
self.p_pe.ax.set_yscale('log')
self.p_pe.ax.get_yaxis().set_major_formatter(FuncFormatter(lambda y, _: '{:g}'.format(y)))
def finish(self):
# Save figures
self.p_attenuation.save()
self.p_pe.save()
self.fw_calibrator.save(self.fw_storage_path_spe)
class ADC2PEvsHVPlotter(Tool):
name = "ADC2PEvsHVPlotter"
description = "For a given hv values, get the conversion from adc " \
"to pe for each pixel."
output_dir = Unicode('', help='Directory to store output').tag(config=True)
aliases = Dict(
dict(spe='TargetioADC2PECalibrator.spe_path',
gm='TargetioADC2PECalibrator.gain_matching_path',
o='ADC2PEvsHVPlotter.output_dir'))
classes = List([
TargetioADC2PECalibrator,
])
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.a2p = None
self.dead = None
sns.set_style("whitegrid")
self.cfmaker = None
def setup(self):
self.log_format = "%(levelname)s: %(message)s [%(name)s.%(funcName)s]"
kwargs = dict(config=self.config, tool=self)
self.a2p = TargetioADC2PECalibrator(**kwargs)
self.dead = Dead()
self.cfmaker = CfMaker(32)
# Save figures
self.output_dir = join(self.output_dir, "plot_adc2pe_vs_hv")
if not exists(self.output_dir):
self.log.info("Creating directory: {}".format(self.output_dir))
makedirs(self.output_dir)
def start(self):
hv_dict = dict()
hv_dict['800'] = [800] * 2048
hv_dict['900'] = [900] * 2048
hv_dict['1000'] = [1000] * 2048
hv_dict['1100'] = [1100] * 2048
hv_dict['800gm'] = [self.a2p.gm800[i // 64] for i in range(2048)]
hv_dict['900gm'] = [self.a2p.gm900[i // 64] for i in range(2048)]
hv_dict['1000gm'] = [self.a2p.gm1000[i // 64] for i in range(2048)]
hv_dict['800gm_c1'] = [self.a2p.gm800_c1[i // 64] for i in range(2048)]
df_list = []
for key, hv in hv_dict.items():
hv_group = int(key.replace("gm", "").replace("_c1", ""))
gm = 'gm' in key
gm_t = 'Gain-matched' if 'gm' in key else 'Non-gain-matched'
adc2pe = self.a2p.get_adc2pe_at_hv(hv, np.arange(2048))
adc2pe = self.dead.mask1d(adc2pe)
spe = 1 / adc2pe
self.cfmaker.SetAll(np.ma.filled(adc2pe, 0).astype(np.float32))
path = join(self.output_dir, "adc2pe_{}.tcal".format(key))
self.cfmaker.Save(path, False)
self.log.info("ADC2PE tcal created: {}".format(path))
self.cfmaker.Clear()
for pix in range(2048):
if pix in self.dead.dead_pixels:
continue
df_list.append(
dict(pixel=pix,
key=key,
hv_group=hv_group,
gm=gm,
gm_t=gm_t,
hv=hv[pix],
adc2pe=adc2pe[pix],
spe=spe[pix]))
df = DataFrame(df_list)
df = df.loc[df['key'] != '800gm_c1']
# Create Plot
fig = plt.figure(figsize=(13, 6))
ax = fig.add_subplot(1, 1, 1)
sns.violinplot(ax=ax,
data=df,
x='hv_group',
y='spe',
hue='gm_t',
split=True,
scale='count',
inner='quartile')
ax.set_xlabel('HV')
ax.set_ylabel('SPE Value (ADC)')
fig_path = join(self.output_dir, "spe_vs_hv.pdf")
fig.savefig(fig_path)
def finish(self):
pass
class BokehGainMatching(Tool):
name = "BokehGainMatching"
description = "Interactively explore the steps in obtaining charge vs hv"
input_path = Unicode('',
help='Path to the numpy array containing the '
'gain and hv').tag(config=True)
aliases = Dict(dict(f='BokehGainMatching.input_path'))
classes = List([])
def __init__(self, **kwargs):
super().__init__(**kwargs)
self._active_pixel = None
self.dead = Dead()
self.charge = None
self.charge_error = None
self.hv = None
self.n_hv = None
self.n_pixels = None
self.n_tmpix = 64
self.modules = None
self.tmpix = None
self.n_tm = None
self.m_pix = None
self.c_pix = None
self.m_tm = None
self.c_tm = None
self.m_tm2048 = None
self.c_tm2048 = None
self.p_camera_pix = None
self.p_plotter_pix = None
self.p_camera_tm = None
self.p_plotter_tm = None
self.w_view_radio = None
self.layout = None
def setup(self):
self.log_format = "%(levelname)s: %(message)s [%(name)s.%(funcName)s]"
kwargs = dict(config=self.config, tool=self)
arrays = np.load(self.input_path)
self.charge = self.dead.mask2d(arrays['charge'])
self.charge = np.ma.masked_where(self.charge <= 0, self.charge)
self.charge_error = np.ma.array(arrays['charge_error'],
mask=self.charge.mask)
self.hv = arrays['rundesc']
self.n_hv, self.n_pixels = self.charge.shape
assert (self.n_hv == self.hv.size)
geom = CameraGeometry.guess(*checm_pixel_pos * u.m,
optical_foclen * u.m)
self.modules = np.arange(self.n_pixels) // self.n_tmpix
self.tmpix = np.arange(self.n_pixels) % self.n_tmpix
self.n_tm = np.unique(self.modules).size
# Init Plots
self.p_camera_pix = Camera(self, "Gain Matching Pixels", geom)
self.p_camera_tm = Camera(self, "Gain Matching TMs", geom)
self.p_plotter_pix = Plotter(**kwargs)
self.p_plotter_tm = Plotter(**kwargs)
def start(self):
# Overcomplicated method instead of just reshaping...
# gain_modules = np.zeros((self.n_hv, self.n_tm, self.n_tmpix))
# hv_r = np.arange(self.n_hv, dtype=np.int)[:, None]
# hv_z = np.zeros(self.n_hv, dtype=np.int)[:, None]
# tm_r = np.arange(self.n_tm, dtype=np.int)[None, :]
# tm_z = np.zeros(self.n_tm, dtype=np.int)[None, :]
# tmpix_r = np.arange(self.n_tmpix, dtype=np.int)[None, :]
# tmpix_z = np.zeros(self.n_tmpix, dtype=np.int)[None, :]
# hv_i = (hv_r + tm_z)[..., None] + tmpix_z
# tm_i = (hv_z + tm_r)[..., None] + tmpix_z
# tmpix_i = (hv_z + tm_z)[..., None] + tmpix_r
# gain_rs = np.reshape(self.charge, (self.n_hv, self.n_tm, self.n_tmpix))
# modules_rs = np.reshape(self.modules, (self.n_tm, self.n_tmpix))
# tmpix_rs = np.reshape(self.tmpix, (self.n_tm, self.n_tmpix))
# tm_j = hv_z[..., None] + modules_rs[None, ...]
# tmpix_j = hv_z[..., None] + tmpix_rs[None, ...]
# gain_modules[hv_i, tm_i, tmpix_i] = gain_rs[hv_i, tm_j, tmpix_j]
# gain_modules_mean = np.mean(gain_modules, axis=2)
shape = (self.n_hv, self.n_tm, self.n_tmpix)
gain_tm = np.reshape(self.charge, shape)
gain_error_tm = np.reshape(self.charge_error, shape)
gain_tm_mean = np.mean(gain_tm, axis=2)
gain_error_tm_mean = np.sqrt(np.sum(gain_error_tm**2, axis=2))
self.m_pix = np.ma.zeros(self.n_pixels, fill_value=0)
self.c_pix = np.ma.zeros(self.n_pixels, fill_value=0)
self.m_tm = np.ma.zeros(self.n_tm, fill_value=0)
self.c_tm = np.ma.zeros(self.n_tm, fill_value=0)
p0 = [0, 5]
bounds = (-np.inf, np.inf) # ([-2000, -10], [2000, 10])
for pix in range(self.n_pixels):
x = self.hv[~self.charge.mask[:, pix]]
y = self.charge[:, pix][~self.charge.mask[:, pix]]
if x.size == 0:
continue
try:
coeff, _ = curve_fit(
gain_func,
x,
y,
p0=p0,
bounds=bounds,
# sigma=y_err[:, pix],
# absolute_sigma=True
)
self.c_pix[pix], self.m_pix[pix] = coeff
except RuntimeError:
self.log.warning("Unable to fit pixel: {}".format(pix))
for tm in range(self.n_tm):
x = self.hv
y = gain_tm_mean[:, tm]
try:
coeff, _ = curve_fit(
gain_func,
x,
y,
p0=p0,
bounds=bounds,
# sigma=y_err_tm[:, tm],
# absolute_sigma=True
)
self.c_tm[tm], self.m_tm[tm] = coeff
except RuntimeError:
self.log.warning("Unable to fit tm: {}".format(tm))
self.m_tm2048 = self.m_tm[:, None] * np.ones((self.n_tm, self.n_tmpix))
self.c_tm2048 = self.c_tm[:, None] * np.ones((self.n_tm, self.n_tmpix))
self.m_pix = self.dead.mask1d(self.m_pix)
self.c_pix = self.dead.mask1d(self.c_pix)
self.m_tm2048 = self.dead.mask1d(self.m_tm2048.ravel())
self.c_tm2048 = self.dead.mask1d(self.c_tm2048.ravel())
# Setup Plots
self.p_camera_pix.enable_pixel_picker()
self.p_camera_pix.add_colorbar()
self.p_camera_tm.enable_pixel_picker()
self.p_camera_tm.add_colorbar()
self.p_plotter_pix.create(self.hv, self.charge, self.charge_error,
self.m_pix, self.c_pix)
self.p_plotter_tm.create(self.hv, gain_tm_mean, gain_error_tm_mean,
self.m_tm, self.c_tm)
# Setup widgets
self.create_view_radio_widget()
self.set_camera_image()
self.active_pixel = 0
# Get bokeh layouts
l_camera_pix = self.p_camera_pix.layout
l_camera_tm = self.p_camera_tm.layout
l_plotter_pix = self.p_plotter_pix.layout
l_plotter_tm = self.p_plotter_tm.layout
# Setup layout
self.layout = layout([[self.w_view_radio],
[l_camera_pix, l_plotter_pix],
[l_camera_tm, l_plotter_tm]])
def finish(self):
curdoc().add_root(self.layout)
curdoc().title = "Charge Vs HV"
output_dir = dirname(self.input_path)
output_path = join(output_dir, 'gain_matching_coeff.npz')
np.savez(output_path,
alpha_pix=np.ma.filled(self.m_pix),
C_pix=np.ma.filled(self.c_pix),
alpha_tm=np.ma.filled(self.m_tm),
C_tm=np.ma.filled(self.c_tm))
self.log.info("Numpy array saved to: {}".format(output_path))
@property
def active_pixel(self):
return self._active_pixel
@active_pixel.setter
def active_pixel(self, val):
if not self._active_pixel == val:
self._active_pixel = val
self.p_camera_pix.active_pixel = val
self.p_camera_tm.active_pixel = val
self.p_plotter_pix.active_pixel = val
self.p_plotter_pix.fig.title.text = 'Pixel {}'.format(val)
module = self.modules[val]
self.p_plotter_tm.active_pixel = module
self.p_plotter_tm.fig.title.text = 'TM {}'.format(module)
def set_camera_image(self):
if self.w_view_radio.active == 0:
self.p_camera_pix.image = self.m_pix
self.p_camera_tm.image = self.m_tm2048
self.p_camera_pix.fig.title.text = 'Gain Matching Pixels (gradient)'
self.p_camera_tm.fig.title.text = 'Gain Matching TMs (gradient)'
elif self.w_view_radio.active == 1:
self.p_camera_pix.image = self.c_pix
self.p_camera_tm.image = self.c_tm2048
self.p_camera_pix.fig.title.text = 'Gain Matching Pixels (intercept)'
self.p_camera_tm.fig.title.text = 'Gain Matching TMs (intercept)'
def create_view_radio_widget(self):
self.w_view_radio = RadioButtonGroup(labels=["gradient", "intercept"],
active=0)
self.w_view_radio.on_click(self.on_view_radio_widget_click)
def on_view_radio_widget_click(self, active):
self.set_camera_image()
class ADC2PEResidualsPlotter(Tool):
name = "ADC2PEResidualsPlotter"
description = "Plot the residuals from the adc2pe calibration"
input_path = Unicode("", help="Path to the adc2pe_residuals numpy "
"file").tag(config=True)
aliases = Dict(dict(i='ADC2PEResidualsPlotter.input_path', ))
classes = List([])
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.dead = None
self.output_dir = None
self.spe = None
self.spe_sigma = None
self.hist = None
self.edges = None
self.between = None
self.fig_spectrum_all = None
self.fig_spectrum_tm_list = None
self.fig_combgaus = None
self.fig_kde = None
self.fig_hist = None
def setup(self):
self.log_format = "%(levelname)s: %(message)s [%(name)s.%(funcName)s]"
self.dead = Dead()
file = np.load(self.input_path)
self.spe = file['spe']
self.spe_sigma = file['spe_sigma']
self.hist = file['hist']
self.edges = file['edges']
self.between = file['between']
self.output_dir = join(dirname(self.input_path),
"plot_adc2pe_residuals")
if not exists(self.output_dir):
self.log.info("Creating directory: {}".format(self.output_dir))
makedirs(self.output_dir)
# Create figures
sns.set_style("whitegrid")
sns.despine()
self.fig_spectrum_all = plt.figure(figsize=(13, 6))
self.fig_spectrum_all.suptitle("SPE Spectrum, All Pixels")
self.fig_spectrum_tm_list = []
for i in range(32):
fig = plt.figure(figsize=(13, 6))
self.fig_spectrum_tm_list.append(plt.figure(figsize=(13, 6)))
self.fig_combgaus = plt.figure(figsize=(13, 6))
self.fig_combgaus.suptitle("Combined 1pe fit, All Pixels")
self.fig_kde = plt.figure(figsize=(13, 6))
self.fig_kde.suptitle("Distribution of SPE, Kernel density estimate")
self.fig_hist = plt.figure(figsize=(13, 6))
self.fig_hist.suptitle("Distribution of SPE, Histogram")
def start(self):
# Normalise histogram
norm = np.sum(np.diff(self.edges, axis=1) * self.hist, axis=1)
hist = self.hist / norm[:, None]
# Roll axis for easier plotting
hist_r = np.rollaxis(hist, 1)
nbins, npix = hist_r.shape
e = self.edges[0]
hist_tops = np.insert(hist_r, np.arange(nbins), hist_r, axis=0)
edges_tops = np.insert(e, np.arange(e.shape[0]), e, axis=0)[1:-1]
# Mask dead pixels
spe = self.dead.mask1d(self.spe)
spe_sigma = self.dead.mask1d(self.spe_sigma)
hist_tops = self.dead.mask2d(hist_tops)
# Spectrum with all pixels
self.log.info("Plotting: spectrum_all")
ax_spectrum_all = self.fig_spectrum_all.add_subplot(1, 1, 1)
ax_spectrum_all.semilogy(edges_tops, hist_tops, color='b', alpha=0.2)
ax_spectrum_all.set_xlabel("Amplitude (p.e.)")
ax_spectrum_all.set_ylabel("Probability")
# Sprectrum for each tm
self.log.info("Plotting: spectrum_tm")
hist_tops_tm = np.reshape(hist_tops, (hist_tops.shape[0], 32, 64))
for tm, fig in enumerate(self.fig_spectrum_tm_list):
ax = fig.add_subplot(1, 1, 1)
ax.set_title("SPE Spectrum, TM {}".format(tm))
ax.semilogy(edges_tops, hist_tops_tm[:, tm], color='b', alpha=0.2)
ax.set_xlabel("Amplitude (p.e.)")
ax.set_ylabel("Probability")
# Combined gaussian of each spe value
self.log.info("Plotting: combined_gaussian")
ax_comgaus = self.fig_combgaus.add_subplot(1, 1, 1)
x = np.linspace(-1, 4, 200)
kernels = []
for val, sigma in zip(spe.compressed(), spe_sigma.compressed()):
kernel = stats.norm(val, sigma).pdf(x)
kernels.append(kernel)
# plt.plot(x, kernel, color="r")
sns.rugplot(spe.compressed(), color=".2", linewidth=1, ax=ax_comgaus)
density = np.sum(kernels, axis=0)
density /= integrate.trapz(density, x)
ax_comgaus.plot(x, density)
ax_comgaus.set_xlabel("SPE Fit Value (p.e.)")
ax_comgaus.set_ylabel("Sum")
# Kernel density estimate
self.log.info("Plotting: spe_kde")
ax_kde = self.fig_kde.add_subplot(1, 1, 1)
sns.rugplot(spe.compressed(), color=".2", linewidth=1, ax=ax_kde)
sns.kdeplot(spe.compressed(), shade=True, ax=ax_kde)
ax_kde.set_xlabel("SPE Fit Value (p.e.)")
ax_kde.set_ylabel("KDE")
# Histogram
self.log.info("Plotting: histogram")
ax_hist = self.fig_hist.add_subplot(1, 1, 1)
sns.distplot(spe.compressed(), kde=False, rug=True, ax=ax_hist)
ax_hist.set_xlabel("SPE Fit Value (p.e.)")
ax_hist.set_ylabel("N")
def finish(self):
output_path = join(self.output_dir, "spectrum_all.png")
self.fig_spectrum_all.savefig(output_path)
self.log.info("Created figure: {}".format(output_path))
output_path = join(self.output_dir, "spectrum_tm{}.png")
for tm, fig in enumerate(self.fig_spectrum_tm_list):
p = output_path.format(tm)
fig.savefig(p)
self.log.info("Created figure: {}".format(p))
output_path = join(self.output_dir, "combined_gaussian.png")
self.fig_combgaus.savefig(output_path)
self.log.info("Created figure: {}".format(output_path))
output_path = join(self.output_dir, "kde.png")
self.fig_kde.savefig(output_path)
self.log.info("Created figure: {}".format(output_path))
output_path = join(self.output_dir, "hist.png")
self.fig_hist.savefig(output_path)
self.log.info("Created figure: {}".format(output_path))
class SPEExtractor(Tool):
name = "SPEExtractor"
description = "Extract the conversion from adc to pe and save as a " \
"numpy array"
aliases = Dict(
dict(
r='EventFileReaderFactory.reader',
f='EventFileReaderFactory.input_path',
max_events='EventFileReaderFactory.max_events',
ped='CameraR1CalibratorFactory.pedestal_path',
tf='CameraR1CalibratorFactory.tf_path',
fitter='SPEFitterFactory.fitter',
))
classes = List([
EventFileReaderFactory,
CameraR1CalibratorFactory,
SPEFitterFactory,
])
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.reader = None
self.r1 = None
self.dl0 = None
self.cleaner = None
self.extractor = None
self.dl1 = None
self.fitter = None
self.dead = None
self.output_dir = None
self.spe = None
def setup(self):
self.log_format = "%(levelname)s: %(message)s [%(name)s.%(funcName)s]"
kwargs = dict(config=self.config, tool=self)
reader_factory = EventFileReaderFactory(**kwargs)
reader_class = reader_factory.get_class()
self.reader = reader_class(**kwargs)
r1_factory = CameraR1CalibratorFactory(origin=self.reader.origin,
**kwargs)
r1_class = r1_factory.get_class()
self.r1 = r1_class(**kwargs)
self.cleaner = CHECMWaveformCleanerAverage(**kwargs)
self.extractor = AverageWfPeakIntegrator(**kwargs)
self.dl0 = CameraDL0Reducer(**kwargs)
self.dl1 = CameraDL1Calibrator(extractor=self.extractor,
cleaner=self.cleaner,
**kwargs)
self.dead = Dead()
fitter_factory = SPEFitterFactory(**kwargs)
fitter_class = fitter_factory.get_class()
self.fitter = fitter_class(**kwargs)
self.output_dir = join(self.reader.output_directory, "extract_spe")
if not exists(self.output_dir):
self.log.info("Creating directory: {}".format(self.output_dir))
makedirs(self.output_dir)
def start(self):
n_events = self.reader.num_events
first_event = self.reader.get_event(0)
telid = list(first_event.r0.tels_with_data)[0]
n_pixels, n_samples = first_event.r0.tel[telid].adc_samples[0].shape
# Prepare storage array
area = np.zeros((n_events, n_pixels))
spe = np.ma.zeros(n_pixels)
spe.mask = np.zeros(spe.shape, dtype=np.bool)
spe.fill_value = 0
source = self.reader.read()
desc = "Looping through file"
with tqdm(total=n_events, desc=desc) as pbar:
for event in source:
pbar.update(1)
index = event.count
self.r1.calibrate(event)
self.dl0.reduce(event)
self.dl1.calibrate(event)
# Perform CHECM Charge Extraction
peak_area = event.dl1.tel[telid].image
area[index] = peak_area
desc = "Fitting pixels"
with tqdm(total=n_pixels, desc=desc) as pbar:
for pix in range(n_pixels):
pbar.update(1)
if not self.fitter.apply(area[:, pix]):
self.log.warning("Pixel {} couldn't be fit".format(pix))
spe.mask = True
continue
spe[pix] = self.fitter.coeff['spe']
self.spe = np.ma.filled(self.dead.mask1d(spe))
def finish(self):
output_path = join(self.output_dir, "spe.npy")
np.save(output_path, self.spe)
self.log.info("spe array saved: {}".format(output_path))
class BokehSPE(Tool):
name = "BokehSPE"
description = "Interactively explore the steps in obtaining and fitting " \
"SPE spectrum"
aliases = Dict(
dict(
r='EventFileReaderFactory.reader',
f='EventFileReaderFactory.input_path',
max_events='EventFileReaderFactory.max_events',
ped='CameraR1CalibratorFactory.pedestal_path',
tf='CameraR1CalibratorFactory.tf_path',
pe='CameraR1CalibratorFactory.pe_path',
fitter='ChargeFitterFactory.fitter',
))
classes = List([
EventFileReaderFactory,
CameraR1CalibratorFactory,
ChargeFitterFactory,
])
def __init__(self, **kwargs):
super().__init__(**kwargs)
self._event = None
self._event_index = None
self._event_id = None
self._active_pixel = 0
self.w_event_index = None
self.w_goto_event_index = None
self.w_hoa = None
self.w_fitspectrum = None
self.w_fitcamera = None
self.layout = None
self.reader = None
self.r1 = None
self.dl0 = None
self.dl1 = None
self.dl1_height = None
self.area = None
self.height = None
self.n_events = None
self.n_pixels = None
self.n_samples = None
self.cleaner = None
self.extractor = None
self.extractor_height = None
self.dead = None
self.fitter = None
self.neighbours2d = None
self.stage_names = None
self.p_camera_area = None
self.p_camera_fit_gain = None
self.p_camera_fit_brightness = None
self.p_fitter = None
self.p_stage_viewer = None
self.p_fit_viewer = None
self.p_fit_table = None
def setup(self):
self.log_format = "%(levelname)s: %(message)s [%(name)s.%(funcName)s]"
kwargs = dict(config=self.config, tool=self)
reader_factory = EventFileReaderFactory(**kwargs)
reader_class = reader_factory.get_class()
self.reader = reader_class(**kwargs)
r1_factory = CameraR1CalibratorFactory(origin=self.reader.origin,
**kwargs)
r1_class = r1_factory.get_class()
self.r1 = r1_class(**kwargs)
self.dl0 = CameraDL0Reducer(**kwargs)
self.cleaner = CHECMWaveformCleanerAverage(**kwargs)
self.extractor = AverageWfPeakIntegrator(**kwargs)
self.extractor_height = SimpleIntegrator(window_shift=0,
window_width=1,
**kwargs)
self.dl1 = CameraDL1Calibrator(extractor=self.extractor,
cleaner=self.cleaner,
**kwargs)
self.dl1_height = CameraDL1Calibrator(extractor=self.extractor_height,
cleaner=self.cleaner,
**kwargs)
self.dead = Dead()
fitter_factory = ChargeFitterFactory(**kwargs)
fitter_class = fitter_factory.get_class()
self.fitter = fitter_class(**kwargs)
self.n_events = self.reader.num_events
first_event = self.reader.get_event(0)
self.n_pixels = first_event.inst.num_pixels[0]
self.n_samples = first_event.r0.tel[0].num_samples
geom = CameraGeometry.guess(*first_event.inst.pixel_pos[0],
first_event.inst.optical_foclen[0])
self.neighbours2d = get_neighbours_2d(geom.pix_x, geom.pix_y)
# Get stage names
self.stage_names = [
'0: raw', '1: baseline_sub', '2: no_pulse', '3: smooth_baseline',
'4: smooth_wf', '5: cleaned'
]
# Init Plots
self.p_camera_area = Camera(self, self.neighbours2d, "Area", geom)
self.p_camera_fit_gain = Camera(self, self.neighbours2d, "Gain", geom)
self.p_camera_fit_brightness = Camera(self, self.neighbours2d,
"Brightness", geom)
self.p_fitter = FitterWidget(fitter=self.fitter, **kwargs)
self.p_stage_viewer = StageViewer(**kwargs)
self.p_fit_viewer = FitViewer(**kwargs)
self.p_fit_table = FitTable(**kwargs)
def start(self):
# Prepare storage array
self.area = np.zeros((self.n_events, self.n_pixels))
self.height = np.zeros((self.n_events, self.n_pixels))
source = self.reader.read()
desc = "Looping through file"
for event in tqdm(source, total=self.n_events, desc=desc):
index = event.count
self.r1.calibrate(event)
self.dl0.reduce(event)
self.dl1.calibrate(event)
peak_area = np.copy(event.dl1.tel[0].image)
self.dl1_height.calibrate(event)
peak_height = np.copy(event.dl1.tel[0].image)
self.area[index] = peak_area
self.height[index] = peak_height
# Setup Plots
self.p_camera_area.enable_pixel_picker()
self.p_camera_area.add_colorbar()
self.p_camera_fit_gain.enable_pixel_picker()
self.p_camera_fit_gain.add_colorbar()
self.p_camera_fit_brightness.enable_pixel_picker()
self.p_camera_fit_brightness.add_colorbar()
self.p_fitter.create()
self.p_stage_viewer.create(self.neighbours2d, self.stage_names)
self.p_fit_viewer.create(self.p_fitter.fitter.subfit_labels)
self.p_fit_table.create()
# Setup widgets
self.create_event_index_widget()
self.create_goto_event_index_widget()
self.event_index = 0
self.create_hoa_widget()
self.create_fitspectrum_widget()
self.create_fitcamera_widget()
# Get bokeh layouts
l_camera_area = self.p_camera_area.layout
l_camera_fit_gain = self.p_camera_fit_gain.layout
l_camera_fit_brightness = self.p_camera_fit_brightness.layout
l_fitter = self.p_fitter.layout
l_stage_viewer = self.p_stage_viewer.layout
l_fit_viewer = self.p_fit_viewer.layout
l_fit_table = self.p_fit_table.layout
# Setup layout
self.layout = layout([
[self.w_hoa, self.w_fitspectrum, self.w_fitcamera],
[l_camera_fit_brightness, l_fit_viewer, l_fitter],
[l_camera_fit_gain, l_fit_table],
[l_camera_area, self.w_goto_event_index, self.w_event_index],
[Div(text="Stage Viewer")],
[l_stage_viewer],
])
def finish(self):
curdoc().add_root(self.layout)
curdoc().title = "Event Viewer"
def fit_spectrum(self, pix):
if self.w_hoa.active == 0:
spectrum = self.area
else:
spectrum = self.height
success = self.p_fitter.fit(spectrum[:, pix])
if not success:
self.log.warning("Pixel {} couldn't be fit".format(pix))
return success
def fit_camera(self):
gain = np.ma.zeros(self.n_pixels)
gain.mask = np.zeros(gain.shape, dtype=np.bool)
brightness = np.ma.zeros(self.n_pixels)
brightness.mask = np.zeros(gain.shape, dtype=np.bool)
fitter = self.p_fitter.fitter.fitter_type
if fitter == 'spe':
coeff = 'lambda_'
elif fitter == 'bright':
coeff = 'mean'
else:
self.log.error("No case for fitter type: {}".format(fitter))
raise ValueError
desc = "Fitting pixels"
for pix in trange(self.n_pixels, desc=desc):
if not self.fit_spectrum(pix):
gain.mask[pix] = True
continue
if fitter == 'spe':
gain[pix] = self.p_fitter.fitter.coeff['spe']
brightness[pix] = self.p_fitter.fitter.coeff[coeff]
gain = np.ma.masked_where(np.isnan(gain), gain)
gain = self.dead.mask1d(gain)
brightness = np.ma.masked_where(np.isnan(brightness), brightness)
brightness = self.dead.mask1d(brightness)
self.p_camera_fit_gain.image = gain
self.p_camera_fit_brightness.image = brightness
@property
def event(self):
return self._event
@event.setter
def event(self, val):
self._event = val
self.r1.calibrate(val)
self.dl0.reduce(val)
self.dl1.calibrate(val)
peak_area = val.dl1.tel[0].image
self._event_index = val.count
self._event_id = val.r0.event_id
self.update_event_index_widget()
stages = self.dl1.cleaner.stages
pulse_window = self.dl1.cleaner.stages['window'][0]
int_window = val.dl1.tel[0].extracted_samples[0]
self.p_camera_area.image = peak_area
self.p_stage_viewer.update_stages(np.arange(self.n_samples), stages,
pulse_window, int_window)
@property
def event_index(self):
return self._event_index
@event_index.setter
def event_index(self, val):
self._event_index = val
self.event = self.reader.get_event(val, False)
@property
def active_pixel(self):
return self._active_pixel
@active_pixel.setter
def active_pixel(self, val):
if not self._active_pixel == val:
self._active_pixel = val
self.fit_spectrum(val)
self.p_camera_area.active_pixel = val
self.p_camera_fit_gain.active_pixel = val
self.p_camera_fit_brightness.active_pixel = val
self.p_stage_viewer.active_pixel = val
self.p_fit_viewer.update(self.p_fitter.fitter)
self.p_fit_table.update(self.p_fitter.fitter)
def create_event_index_widget(self):
self.w_event_index = TextInput(title="Event Index:", value='')
def update_event_index_widget(self):
if self.w_event_index:
self.w_event_index.value = str(self.event_index)
def create_goto_event_index_widget(self):
self.w_goto_event_index = Button(label="GOTO Index", width=100)
self.w_goto_event_index.on_click(self.on_goto_event_index_widget_click)
def on_goto_event_index_widget_click(self):
self.event_index = int(self.w_event_index.value)
def on_event_index_widget_change(self, attr, old, new):
if self.event_index != int(self.w_event_index.value):
self.event_index = int(self.w_event_index.value)
def create_hoa_widget(self):
self.w_hoa = RadioGroup(labels=['area', 'height'], active=0)
self.w_hoa.on_click(self.on_hoa_widget_select)
def on_hoa_widget_select(self, active):
self.fit_spectrum(self.active_pixel)
self.p_fit_viewer.update(self.p_fitter.fitter)
self.p_fit_table.update(self.p_fitter.fitter)
def create_fitspectrum_widget(self):
self.w_fitspectrum = Button(label='Fit Spectrum')
self.w_fitspectrum.on_click(self.on_fitspectrum_widget_select)
def on_fitspectrum_widget_select(self):
t = time()
self.fit_spectrum(self.active_pixel)
self.log.info("Fit took {} seconds".format(time() - t))
self.p_fit_viewer.update(self.p_fitter.fitter)
self.p_fit_table.update(self.p_fitter.fitter)
def create_fitcamera_widget(self):
self.w_fitcamera = Button(label='Fit Camera')
self.w_fitcamera.on_click(self.on_fitcamera_widget_select)
def on_fitcamera_widget_select(self):
self.fit_camera()
self.fit_spectrum(self.active_pixel)
self.p_fit_viewer.update(self.p_fitter.fitter)
self.p_fit_table.update(self.p_fitter.fitter)