class BokehGainMatching(Tool):
name = "BokehGainMatching"
description = "Interactively explore the steps in obtaining charge vs hv"
input_path = Unicode('',
help='Path to the numpy array containing the '
'gain and hv').tag(config=True)
aliases = Dict(dict(f='BokehGainMatching.input_path'))
classes = List([])
def __init__(self, **kwargs):
super().__init__(**kwargs)
self._active_pixel = None
self._active_run = None
self.dead = Dead()
self.charge = None
self.charge_error = None
self.rundesc = None
self.charge_tm = None
self.charge_error_tm = None
self.mean_tm2048 = None
self.tmpixspread_tm2048 = None
self.n_run = None
self.n_pixels = None
self.n_tmpix = 64
self.modules = None
self.tmpix = None
self.n_tm = None
self.p_c_pix = None
self.p_p_pix = None
self.p_c_tm = None
self.p_p_tm = None
self.p_c_tmpixspread = None
self.p_b_tmpixspread = None
self.p_b_tmspread = None
self.p_b_pixspread = None
self.p_c_pix_title = 'Charge Across Pixels, Run: {}'
self.p_c_tm_title = 'Mean Charge Across TMs, Run: {}'
self.p_c_tmpixspread_title = 'Median Charge Across TMs, Run: {}'
self.p_p_pix_title = 'Charge vs Runs, Error: fit stddev, Pixel: {}'
self.p_p_tm_title = 'Charge vs Runs, Error: combined pixel, TM: {}'
self.p_b_tmpixspread_title = 'Charge Spread vs Runs, TM: {}'
self.layout = None
def setup(self):
self.log_format = "%(levelname)s: %(message)s [%(name)s.%(funcName)s]"
kwargs = dict(config=self.config, tool=self)
arrays = np.load(self.input_path)
self.charge = self.dead.mask2d(arrays['charge'])
self.charge_error = self.dead.mask2d(arrays['charge_error'])
self.rundesc = arrays['rundesc']
self.n_run, self.n_pixels = self.charge.shape
assert (self.n_run == self.rundesc.size)
geom = CameraGeometry.guess(*checm_pixel_pos * u.m,
optical_foclen * u.m)
self.modules = np.arange(self.n_pixels) // self.n_tmpix
self.tmpix = np.arange(self.n_pixels) % self.n_tmpix
self.n_tm = np.unique(self.modules).size
# Init Plots
self.p_c_pix = Camera(self, "", geom)
self.p_c_tm = Camera(self, "", geom)
self.p_c_tmpixspread = Camera(self, "", geom)
self.p_p_pix = Plotter(**kwargs)
self.p_p_tm = Plotter(**kwargs)
self.p_b_tmpixspread = BoxPlotter(**kwargs)
self.p_b_tmspread = BoxPlotter(**kwargs)
self.p_b_pixspread = BoxPlotter(**kwargs)
def start(self):
shape_tm = (self.n_run, self.n_tm, self.n_tmpix)
shape_4d = (self.n_run, self.n_tm, self.n_tmpix, self.n_tmpix)
shape_pix = (self.n_run, self.n_pixels, self.n_tmpix)
self.charge_tm = np.reshape(self.charge, shape_tm)
self.charge_error_tm = np.reshape(self.charge_error, shape_tm)
charge_tm_mean = np.mean(self.charge_tm, axis=2)
charge_error_tm_mean = np.sqrt(np.sum(self.charge_error_tm**2, axis=2))
self.mean_tm2048 = charge_tm_mean[..., None] * np.ones(shape_tm)
tm_spread = self.charge_tm[:, :, None, :] * np.ones(shape_4d)
self.tmpixspread_tm2048 = np.reshape(tm_spread, shape_pix)
# Setup Plots
self.p_c_pix.enable_pixel_picker()
self.p_c_pix.add_colorbar()
self.p_c_tm.enable_pixel_picker()
self.p_c_tm.add_colorbar()
self.p_c_tmpixspread.enable_pixel_picker()
self.p_c_tmpixspread.add_colorbar()
self.p_p_pix.create(self.rundesc, self.charge, self.charge_error)
self.p_p_tm.create(self.rundesc, charge_tm_mean, charge_error_tm_mean)
self.p_b_tmpixspread.create()
self.p_b_tmspread.create()
self.p_b_pixspread.create()
self.p_b_tmspread.fig.title.text = 'Mean TM Charge Spread vs Runs'
self.p_b_pixspread.fig.title.text = 'Pixel Spread vs Runs'
self.p_b_tmspread.update(self.rundesc, charge_tm_mean)
self.p_b_pixspread.update(self.rundesc, self.charge)
self.p_p_pix.enable_run_picker()
self.p_p_tm.enable_run_picker()
self.p_b_tmpixspread.enable_run_picker()
self.p_b_tmspread.enable_run_picker()
self.p_b_pixspread.enable_run_picker()
# Setup widgets
self.active_pixel = 0
self.active_run = 0
# Get bokeh layouts
l_camera_pix = self.p_c_pix.layout
l_camera_tm = self.p_c_tm.layout
l_camera_tmpixspread = self.p_c_tmpixspread.layout
l_plotter_pix = self.p_p_pix.layout
l_plotter_tm = self.p_p_tm.layout
l_boxplotter_tmpixspread = self.p_b_tmpixspread.layout
l_boxplotter_tmspread = self.p_b_tmspread.layout
l_boxplotter_pixspread = self.p_b_pixspread.layout
# Setup layout
self.layout = layout([[l_camera_pix, l_plotter_pix],
[l_camera_tm, l_plotter_tm],
[l_camera_tmpixspread, l_boxplotter_tmpixspread],
[l_boxplotter_tmspread, l_boxplotter_pixspread]])
def finish(self):
curdoc().add_root(self.layout)
curdoc().title = "Charge Vs Run"
@property
def active_pixel(self):
return self._active_pixel
@active_pixel.setter
def active_pixel(self, val):
if not self._active_pixel == val:
self._active_pixel = val
self.p_c_pix.active_pixel = val
self.p_c_tm.active_pixel = val
self.p_c_tmpixspread.active_pixel = val
self.p_p_pix.active_pixel = val
self.p_p_pix.fig.title.text = self.p_p_pix_title.format(val)
module = self.modules[val]
self.p_p_tm.active_pixel = module
self.p_p_tm.fig.title.text = self.p_p_tm_title.format(module)
self.p_b_tmpixspread.update(self.rundesc,
self.tmpixspread_tm2048[:, val])
t = self.p_b_tmpixspread_title
self.p_b_tmpixspread.fig.title.text = t.format(module)
@property
def active_run(self):
return self._active_run
@active_run.setter
def active_run(self, val):
if not self._active_run == val:
self._active_run = val
self.p_p_pix.active_run = val
self.p_p_tm.active_run = val
self.p_b_tmpixspread.active_run = val
self.p_b_tmspread.active_run = val
self.p_b_pixspread.active_run = val
self.set_camera_image()
self.p_c_pix.fig.title.text = self.p_c_pix_title.format(val)
self.p_c_tm.fig.title.text = self.p_c_tm_title.format(val)
t = self.p_c_tmpixspread_title
self.p_c_tmpixspread.fig.title.text = t.format(val)
def set_camera_image(self):
r = self.active_run
self.p_c_pix.image = self.charge[r]
self.p_c_tm.image = self.mean_tm2048[r]
self.p_c_tmpixspread.image = np.median(self.tmpixspread_tm2048[r],
axis=1)
class BokehGainMatching(Tool):
name = "BokehGainMatching"
description = "Interactively explore the steps in obtaining charge vs hv"
input_path = Unicode('',
help='Path to the numpy array containing the '
'gain and hv').tag(config=True)
aliases = Dict(dict(f='BokehGainMatching.input_path'))
classes = List([])
def __init__(self, **kwargs):
super().__init__(**kwargs)
self._active_pixel = None
self.dead = Dead()
self.charge = None
self.charge_error = None
self.hv = None
self.n_hv = None
self.n_pixels = None
self.n_tmpix = 64
self.modules = None
self.tmpix = None
self.n_tm = None
self.m_pix = None
self.c_pix = None
self.m_tm = None
self.c_tm = None
self.m_tm2048 = None
self.c_tm2048 = None
self.p_camera_pix = None
self.p_plotter_pix = None
self.p_camera_tm = None
self.p_plotter_tm = None
self.w_view_radio = None
self.layout = None
def setup(self):
self.log_format = "%(levelname)s: %(message)s [%(name)s.%(funcName)s]"
kwargs = dict(config=self.config, tool=self)
arrays = np.load(self.input_path)
self.charge = self.dead.mask2d(arrays['charge'])
self.charge = np.ma.masked_where(self.charge <= 0, self.charge)
self.charge_error = np.ma.array(arrays['charge_error'],
mask=self.charge.mask)
self.hv = arrays['rundesc']
self.n_hv, self.n_pixels = self.charge.shape
assert (self.n_hv == self.hv.size)
geom = CameraGeometry.guess(*checm_pixel_pos * u.m,
optical_foclen * u.m)
self.modules = np.arange(self.n_pixels) // self.n_tmpix
self.tmpix = np.arange(self.n_pixels) % self.n_tmpix
self.n_tm = np.unique(self.modules).size
# Init Plots
self.p_camera_pix = Camera(self, "Gain Matching Pixels", geom)
self.p_camera_tm = Camera(self, "Gain Matching TMs", geom)
self.p_plotter_pix = Plotter(**kwargs)
self.p_plotter_tm = Plotter(**kwargs)
def start(self):
# Overcomplicated method instead of just reshaping...
# gain_modules = np.zeros((self.n_hv, self.n_tm, self.n_tmpix))
# hv_r = np.arange(self.n_hv, dtype=np.int)[:, None]
# hv_z = np.zeros(self.n_hv, dtype=np.int)[:, None]
# tm_r = np.arange(self.n_tm, dtype=np.int)[None, :]
# tm_z = np.zeros(self.n_tm, dtype=np.int)[None, :]
# tmpix_r = np.arange(self.n_tmpix, dtype=np.int)[None, :]
# tmpix_z = np.zeros(self.n_tmpix, dtype=np.int)[None, :]
# hv_i = (hv_r + tm_z)[..., None] + tmpix_z
# tm_i = (hv_z + tm_r)[..., None] + tmpix_z
# tmpix_i = (hv_z + tm_z)[..., None] + tmpix_r
# gain_rs = np.reshape(self.charge, (self.n_hv, self.n_tm, self.n_tmpix))
# modules_rs = np.reshape(self.modules, (self.n_tm, self.n_tmpix))
# tmpix_rs = np.reshape(self.tmpix, (self.n_tm, self.n_tmpix))
# tm_j = hv_z[..., None] + modules_rs[None, ...]
# tmpix_j = hv_z[..., None] + tmpix_rs[None, ...]
# gain_modules[hv_i, tm_i, tmpix_i] = gain_rs[hv_i, tm_j, tmpix_j]
# gain_modules_mean = np.mean(gain_modules, axis=2)
shape = (self.n_hv, self.n_tm, self.n_tmpix)
gain_tm = np.reshape(self.charge, shape)
gain_error_tm = np.reshape(self.charge_error, shape)
gain_tm_mean = np.mean(gain_tm, axis=2)
gain_error_tm_mean = np.sqrt(np.sum(gain_error_tm**2, axis=2))
self.m_pix = np.ma.zeros(self.n_pixels, fill_value=0)
self.c_pix = np.ma.zeros(self.n_pixels, fill_value=0)
self.m_tm = np.ma.zeros(self.n_tm, fill_value=0)
self.c_tm = np.ma.zeros(self.n_tm, fill_value=0)
p0 = [0, 5]
bounds = (-np.inf, np.inf) # ([-2000, -10], [2000, 10])
for pix in range(self.n_pixels):
x = self.hv[~self.charge.mask[:, pix]]
y = self.charge[:, pix][~self.charge.mask[:, pix]]
if x.size == 0:
continue
try:
coeff, _ = curve_fit(
gain_func,
x,
y,
p0=p0,
bounds=bounds,
# sigma=y_err[:, pix],
# absolute_sigma=True
)
self.c_pix[pix], self.m_pix[pix] = coeff
except RuntimeError:
self.log.warning("Unable to fit pixel: {}".format(pix))
for tm in range(self.n_tm):
x = self.hv
y = gain_tm_mean[:, tm]
try:
coeff, _ = curve_fit(
gain_func,
x,
y,
p0=p0,
bounds=bounds,
# sigma=y_err_tm[:, tm],
# absolute_sigma=True
)
self.c_tm[tm], self.m_tm[tm] = coeff
except RuntimeError:
self.log.warning("Unable to fit tm: {}".format(tm))
self.m_tm2048 = self.m_tm[:, None] * np.ones((self.n_tm, self.n_tmpix))
self.c_tm2048 = self.c_tm[:, None] * np.ones((self.n_tm, self.n_tmpix))
self.m_pix = self.dead.mask1d(self.m_pix)
self.c_pix = self.dead.mask1d(self.c_pix)
self.m_tm2048 = self.dead.mask1d(self.m_tm2048.ravel())
self.c_tm2048 = self.dead.mask1d(self.c_tm2048.ravel())
# Setup Plots
self.p_camera_pix.enable_pixel_picker()
self.p_camera_pix.add_colorbar()
self.p_camera_tm.enable_pixel_picker()
self.p_camera_tm.add_colorbar()
self.p_plotter_pix.create(self.hv, self.charge, self.charge_error,
self.m_pix, self.c_pix)
self.p_plotter_tm.create(self.hv, gain_tm_mean, gain_error_tm_mean,
self.m_tm, self.c_tm)
# Setup widgets
self.create_view_radio_widget()
self.set_camera_image()
self.active_pixel = 0
# Get bokeh layouts
l_camera_pix = self.p_camera_pix.layout
l_camera_tm = self.p_camera_tm.layout
l_plotter_pix = self.p_plotter_pix.layout
l_plotter_tm = self.p_plotter_tm.layout
# Setup layout
self.layout = layout([[self.w_view_radio],
[l_camera_pix, l_plotter_pix],
[l_camera_tm, l_plotter_tm]])
def finish(self):
curdoc().add_root(self.layout)
curdoc().title = "Charge Vs HV"
output_dir = dirname(self.input_path)
output_path = join(output_dir, 'gain_matching_coeff.npz')
np.savez(output_path,
alpha_pix=np.ma.filled(self.m_pix),
C_pix=np.ma.filled(self.c_pix),
alpha_tm=np.ma.filled(self.m_tm),
C_tm=np.ma.filled(self.c_tm))
self.log.info("Numpy array saved to: {}".format(output_path))
@property
def active_pixel(self):
return self._active_pixel
@active_pixel.setter
def active_pixel(self, val):
if not self._active_pixel == val:
self._active_pixel = val
self.p_camera_pix.active_pixel = val
self.p_camera_tm.active_pixel = val
self.p_plotter_pix.active_pixel = val
self.p_plotter_pix.fig.title.text = 'Pixel {}'.format(val)
module = self.modules[val]
self.p_plotter_tm.active_pixel = module
self.p_plotter_tm.fig.title.text = 'TM {}'.format(module)
def set_camera_image(self):
if self.w_view_radio.active == 0:
self.p_camera_pix.image = self.m_pix
self.p_camera_tm.image = self.m_tm2048
self.p_camera_pix.fig.title.text = 'Gain Matching Pixels (gradient)'
self.p_camera_tm.fig.title.text = 'Gain Matching TMs (gradient)'
elif self.w_view_radio.active == 1:
self.p_camera_pix.image = self.c_pix
self.p_camera_tm.image = self.c_tm2048
self.p_camera_pix.fig.title.text = 'Gain Matching Pixels (intercept)'
self.p_camera_tm.fig.title.text = 'Gain Matching TMs (intercept)'
def create_view_radio_widget(self):
self.w_view_radio = RadioButtonGroup(labels=["gradient", "intercept"],
active=0)
self.w_view_radio.on_click(self.on_view_radio_widget_click)
def on_view_radio_widget_click(self, active):
self.set_camera_image()
class ADC2PEResidualsPlotter(Tool):
name = "ADC2PEResidualsPlotter"
description = "Plot the residuals from the adc2pe calibration"
input_path = Unicode("", help="Path to the adc2pe_residuals numpy "
"file").tag(config=True)
aliases = Dict(dict(i='ADC2PEResidualsPlotter.input_path', ))
classes = List([])
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.dead = None
self.output_dir = None
self.spe = None
self.spe_sigma = None
self.hist = None
self.edges = None
self.between = None
self.fig_spectrum_all = None
self.fig_spectrum_tm_list = None
self.fig_combgaus = None
self.fig_kde = None
self.fig_hist = None
def setup(self):
self.log_format = "%(levelname)s: %(message)s [%(name)s.%(funcName)s]"
self.dead = Dead()
file = np.load(self.input_path)
self.spe = file['spe']
self.spe_sigma = file['spe_sigma']
self.hist = file['hist']
self.edges = file['edges']
self.between = file['between']
self.output_dir = join(dirname(self.input_path),
"plot_adc2pe_residuals")
if not exists(self.output_dir):
self.log.info("Creating directory: {}".format(self.output_dir))
makedirs(self.output_dir)
# Create figures
sns.set_style("whitegrid")
sns.despine()
self.fig_spectrum_all = plt.figure(figsize=(13, 6))
self.fig_spectrum_all.suptitle("SPE Spectrum, All Pixels")
self.fig_spectrum_tm_list = []
for i in range(32):
fig = plt.figure(figsize=(13, 6))
self.fig_spectrum_tm_list.append(plt.figure(figsize=(13, 6)))
self.fig_combgaus = plt.figure(figsize=(13, 6))
self.fig_combgaus.suptitle("Combined 1pe fit, All Pixels")
self.fig_kde = plt.figure(figsize=(13, 6))
self.fig_kde.suptitle("Distribution of SPE, Kernel density estimate")
self.fig_hist = plt.figure(figsize=(13, 6))
self.fig_hist.suptitle("Distribution of SPE, Histogram")
def start(self):
# Normalise histogram
norm = np.sum(np.diff(self.edges, axis=1) * self.hist, axis=1)
hist = self.hist / norm[:, None]
# Roll axis for easier plotting
hist_r = np.rollaxis(hist, 1)
nbins, npix = hist_r.shape
e = self.edges[0]
hist_tops = np.insert(hist_r, np.arange(nbins), hist_r, axis=0)
edges_tops = np.insert(e, np.arange(e.shape[0]), e, axis=0)[1:-1]
# Mask dead pixels
spe = self.dead.mask1d(self.spe)
spe_sigma = self.dead.mask1d(self.spe_sigma)
hist_tops = self.dead.mask2d(hist_tops)
# Spectrum with all pixels
self.log.info("Plotting: spectrum_all")
ax_spectrum_all = self.fig_spectrum_all.add_subplot(1, 1, 1)
ax_spectrum_all.semilogy(edges_tops, hist_tops, color='b', alpha=0.2)
ax_spectrum_all.set_xlabel("Amplitude (p.e.)")
ax_spectrum_all.set_ylabel("Probability")
# Sprectrum for each tm
self.log.info("Plotting: spectrum_tm")
hist_tops_tm = np.reshape(hist_tops, (hist_tops.shape[0], 32, 64))
for tm, fig in enumerate(self.fig_spectrum_tm_list):
ax = fig.add_subplot(1, 1, 1)
ax.set_title("SPE Spectrum, TM {}".format(tm))
ax.semilogy(edges_tops, hist_tops_tm[:, tm], color='b', alpha=0.2)
ax.set_xlabel("Amplitude (p.e.)")
ax.set_ylabel("Probability")
# Combined gaussian of each spe value
self.log.info("Plotting: combined_gaussian")
ax_comgaus = self.fig_combgaus.add_subplot(1, 1, 1)
x = np.linspace(-1, 4, 200)
kernels = []
for val, sigma in zip(spe.compressed(), spe_sigma.compressed()):
kernel = stats.norm(val, sigma).pdf(x)
kernels.append(kernel)
# plt.plot(x, kernel, color="r")
sns.rugplot(spe.compressed(), color=".2", linewidth=1, ax=ax_comgaus)
density = np.sum(kernels, axis=0)
density /= integrate.trapz(density, x)
ax_comgaus.plot(x, density)
ax_comgaus.set_xlabel("SPE Fit Value (p.e.)")
ax_comgaus.set_ylabel("Sum")
# Kernel density estimate
self.log.info("Plotting: spe_kde")
ax_kde = self.fig_kde.add_subplot(1, 1, 1)
sns.rugplot(spe.compressed(), color=".2", linewidth=1, ax=ax_kde)
sns.kdeplot(spe.compressed(), shade=True, ax=ax_kde)
ax_kde.set_xlabel("SPE Fit Value (p.e.)")
ax_kde.set_ylabel("KDE")
# Histogram
self.log.info("Plotting: histogram")
ax_hist = self.fig_hist.add_subplot(1, 1, 1)
sns.distplot(spe.compressed(), kde=False, rug=True, ax=ax_hist)
ax_hist.set_xlabel("SPE Fit Value (p.e.)")
ax_hist.set_ylabel("N")
def finish(self):
output_path = join(self.output_dir, "spectrum_all.png")
self.fig_spectrum_all.savefig(output_path)
self.log.info("Created figure: {}".format(output_path))
output_path = join(self.output_dir, "spectrum_tm{}.png")
for tm, fig in enumerate(self.fig_spectrum_tm_list):
p = output_path.format(tm)
fig.savefig(p)
self.log.info("Created figure: {}".format(p))
output_path = join(self.output_dir, "combined_gaussian.png")
self.fig_combgaus.savefig(output_path)
self.log.info("Created figure: {}".format(output_path))
output_path = join(self.output_dir, "kde.png")
self.fig_kde.savefig(output_path)
self.log.info("Created figure: {}".format(output_path))
output_path = join(self.output_dir, "hist.png")
self.fig_hist.savefig(output_path)
self.log.info("Created figure: {}".format(output_path))
class ADC2PEPlots(Tool):
name = "ADC2PEPlots"
description = "Create plots related to adc2pe"
aliases = Dict(dict(max_events='TargetioFileReader.max_events'))
classes = List([
TargetioFileReader,
TargetioR1Calibrator,
])
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.reader_dict = dict()
self.dl0 = None
self.dl1 = None
self.dead = None
self.dummy_event = None
self.fw_calibrator = None
self.n_pixels = None
self.n_samples = None
self.df_file = None
self.poi = [1825, 1203]
self.p_scatter_led = None
self.p_scatter_led_width = None
def setup(self):
self.log_format = "%(levelname)s: %(message)s [%(name)s.%(funcName)s]"
kwargs = dict(config=self.config, tool=self)
self.fw_calibrator = FWCalibrator(**kwargs)
dfl = []
base_path = "/Volumes/gct-jason/data/170322/led/Run{:05}_r1_adc.tio"
base_path_pe = "/Volumes/gct-jason/data/170322/led/Run{:05}_r1_pe.tio"
dfl.append(
dict(path=base_path.format(4333), type="LED", cal=False, level=0))
dfl.append(
dict(path=base_path.format(4334), type="LED", cal=False, level=1))
dfl.append(
dict(path=base_path.format(4335), type="LED", cal=False, level=2))
dfl.append(
dict(path=base_path.format(4336), type="LED", cal=False, level=3))
dfl.append(
dict(path=base_path.format(4337), type="LED", cal=False, level=4))
dfl.append(
dict(path=base_path.format(4338), type="LED", cal=False, level=5))
dfl.append(
dict(path=base_path.format(4339), type="LED", cal=False, level=6))
dfl.append(
dict(path=base_path.format(4340), type="LED", cal=False, level=7))
dfl.append(
dict(path=base_path.format(4341), type="LED", cal=False, level=8))
dfl.append(
dict(path=base_path.format(4342), type="LED", cal=False, level=9))
dfl.append(
dict(path=base_path.format(4343), type="LED", cal=False, level=10))
dfl.append(
dict(path=base_path.format(4344), type="LED", cal=False, level=11))
dfl.append(
dict(path=base_path.format(4345), type="LED", cal=False, level=12))
dfl.append(
dict(path=base_path.format(4346), type="LED", cal=False, level=13))
dfl.append(
dict(path=base_path.format(4347), type="LED", cal=False, level=14))
dfl.append(
dict(path=base_path.format(4348), type="LED", cal=False, level=15))
dfl.append(
dict(path=base_path.format(4349), type="LED", cal=False, level=16))
dfl.append(
dict(path=base_path.format(4350), type="LED", cal=False, level=17))
dfl.append(
dict(path=base_path.format(4351), type="LED", cal=False, level=18))
dfl.append(
dict(path=base_path.format(4352), type="LED", cal=False, level=19))
dfl.append(
dict(path=base_path.format(4353), type="LED", cal=False, level=20))
dfl.append(
dict(path=base_path.format(4354), type="LED", cal=False, level=21))
dfl.append(
dict(path=base_path.format(4355), type="LED", cal=False, level=22))
dfl.append(
dict(path=base_path.format(4356), type="LED", cal=False, level=23))
dfl.append(
dict(path=base_path.format(4357), type="LED", cal=False, level=24))
dfl.append(
dict(path=base_path.format(4358), type="LED", cal=False, level=25))
dfl.append(
dict(path=base_path.format(4359), type="LED", cal=False, level=26))
dfl.append(
dict(path=base_path.format(4360), type="LED", cal=False, level=27))
dfl.append(
dict(path=base_path.format(4361), type="LED", cal=False, level=28))
dfl.append(
dict(path=base_path.format(4362), type="LED", cal=False, level=29))
dfl.append(
dict(path=base_path.format(4363), type="LED", cal=False, level=30))
dfl.append(
dict(path=base_path.format(4364), type="LED", cal=False, level=31))
dfl.append(
dict(path=base_path.format(4365), type="LED", cal=False, level=32))
dfl.append(
dict(path=base_path.format(4366), type="LED", cal=False, level=33))
dfl.append(
dict(path=base_path.format(4367), type="LED", cal=False, level=34))
dfl.append(
dict(path=base_path.format(4368), type="LED", cal=False, level=35))
dfl.append(
dict(path=base_path.format(4369), type="LED", cal=False, level=36))
dfl.append(
dict(path=base_path.format(4370), type="LED", cal=False, level=37))
dfl.append(
dict(path=base_path.format(4371), type="LED", cal=False, level=38))
dfl.append(
dict(path=base_path.format(4372), type="LED", cal=False, level=39))
dfl.append(
dict(path=base_path_pe.format(4333), type="LED", cal=True,
level=0))
dfl.append(
dict(path=base_path_pe.format(4334), type="LED", cal=True,
level=1))
dfl.append(
dict(path=base_path_pe.format(4335), type="LED", cal=True,
level=2))
dfl.append(
dict(path=base_path_pe.format(4336), type="LED", cal=True,
level=3))
dfl.append(
dict(path=base_path_pe.format(4337), type="LED", cal=True,
level=4))
dfl.append(
dict(path=base_path_pe.format(4338), type="LED", cal=True,
level=5))
dfl.append(
dict(path=base_path_pe.format(4339), type="LED", cal=True,
level=6))
dfl.append(
dict(path=base_path_pe.format(4340), type="LED", cal=True,
level=7))
dfl.append(
dict(path=base_path_pe.format(4341), type="LED", cal=True,
level=8))
dfl.append(
dict(path=base_path_pe.format(4342), type="LED", cal=True,
level=9))
dfl.append(
dict(path=base_path_pe.format(4343),
type="LED",
cal=True,
level=10))
dfl.append(
dict(path=base_path_pe.format(4344),
type="LED",
cal=True,
level=11))
dfl.append(
dict(path=base_path_pe.format(4345),
type="LED",
cal=True,
level=12))
dfl.append(
dict(path=base_path_pe.format(4346),
type="LED",
cal=True,
level=13))
dfl.append(
dict(path=base_path_pe.format(4347),
type="LED",
cal=True,
level=14))
dfl.append(
dict(path=base_path_pe.format(4348),
type="LED",
cal=True,
level=15))
dfl.append(
dict(path=base_path_pe.format(4349),
type="LED",
cal=True,
level=16))
dfl.append(
dict(path=base_path_pe.format(4350),
type="LED",
cal=True,
level=17))
dfl.append(
dict(path=base_path_pe.format(4351),
type="LED",
cal=True,
level=18))
dfl.append(
dict(path=base_path_pe.format(4352),
type="LED",
cal=True,
level=19))
dfl.append(
dict(path=base_path_pe.format(4353),
type="LED",
cal=True,
level=20))
dfl.append(
dict(path=base_path_pe.format(4354),
type="LED",
cal=True,
level=21))
dfl.append(
dict(path=base_path_pe.format(4355),
type="LED",
cal=True,
level=22))
dfl.append(
dict(path=base_path_pe.format(4356),
type="LED",
cal=True,
level=23))
dfl.append(
dict(path=base_path_pe.format(4357),
type="LED",
cal=True,
level=24))
dfl.append(
dict(path=base_path_pe.format(4358),
type="LED",
cal=True,
level=25))
dfl.append(
dict(path=base_path_pe.format(4359),
type="LED",
cal=True,
level=26))
dfl.append(
dict(path=base_path_pe.format(4360),
type="LED",
cal=True,
level=27))
dfl.append(
dict(path=base_path_pe.format(4361),
type="LED",
cal=True,
level=28))
dfl.append(
dict(path=base_path_pe.format(4362),
type="LED",
cal=True,
level=29))
dfl.append(
dict(path=base_path_pe.format(4363),
type="LED",
cal=True,
level=30))
dfl.append(
dict(path=base_path_pe.format(4364),
type="LED",
cal=True,
level=31))
dfl.append(
dict(path=base_path_pe.format(4365),
type="LED",
cal=True,
level=32))
dfl.append(
dict(path=base_path_pe.format(4366),
type="LED",
cal=True,
level=33))
dfl.append(
dict(path=base_path_pe.format(4367),
type="LED",
cal=True,
level=34))
dfl.append(
dict(path=base_path_pe.format(4368),
type="LED",
cal=True,
level=35))
dfl.append(
dict(path=base_path_pe.format(4369),
type="LED",
cal=True,
level=36))
dfl.append(
dict(path=base_path_pe.format(4370),
type="LED",
cal=True,
level=37))
dfl.append(
dict(path=base_path_pe.format(4371),
type="LED",
cal=True,
level=38))
dfl.append(
dict(path=base_path_pe.format(4372),
type="LED",
cal=True,
level=39))
for d in dfl:
d['reader'] = TargetioFileReader(input_path=d['path'], **kwargs)
self.df_file = pd.DataFrame(dfl)
cleaner = CHECMWaveformCleanerAverage(**kwargs)
extractor = AverageWfPeakIntegrator(**kwargs)
self.dl0 = CameraDL0Reducer(**kwargs)
self.dl1 = CameraDL1Calibrator(extractor=extractor,
cleaner=cleaner,
**kwargs)
self.dead = Dead()
self.dummy_event = dfl[0]['reader'].get_event(0)
telid = list(self.dummy_event.r0.tels_with_data)[0]
r1 = self.dummy_event.r1.tel[telid].pe_samples[0]
self.n_pixels, self.n_samples = r1.shape
script = "checm_paper_led"
self.p_scatter_led = Scatter(**kwargs,
script=script,
figure_name="scatter_led",
shape='wide')
self.p_scatter_led_width = Scatter(**kwargs,
script=script,
figure_name="scatter_led_width",
shape='wide')
def start(self):
df_list = []
dead = self.dead.get_pixel_mask()
kernel = general_gaussian(3, p=1.0, sig=1)
x_base = np.arange(self.n_samples)
x_interp = np.linspace(0, self.n_samples - 1, 300)
ind = np.indices((self.n_pixels, x_interp.size))[1]
r_ind = ind[:, ::-1]
ind_x = x_interp[ind]
r_ind_x = x_interp[r_ind]
saturation_recovery_file = np.load(
"/Volumes/gct-jason/plots/checm_paper/checm_paper_recovery/saturation_recovery.npz"
)
gradient = saturation_recovery_file['gradient']
intercept = saturation_recovery_file['intercept']
desc1 = 'Looping through files'
n_rows = len(self.df_file.index)
for index, row in tqdm(self.df_file.iterrows(),
total=n_rows,
desc=desc1):
path = row['path']
reader = row['reader']
type_ = row['type']
cal = row['cal']
level = row['level']
cal_t = 'Calibrated' if cal else 'Uncalibrated'
source = reader.read()
n_events = reader.num_events
dl1 = np.zeros((n_events, self.n_pixels))
width = np.zeros((n_events, self.n_pixels))
low_max = np.zeros((n_events, self.n_pixels), dtype=np.bool)
desc2 = "Extracting Charge"
for event in tqdm(source, desc=desc2, total=n_events):
ev = event.count
self.dl0.reduce(event)
self.dl1.calibrate(event)
dl1[ev] = event.dl1.tel[0].image[0]
dl0 = event.dl0.tel[0].pe_samples[0]
cleaned = event.dl1.tel[0].cleaned[0]
smooth_flat = np.convolve(dl0.ravel(), kernel, "same")
smoothed = np.reshape(smooth_flat, dl0.shape)
samples_std = np.std(dl0, axis=1)
smooth_baseline_std = np.std(smoothed, axis=1)
with np.errstate(divide='ignore', invalid='ignore'):
smoothed *= (samples_std / smooth_baseline_std)[:, None]
smoothed[~np.isfinite(smoothed)] = 0
dl0 = smoothed
f = interpolate.interp1d(x_base, dl0, kind=3, axis=1)
dl0 = f(x_interp)
grad = np.gradient(dl0)[1]
t_max = x_interp[np.argmax(dl0, 1)]
t_start = t_max - 2
t_end = t_max + 2
t_window = (ind_x >= t_start[..., None]) & (ind_x <
t_end[..., None])
t_windowed = np.ma.array(dl0, mask=~t_window)
t_windowed_ind = np.ma.array(ind_x, mask=~t_window)
max_ = np.max(dl0, axis=1)
reversed_ = dl0[:, ::-1]
peak_time_i = np.ones(dl0.shape) * t_max[:, None]
mask_before = np.ma.masked_less(ind_x, peak_time_i).mask
mask_after = np.ma.masked_greater(r_ind_x, peak_time_i).mask
masked_bef = np.ma.masked_array(dl0, mask_before)
masked_aft = np.ma.masked_array(reversed_, mask_after)
pe_width = 20
d_l = np.diff(np.sign(pe_width - masked_aft))
d_r = np.diff(np.sign(pe_width - masked_bef))
t_l = x_interp[r_ind[0, np.argmax(d_l, axis=1) + 1]]
t_r = x_interp[ind[0, np.argmax(d_r, axis=1) + 1]]
width[ev] = t_r - t_l
low_max[ev] = (max_ < pe_width)
width[ev, low_max[ev]] = 0
charge_masked = self.dead.mask2d(dl1).compressed()
charge_camera = np.mean(charge_masked)
q75, q25 = np.percentile(charge_masked, [75, 25])
charge_err_top_camera = q75 - charge_camera
charge_err_bottom_camera = charge_camera - q25
width = np.ma.masked_array(width, mask=low_max)
ch = gradient[None, :] * width + intercept[None, :]
recovered_charge = 10**(ch**2)
desc3 = "Aggregate charge per pixel"
for pix in trange(self.n_pixels, desc=desc3):
pixel_area = dl1[:, pix]
pixel_width = width[:, pix]
pixel_low_max = low_max[:, pix].all()
pixel_rec_ch = recovered_charge[:, pix]
if pix in self.dead.dead_pixels:
continue
charge = np.mean(pixel_area)
q75, q25 = np.percentile(pixel_area, [75, 25])
charge_err_top = q75 - charge
charge_err_bottom = charge - q25
w = np.mean(pixel_width)
w_err = np.std(pixel_width)
rec_charge = np.mean(pixel_rec_ch)
q75, q25 = np.percentile(pixel_rec_ch, [75, 25])
rec_charge_err_top = q75 - rec_charge
rec_charge_err_bottom = rec_charge - q25
df_list.append(
dict(type=type_,
level=level,
cal=cal,
cal_t=cal_t,
pixel=pix,
tm=pix // 64,
charge=charge,
charge_err_top=charge_err_top,
charge_err_bottom=charge_err_bottom,
charge_camera=charge_camera,
charge_err_top_camera=charge_err_top_camera,
charge_err_bottom_camera=charge_err_bottom_camera,
width=w,
width_err=w_err,
low_max=pixel_low_max,
recovered_charge=rec_charge,
rec_charge_err_top=rec_charge_err_top,
rec_charge_err_bottom=rec_charge_err_bottom))
df = pd.DataFrame(df_list)
store = pd.HDFStore('/Volumes/gct-jason/plots/checm_paper/df/led.h5')
store['df'] = df
store = pd.HDFStore('/Volumes/gct-jason/plots/checm_paper/df/led.h5')
df = store['df']
# Scale ADC values to match p.e.
type_list = np.unique(df['type'])
for t in type_list:
df_t = df.loc[df['type'] == t]
level_list = np.unique(df_t['level'])
for l in level_list:
df_l = df_t.loc[df_t['level'] == l]
median_cal = np.median(df_l.loc[df_l['cal'], 'charge'])
median_uncal = np.median(df_l.loc[~df_l['cal'], 'charge'])
ratio = median_cal / median_uncal
b = (df['type'] == t) & (df['level'] == l) & (~df['cal'])
df.loc[b, 'charge'] *= ratio
df_led = df.loc[(df['type'] == 'LED') & (df['cal'])]
# Create figures
self.p_scatter_led.create("LED", "Charge (p.e.)", "LED Distribution")
self.p_scatter_led.set_y_log()
output_np = join(self.p_scatter_led.output_dir, "pix{}_dr_led.npz")
for ip, p in enumerate(self.poi):
df_pix = df_led.loc[df_led['pixel'] == p]
x = df_pix['level']
y = df_pix['charge']
y_err = [df_pix['charge_err_bottom'], df_pix['charge_err_top']]
label = "Pixel {}".format(p)
self.p_scatter_led.add(x, y, None, y_err, label)
self.log.info("Saving numpy array: {}".format(output_np.format(p)))
np.savez(output_np.format(p), x=x, y=y, x_err=None, y_err=y_err)
self.p_scatter_led.add_legend()
self.p_scatter_led_width.create("Width (ns)", "Charge (p.e.)",
"LED Saturation Recovery")
for ip, p in enumerate(self.poi):
df_pix = df_led.loc[df_led['pixel'] == p]
x = df_pix['width']
y = df_pix['charge']
x_err = df_pix['width_err']
y_err = [df_pix['charge_err_bottom'], df_pix['charge_err_top']]
label = "Pixel {}, Pulse Integration".format(p)
self.p_scatter_led_width.add(x, y, x_err, y_err, label)
x = df_pix['width']
y = df_pix['recovered_charge']
x_err = df_pix['width_err']
y_err = [
df_pix['rec_charge_err_bottom'], df_pix['rec_charge_err_top']
]
label = "Pixel {}, Saturation Recovery".format(p)
self.p_scatter_led_width.add(x, y, x_err, y_err, label)
self.p_scatter_led_width.set_y_log()
self.p_scatter_led_width.add_legend()
def finish(self):
# Save figures
self.p_scatter_led.save()
self.p_scatter_led_width.save()
class ChargeVsRunExtractor(Tool):
name = "ChargeVsRunExtractor"
description = "Extract charge (gaussing fit mean of each pixel) " \
"vs some run descriptor."
rundesc_list = List(Int,
None,
allow_none=True,
help='List of the description value for each input '
'file').tag(config=True)
output_path = Unicode(None,
allow_none=True,
help='Path to save the numpy array').tag(config=True)
calc_mean = Bool(False,
help='Extract the mean and stdev directly insted '
'of fitting the file.').tag(config=True)
aliases = Dict(
dict(
f='TargetioFileLooper.single_file',
N='TargetioFileLooper.max_files',
max_events='TargetioFileLooper.max_events',
ped='CameraR1CalibratorFactory.pedestal_path',
tf='CameraR1CalibratorFactory.tf_path',
pe='CameraR1CalibratorFactory.pe_path',
O='ChargeVsRunExtractor.output_path',
))
flags = Dict(
dict(mean=({
'ChargeVsRunExtractor': {
'calc_mean': True
}
}, 'Extract the mean and stdev directly insted '
'of fitting the file.')))
classes = List([
TargetioFileLooper,
CameraR1CalibratorFactory,
])
def __init__(self, **kwargs):
super().__init__(**kwargs)
self._event = None
self._event_index = None
self._event_id = None
self._active_pixel = None
self.w_event_index = None
self.layout = None
self.file_looper = None
self.r1 = None
self.dl0 = None
self.cleaner = None
self.extractor = None
self.dl1 = None
self.n_pixels = None
self.n_samples = None
self.n_modules = 32
self.cleaner = None
self.extractor = None
self.fitter = None
self.dead = None
self.charge = None
self.charge_err = None
def setup(self):
self.log_format = "%(levelname)s: %(message)s [%(name)s.%(funcName)s]"
kwargs = dict(config=self.config, tool=self)
self.file_looper = TargetioFileLooper(**kwargs)
r1_factory = CameraR1CalibratorFactory(origin='targetio', **kwargs)
r1_class = r1_factory.get_class()
self.r1 = r1_class(**kwargs)
self.cleaner = CHECMWaveformCleanerAverage(**kwargs)
self.extractor = AverageWfPeakIntegrator(**kwargs)
self.dl0 = CameraDL0Reducer(**kwargs)
self.fitter = CHECBrightFitter(**kwargs)
self.dead = Dead()
file_reader_list = self.file_looper.file_reader_list
first_event = file_reader_list[0].get_event(0)
telid = list(first_event.r0.tels_with_data)[0]
r0 = first_event.r0.tel[telid].adc_samples[0]
self.n_pixels, self.n_samples = r0.shape
self.rundesc_list = self.rundesc_list[:self.file_looper.num_readers]
assert (len(file_reader_list) == len(self.rundesc_list))
def start(self):
n_rundesc = len(self.rundesc_list)
# Prepare storage array
self.charge = np.ma.zeros((n_rundesc, self.n_pixels))
self.charge.mask = np.zeros(self.charge.shape, dtype=np.bool)
self.charge.fill_value = 0
self.charge_err = np.ma.zeros((n_rundesc, self.n_pixels))
self.charge_err.mask = np.zeros(self.charge_err.shape, dtype=np.bool)
self.charge_err.fill_value = 0
telid = 0
desc1 = "Looping over runs"
iterable = enumerate(self.get_next_file())
for fn, fr in tqdm(iterable, total=n_rundesc, desc=desc1):
source = fr.read()
n_events = fr.num_events
area = np.zeros((n_events, self.n_pixels))
desc2 = "Extracting area from events"
for event in tqdm(source, total=n_events, desc=desc2):
ev = event.count
self.r1.calibrate(event)
self.dl0.reduce(event)
kwargs = dict(config=self.config,
tool=self,
extractor=self.extractor,
cleaner=self.cleaner)
dl1 = CameraDL1Calibrator(**kwargs)
dl1.calibrate(event)
# Perform CHECM Charge Extraction
area[ev] = event.dl1.tel[telid].image
desc2 = "Fitting pixels"
for pix in trange(self.n_pixels, desc=desc2):
pixel_area = area[:, pix]
if pix in self.dead.dead_pixels:
continue
if self.calc_mean:
self.charge[fn, pix] = np.mean(pixel_area)
self.charge_err[fn, pix] = np.std(pixel_area)
else:
if not self.fitter.apply(pixel_area):
self.log.warning(
"FN {} Pixel {} could not be fitted".format(
fn, pix))
self.charge.mask[fn, pix] = True
self.charge_err.mask[fn, pix] = True
continue
self.charge[fn, pix] = self.fitter.coeff['mean']
self.charge_err[fn, pix] = self.fitter.coeff['stddev']
self.charge = np.ma.filled(self.dead.mask2d(self.charge))
self.charge_err = np.ma.filled(self.dead.mask2d(self.charge_err))
def finish(self):
# Save figures
output_dir = dirname(self.output_path)
if not exists(output_dir):
self.log.info("Creating directory: {}".format(output_dir))
makedirs(output_dir)
np.savez(self.output_path,
charge=self.charge,
charge_error=self.charge_err,
rundesc=self.rundesc_list)
self.log.info("Numpy array saved to: {}".format(self.output_path))
def get_next_file(self):
for fr in self.file_looper.file_reader_list:
yield fr