def process(amplitude_path, hv_path, output_dir, spoi):
with HDF5Reader(amplitude_path) as reader:
df_amp = reader.read("data")
mapping = reader.read_mapping()
superpixels = mapping['superpixel'].values.astype(np.int)
df_amp['superpixel'] = superpixels[df_amp['pixel']]
df_amp = (df_amp
.drop(['amplitude', 'pixel'], axis=1)
.set_index('t_cpu')
.groupby([pd.Grouper(freq='60S'), 'superpixel'])
.mean()
.reset_index())
with HDF5Reader(hv_path) as reader:
df_hv = reader.read("data")
# df_hv = (df_hv
# .set_index('t_cpu')
# .groupby([pd.Grouper(freq='387S'), 'superpixel'])
# .mean()
# .reset_index()
# )
df_hv = (df_hv
.sort_values("t_cpu")
)
df = pd.merge_asof(df_amp, df_hv,
on="t_cpu", by='superpixel',
direction='nearest',
suffixes=('_amp', '_trig'))
p_ampvshv_all = AmpVsHV(switch_backend=True)
x = df['amplitude_sp'].values
y = df['hv'].values
p_ampvshv_all.plot(x, y)
p_ampvshv_all.save(os.path.join(output_dir, "all.pdf"))
p_ampvshv_all.ax.set_ylim((50, 70))
p_ampvshv_all.save(os.path.join(output_dir, "all_zoom.pdf"))
# p_ampvshv_all.ax.set_xlim((0, 200))
# p_ampvshv_all.save(os.path.join(output_dir, "all_zoomx.pdf"))
df['tm'] = df['superpixel'] // 16
df = df.loc[df['hv'] > 56]
ymin = df['amplitude_sp'].min()
ymax = df['amplitude_sp'].max()
p_ampvshv_tm = AmpVsHV(switch_backend=True)
with PdfPages(os.path.join(output_dir, "tm.pdf")) as pdf:
for tm, group_tm in tqdm(df.groupby("tm"), total=32):
for sp, group_sp in group_tm.groupby("superpixel"):
color = next(p_ampvshv_tm.ax._get_lines.prop_cycler)['color']
label = "SP={}".format(sp)
x = group_sp['hv'].values
y = group_sp['amplitude_sp'].values
p_ampvshv_tm.plot(x, y, color=color, label=label)
p_ampvshv_tm.ax.set_title("TM {}".format(tm))
p_ampvshv_tm.finish()
p_ampvshv_tm.add_legend(5)
p_ampvshv_tm.ax.set_xlim((65, 70))
p_ampvshv_tm.ax.set_ylim((ymin, ymax))
pdf.savefig(p_ampvshv_tm.fig)
p_ampvshv_tm.ax.clear()
def process(file):
runlist_path = file.runlist_path
fw_path = file.fw_path
ff_path = file.ff_path
output_path = file.charge_resolution_path
df_runs = open_runlist_dl1(runlist_path)
df_runs['transmission'] = 1 / df_runs['fw_atten']
n_runs = df_runs.index.size
mapping = df_runs.iloc[0]['reader'].mapping
n_pixels = df_runs.iloc[0]['reader'].n_pixels
with HDF5Reader(fw_path) as reader:
df = reader.read("data")
fw_m = df['fw_m'].values
fw_merr = df['fw_merr'].values
with HDF5Reader(ff_path) as reader:
df = reader.read("data")
ff_m = df['ff_m'].values
ff_c = df['ff_c'].values
cr = ChargeResolution()
cs = ChargeStatistics()
desc0 = "Looping over files"
it = enumerate(df_runs.iterrows())
for i, (_, row) in tqdm(it, total=n_runs, desc=desc0):
reader = row['reader']
transmission = row['transmission']
n_rows = n_pixels * 1000
pixel, charge = reader.select_columns(['pixel', 'charge'], stop=n_rows)
true = transmission * fw_m[pixel]
measured = (charge - ff_c[pixel]) / ff_m[pixel]
cr.add(pixel, true, measured)
cs.add(pixel, true, measured)
reader.store.close()
df_cr_pixel, df_cr_camera = cr.finish()
df_cs_pixel, df_cs_camera = cs.finish()
def add_error(df):
df['true_err'] = df['true'] / fw_m[df['pixel']] * fw_merr[df['pixel']]
add_error(df_cr_pixel)
with HDF5Writer(output_path) as writer:
writer.write(
charge_resolution_pixel=df_cr_pixel,
charge_resolution_camera=df_cr_camera,
charge_statistics_pixel=df_cs_pixel,
charge_statistics_camera=df_cs_camera,
)
writer.write_mapping(mapping)
writer.write_metadata(n_pixels=n_pixels)
def process_pix(input_path, output_dir):
with HDF5Reader(input_path) as reader:
df = reader.read("data")
metadata = reader.read_metadata()
p_amplitude = TimePlotter("Waveform Maximum (mV)")
p_baseline = TimePlotter("Baseline (average of first 20 samples) (mV)")
for superpixel, group_sp in df.groupby("superpixel"):
color = next(p_amplitude.ax._get_lines.prop_cycler)['color']
sp_plotted = []
for pixel, group_pix in group_sp.groupby("pixel"):
group_pix = group_pix.set_index('t_cpu')
group_pix = group_pix.resample("60S").mean()
x = group_pix.index.values
y_amp = group_pix['amplitude'].values
y_base = group_pix['baseline'].values
label = metadata["sp{}".format(superpixel)] \
if superpixel not in sp_plotted else None
p_amplitude.plot(x, y_amp, color, label)
p_baseline.plot(x, y_base, color, label)
sp_plotted.append(superpixel)
p_amplitude.save(os.path.join(output_dir, "amplitude.pdf"))
p_baseline.save(os.path.join(output_dir, "baseline.pdf"))
def process(file, name):
stats_path = file.stats_path
poi = file.poi
with HDF5Reader(stats_path) as reader:
df = reader.read("data")
df_p = df.loc[df['pixel'] == poi]
p_measured_mean.plot(df_p['true'], df_p['measured']['mean'], name)
p_measured_res.plot(df_p['true'], df_p['measured']['res'], name)
p_measured_std.plot(df_p['true'], df_p['measured']['std'], name)
p_measured_rms.plot(df_p['true'], df_p['measured']['rms'], name)
p_charge_mean.plot(df_p['true'], df_p['charge']['mean'], name)
p_charge_std.plot(df_p['true'], df_p['charge']['std'], name)
p_charge_rms.plot(df_p['true'], df_p['charge']['rms'], name)
y = np.abs(df_p['measured']['mean'] / df_p['true'] - 1)
p_measured_close.plot(df_p['true'], y, name)
y = df_p['measured']['std'] / df_p['measured']['mean']
p_measured_std_norm.plot(df_p['true'], y, name)
y = df_p['charge']['std'] / df_p['charge']['mean']
p_charge_std_norm.plot(df_p['true'], y, name)
def process(input_path, output_dir, spoi):
with HDF5Reader(input_path) as reader:
df = reader.read("data")
mapping = reader.read_mapping()
superpixels = mapping['superpixel'].values.astype(np.int)
df['superpixel'] = superpixels[df['pixel']]
p_amplitude = TimePlotter("Waveform Maximum (mV)", switch_backend=True)
p_amplitudesp = TimePlotter("Superpixel-Waveform Maximum (mV)",
switch_backend=True)
spoi.extend([0, 1])
desc = "Plotting superpixels"
spoi = np.arange(512)
for superpixel in tqdm(spoi, total=len(spoi), desc=desc):
df_sp = df.loc[df['superpixel'] == superpixel]
df_sp = (df_sp.set_index('t_cpu').groupby(
[pd.Grouper(freq='387S'), 'pixel']).mean().reset_index())
color = next(p_amplitude.ax._get_lines.prop_cycler)['color']
label = "SP={}".format(superpixel)
# if df_sp['amplitude_sp'].max() < 10:
# continue
# elif df_sp['amplitude_sp'].max() > 700:
# min_ = df_sp['amplitude_sp'].min()
# max_ = df_sp['amplitude_sp'].max()
# mean = df_sp['amplitude_sp'].mean()
# low = 0.8 * mean
# high = 1.2 * mean
# embed()
# if (min_ > low) & (max_ < high):
# continue
# if df_sp['amplitude_sp'].values[0] < 100:
# continue
first = True
for pixel, group in df_sp.groupby('pixel'):
group = (
group.set_index('t_cpu')
# .resample("387S").mean()
)
if first:
x = group.index.values
y = group['amplitude'].values
ysp = group['amplitude_sp'].values
p_amplitude.plot(x, y, color, label)
p_amplitudesp.plot(x, ysp, color, label)
first = False
else:
x = group.index.values
y = group['amplitude'].values
p_amplitude.plot(x, y, color)
p_amplitude.save(os.path.join(output_dir, "amplitude.pdf"))
p_amplitudesp.ax.axhline(60, color='black')
p_amplitudesp.save(os.path.join(output_dir, "amplitude_sp.pdf"))
def process(input_path, output_path):
with HDF5Reader(input_path) as reader:
df = reader.read("data")
df = setup_cells(df)
df = df.loc[~(df['r0'] == 0)]
cells = np.unique(df['cell'].values)[:128]
for c in cells:
p_cellwf = CellWaveform(switch_backend=True)
p_cellwf.plot(df, c)
p_cellwf.save(output_path.format(c))
def process(input_path, output_dir, coi):
with HDF5Reader(input_path) as reader:
df = reader.read("data")
df = setup_cells(df)
df_cell = df.loc[df['cell'] == coi]
# p_diff = DiffPlotter()
# p_diff.plot(df_cell)
# p_diff.save(os.path.join(output_dir, "diff.pdf"))
p_wf = CellWaveform()
p_wf.plot(df_cell)
p_wf.save(os.path.join(output_dir, "cellwf.pdf"))
def process(file):
dl1_paths = file.dl1_paths
pde = file.pde
mc_calib_path = file.mc_calib_path
output_path = file.intensity_resolution_path
n_runs = len(dl1_paths)
reader_list = [DL1Reader(p) for p in dl1_paths]
mapping = reader_list[0].mapping
n_pixels = reader_list[0].n_pixels
n_rows = n_pixels * 1000
with HDF5Reader(mc_calib_path) as reader:
df = reader.read("data")
mc_m = df['mc_m'].values
cr = ChargeResolution(mc_true=True)
cs = ChargeStatistics()
desc0 = "Looping over files"
for reader in tqdm(reader_list, total=n_runs, desc=desc0):
pixel, charge, true = reader.select_columns(
['pixel', 'charge', 'mc_true'], stop=n_rows)
true_photons = true / pde
measured = charge / mc_m[pixel]
f = true > 0
true_photons = true_photons[f]
measured = measured[f]
cr.add(pixel, true_photons, measured)
cs.add(pixel, true_photons, measured)
reader.store.close()
df_cr_pixel, df_cr_camera = cr.finish()
df_cs_pixel, df_cs_camera = cs.finish()
with HDF5Writer(output_path) as writer:
writer.write(
charge_resolution_pixel=df_cr_pixel,
charge_resolution_camera=df_cr_camera,
charge_statistics_pixel=df_cs_pixel,
charge_statistics_camera=df_cs_camera,
)
writer.write_mapping(mapping)
writer.write_metadata(n_pixels=n_pixels)
def process(data_path, plot_path):
with HDF5Reader(data_path) as reader:
df = reader.read("data")
n_files = reader.read_metadata()['n_files']
n_samples = df['isam'].max() + 1
nan = df.groupby('ifile').count()['wf'] < n_samples
nan_where = np.where(~nan)[0]
df = df.loc[df['ifile'].isin(nan_where)]
p_wf = Waveform(sidebyside=True)
for ifile in range(n_files):
# df_event = df.loc[df['ifile'] == ifile]
# y = df_event['wf'].values
# file = df_event['file'].values[0]
# print(file, y.std())
p_wf.plot(df, ifile)
# embed()
p_wf.save(plot_path)
def process_sp(input_path, output_dir):
with HDF5Reader(input_path) as reader:
df = reader.read("data_sum")
metadata = reader.read_metadata()
p_amplitude = TimePlotter("Waveform Maximum (summed WF) (mV)")
p_baseline = TimePlotter(
"Baseline (summed WF) (average of first 20 samples) (mV)")
for superpixel, group_sp in df.groupby("superpixel"):
color = next(p_amplitude.ax._get_lines.prop_cycler)['color']
group_sp = group_sp.set_index('t_cpu')
group_sp = group_sp.resample("60S").mean()
x = group_sp.index.values
y_amp = group_sp['amplitude'].values
y_base = group_sp['baseline'].values
label = metadata["sp{}".format(superpixel)]
p_amplitude.plot(x, y_amp, color, label)
p_baseline.plot(x, y_base, color, label)
p_amplitude.save(os.path.join(output_dir, "amplitude_sum.pdf"))
p_baseline.save(os.path.join(output_dir, "baseline_sum.pdf"))
def process(file):
runlist_path = file.runlist_path
fw_path = file.fw_path
ff_path = file.ff_path
output_path = file.stats_path
df_runs = open_runlist_dl1(runlist_path)
df_runs['transmission'] = 1 / df_runs['fw_atten']
n_runs = df_runs.index.size
mapping = df_runs.iloc[0]['reader'].mapping
n_pixels = df_runs.iloc[0]['reader'].n_pixels
with HDF5Reader(fw_path) as reader:
df = reader.read("data")
fw_m = df['fw_m'].values
fw_merr = df['fw_merr'].values
with HDF5Reader(ff_path) as reader:
df = reader.read("data")
ff_m = df['ff_m'].values
ff_c = df['ff_c'].values
df_list = []
desc0 = "Looping over files"
it = enumerate(df_runs.iterrows())
for i, (run, row) in tqdm(it, total=n_runs, desc=desc0):
reader = row['reader']
transmission = row['transmission']
fw_pos = row['fw_pos']
n_rows = n_pixels * 1000
pixel, charge = reader.select_columns(['pixel', 'charge'], stop=n_rows)
true = transmission * fw_m[pixel]
df = pd.DataFrame(
dict(
pixel=pixel,
charge=charge,
measured=(charge - ff_c[pixel]) / ff_m[pixel],
run=run,
transmission=transmission,
fw_pos=fw_pos,
true=true,
))
trans = df.groupby('pixel').transform('mean')
df['charge_mean'] = trans['charge']
df['measured_mean'] = trans['measured']
gb = df.groupby('pixel')
df_stats = gb.agg({
'charge': ['mean', 'std'],
'measured': ['mean', 'std']
})
df_stats['run'] = run
df_stats['transmission'] = transmission
df_stats['fw_pos'] = fw_pos
df_stats['true'] = transmission * fw_m
df_stats['pixel'] = df_stats.index
df_stats.loc[:, ('measured',
'res')] = gb.apply(charge_resolution_df).values
df_stats.loc[:, ('charge', 'rms')] = gb.apply(rms_charge_df).values
df_stats.loc[:, ('measured', 'rms')] = gb.apply(rms_measured_df).values
df_list.append(df_stats)
reader.store.close()
df = pd.concat(df_list, ignore_index=True)
with HDF5Writer(output_path) as writer:
writer.write(data=df)
writer.write_mapping(mapping)
writer.write_metadata(n_pixels=n_pixels)
def process(amplitude_path, trigger_path, output_dir, spoi):
with HDF5Reader(amplitude_path) as reader:
df_amp = reader.read("data")
mapping = reader.read_mapping()
superpixels = mapping['superpixel'].values.astype(np.int)
df_amp['superpixel'] = superpixels[df_amp['pixel']]
df_amp = (df_amp.drop(['amplitude', 'pixel'],
axis=1).set_index('t_cpu').groupby(
[pd.Grouper(freq='60S'),
'superpixel']).mean().reset_index())
with HDF5Reader(trigger_path) as reader:
df_trig = reader.read("data")
df = pd.merge_asof(df_amp,
df_trig,
on="t_cpu",
by='superpixel',
direction='nearest',
suffixes=('_amp', '_trig'))
gb = list(df.groupby("t_cpu"))
diff = gb[-1][1]
diff['amplitude_sp'] -= gb[0][1]['amplitude_sp'].values
diff['count'] -= gb[0][1]['count'].values
p_ampvstrig_all = AmpVsTrigger(switch_backend=True)
superpixels = np.unique(df.loc[df['count'] < 200]['superpixel'])
df_above = df.loc[df['superpixel'].isin(superpixels)]
df_below = df.loc[~df['superpixel'].isin(superpixels)]
x = df_above['amplitude_sp'].values
y = df_above['count'].values
p_ampvstrig_all.plot(x, y)
x = df_below['amplitude_sp'].values
y = df_below['count'].values
p_ampvstrig_all.plot(x, y)
p_ampvstrig_all.save(os.path.join(output_dir, "all.pdf"))
p_ampvstrig_all.ax.set_ylim((0, 500))
p_ampvstrig_all.save(os.path.join(output_dir, "all_zoom.pdf"))
p_ampvstrig_all.ax.set_xlim((0, 200))
p_ampvstrig_all.save(os.path.join(output_dir, "all_zoomx.pdf"))
# p_ampvstrig_spoi = AmpVsTrigger(switch_backend=True)
# for superpixel in tqdm(spoi, total=len(spoi)):
# df_sp = df.loc[df['superpixel'] == superpixel]
# label = "SP={}".format(superpixel)
# x = df_sp['amplitude_sp'].values
# y = df_sp['count'].values
# p_ampvstrig_spoi.plot(x, y, label=label)
# p_ampvstrig_spoi.ax.set_xlim((0, 200))
# p_ampvstrig_spoi.add_legend('best')
# p_ampvstrig_spoi.save(os.path.join(output_dir, "spoi.pdf"))
# p_diff = dAmpVsdTrigger(switch_backend=True)
# x = diff['amplitude_sp'].values
# y = np.abs(diff['count'].values)
# p_diff.plot(x, y)
# p_diff.save(os.path.join(output_dir, "dAmpVsdTrigger.pdf"))
# p_diff.ax.set_ylim((-500, 10))
# p_diff.save(os.path.join(output_dir, "dAmpVsdTrigger_zoom.pdf"))
p_ampvstrig_sp = AmpVsTrigger(switch_backend=True)
desc = "Plotting superpixels"
with PdfPages(os.path.join(output_dir, "superpixels.pdf")) as pdf:
for superpixel in tqdm(spoi, total=len(spoi), desc=desc):
df_sp = df.loc[df['superpixel'] == superpixel]
color = next(p_ampvstrig_sp.ax._get_lines.prop_cycler)['color']
label = "SP={}".format(superpixel)
x = df_sp['amplitude_sp'].values
y = df_sp['count'].values
p_ampvstrig_sp.plot(x, y, color=color)
p_ampvstrig_sp.ax.set_title("SP {}".format(superpixel))
p_ampvstrig_sp.finish()
pdf.savefig(p_ampvstrig_sp.fig)
p_ampvstrig_sp.ax.clear()
def process(file):
runlist_path = file.spe_runlist_path
spe_path = file.spe_path
profile_path = file.illumination_profile_path
dead = file.dead
fw_path = file.fw_path
plot_dir = file.fw_plot_dir
pde = file.pde
df_runs = open_runlist_dl1(runlist_path, False)
df_runs['transmission'] = 1/df_runs['fw_atten']
store_spe = pd.HDFStore(spe_path)
df_spe = store_spe['coeff_pixel']
df_spe_err = store_spe['errors_pixel']
mapping = store_spe['mapping']
with warnings.catch_warnings():
warnings.simplefilter('ignore', UserWarning)
mapping.metadata = store_spe.get_storer('mapping').attrs.metadata
meta_spe = store_spe.get_storer('metadata').attrs.metadata
n_spe_illuminations = meta_spe['n_illuminations']
spe_files = meta_spe['files']
n_pixels = meta_spe['n_pixels']
mean_opct = df_spe['opct'].mean()
if pde is None:
pe2photons = PE2Photons().convert(mean_opct)
else:
pe2photons = 1/pde
print("PDE = {:.3f}".format(1/pe2photons))
print("OPCT = {:.3f}".format(mean_opct))
spe_transmission = []
pattern = '(.+?)/Run(.+?)_dl1.h5'
for path in spe_files:
try:
reg_exp = re.search(pattern, path)
if reg_exp:
run = int(reg_exp.group(2))
spe_transmission.append(df_runs.loc[run]['transmission'])
except AttributeError:
print("Problem with Regular Expression, "
"{} does not match patten {}".format(path, pattern))
pix_lambda = np.zeros((n_spe_illuminations, n_pixels))
pix_lambda_err = np.zeros((n_spe_illuminations, n_pixels))
for ill in range(n_spe_illuminations):
key = "lambda_" + str(ill)
lambda_ = df_spe[['pixel', key]].sort_values('pixel')[key].values * pe2photons
lambda_err = df_spe_err[['pixel', key]].sort_values('pixel')[key].values
pix_lambda[ill] = lambda_
pix_lambda_err[ill] = lambda_err
if profile_path:
with HDF5Reader(profile_path) as reader:
correction = reader.read("correction")['correction']
else:
correction = np.ones(n_pixels)
df_list = []
for i in range(n_spe_illuminations):
df_list.append(pd.DataFrame(dict(
pixel=np.arange(n_pixels),
correction=correction,
transmission=spe_transmission[i],
lambda_=pix_lambda[i],
lambda_err=pix_lambda_err[i],
)))
df = pd.concat(df_list)
# Obtain calibration
dead_mask = np.zeros(n_pixels, dtype=np.bool)
dead_mask[dead] = True
transmission = np.unique(df['transmission'].values)
lambda_ = []
lambda_err = []
corrections = []
for i in range(len(transmission)):
df_t = df.loc[df['transmission'] == transmission[i]]
lambda_.append(df_t['lambda_'].values)
lambda_err.append(df_t['lambda_err'].values)
corrections.append(df_t['correction'].values)
correction = corrections[0]
lambda_ = np.array(lambda_)
lambda_err = np.array(lambda_err)
c_list = []
m_list = []
merr_list = []
for pix in range(n_pixels):
x = transmission
y = lambda_[:, pix]
yerr = lambda_err[:, pix]
w = 1/yerr
cp, mp = polyfit(x, y, 1, w=w)
c_list.append(cp)
m_list.append(mp)
w2 = w**2
merrp = np.sqrt(np.sum(w2)/(np.sum(w2)*np.sum(w2*x**2) - (np.sum(w2*x))**2))
merr_list.append(merrp)
c = np.array(c_list)
m = np.array(m_list)
merr = np.array(merr_list)
# Exlude low gradients (dead pixels)
# dead_mask[m < 1000] = True
merr_corrected = merr / correction
merr_corrected_d = merr_corrected[~dead_mask]
m_corrected = m / correction
m_corrected_d = m_corrected[~dead_mask]
w = 1/merr_corrected_d
m_avg = np.average(m_corrected_d, weights=w)
m_pix = m_avg * correction
m_avg_std = np.sqrt(np.average((m_corrected_d - m_avg) ** 2, weights=w))
m_pix_std = m_avg_std * correction
print("{:.3f} ± {:.3f}".format(m_avg, m_avg_std))
df_calib = pd.DataFrame(dict(
pixel=np.arange(n_pixels),
fw_m=m_pix,
fw_merr=m_pix_std,
))
df_calib = df_calib.sort_values('pixel')
with HDF5Writer(fw_path) as writer:
writer.write(data=df_calib)
writer.write_mapping(mapping)
writer.write_metadata(
n_pixels=n_pixels,
fw_m_camera=m_avg,
fw_merr_camera=m_avg_std,
)
p_fit = FitPlotter()
l = np.s_[:5]
p_fit.plot(transmission, lambda_[:, l], lambda_err[:, l], c[l], m[l])
p_fit.save(os.path.join(plot_dir, "fw_calibration_fit.pdf"))
p_line = LinePlotter()
p_line.plot(m_avg, m_pix, m_avg_std)
p_line.save(os.path.join(plot_dir, "fw_calibration.pdf"))
p_hist = HistPlotter()
p_hist.plot(m_corrected[~dead_mask])
p_hist.save(os.path.join(plot_dir, "relative_pde.pdf"))
def process(file):
charge_averages_path = file.charge_averages_path
fw_path = file.fw_path
ff_path = file.ff_path
plot_dir = file.ff_plot_dir
poi = file.poi
with HDF5Reader(charge_averages_path) as reader:
df_avg = reader.read("data")
mapping = reader.read_mapping()
metadata = reader.read_metadata()
with HDF5Reader(fw_path) as reader:
df_fw = reader.read("data")
fw_m = df_fw['fw_m'].values
fw_merr = df_fw['fw_merr'].values
pixel = df_avg['pixel'].values
transmission = df_avg['transmission'].values
df_avg['illumination'] = transmission * fw_m[pixel]
df_avg['illumination_err'] = transmission * fw_merr[pixel]
d_list = []
for pix in np.unique(df_avg['pixel']):
df_p = df_avg.loc[df_avg['pixel'] == pix]
true = df_p['illumination'].values
true_err = df_p['illumination_err'].values
measured = df_p['mean'].values
measured_std = df_p['std'].values
flag = np.zeros(true.size, dtype=np.bool)
flag[np.abs(true - 50).argsort()[:3]] = True
x = true[flag]
y = measured[flag]
y_err = measured_std[flag]
p, f = polyfit(x, y, [1], w=1 / y_err, full=True)
ff_c, ff_m = p
# n = x.size
# sy = np.sqrt(np.sum((y - polyval(x, p))**2) / (n - 1))
# sm = sy * np.sqrt(1/(np.sum((x - np.mean(x))**2)))
# ff_merr = sm
ff_merr = 0
d_list.append(dict(
pixel=pix,
ff_c=ff_c,
ff_m=ff_m,
ff_merr=ff_merr,
))
if pix == poi:
print("{:.3f} ± {:.3f}".format(ff_m, ff_merr))
p_fit = FitPlotter()
p_fit.plot(true, measured, true_err, measured_std, flag, ff_c,
ff_m, ff_merr)
p_fit.save(os.path.join(plot_dir, "flat_fielding.pdf"))
df_calib = pd.DataFrame(d_list)
df_calib = df_calib.sort_values('pixel')
with HDF5Writer(ff_path) as writer:
writer.write(data=df_calib)
writer.write_mapping(mapping)
writer.write_metadata(**metadata)
p_hist2d = Hist2D()
p_hist2d.plot(df_avg['illumination'].values, df_avg['mean'].values)
p_hist2d.save(os.path.join(plot_dir, "pixel_averages.pdf"))
def process(file):
runlist_path = file.runlist_path
output_path = file.saturation_recovery_path
fw_path = file.fw_path
plot_path = file.saturation_recovery_plot_path
poi = file.poi
df_runs = open_runlist_dl1(runlist_path)
df_runs['transmission'] = 1 / df_runs['fw_atten']
n_runs = df_runs.index.size
mapping = df_runs.iloc[0]['reader'].mapping
n_pixels = df_runs.iloc[0]['reader'].n_pixels
cs = ChargeStatistics()
desc0 = "Looping over files"
it = enumerate(df_runs.iterrows())
for i, (_, row) in tqdm(it, total=n_runs, desc=desc0):
reader = row['reader']
transmission = row['transmission']
n_rows = n_pixels * 1000
pixel, charge = reader.select_columns(['pixel', 'saturation_coeff'],
stop=n_rows)
cs.add(pixel, transmission, charge)
reader.store.close()
df_pixel, df_camera = cs.finish()
df = df_pixel[["pixel", "amplitude", "mean", "std"]].copy()
df = df.rename(columns={"amplitude": "transmission"})
df_runs2 = df_runs[['transmission', 'pe_expected', 'fw_pos']].copy()
df_runs2['run_number'] = df_runs2.index
df = pd.merge(df, df_runs2, on='transmission')
with HDF5Reader(fw_path) as reader:
df_fw = reader.read("data")
fw_m = df_fw['fw_m'].values
fw_merr = df_fw['fw_merr'].values
pixel = df['pixel'].values
transmission = df['transmission'].values
df['illumination'] = transmission * fw_m[pixel]
df['illumination_err'] = transmission * fw_merr[pixel]
d_list = []
for pix in np.unique(df['pixel']):
df_p = df.loc[df['pixel'] == pix]
true = df_p['illumination'].values
true_err = df_p['illumination_err'].values
measured = df_p['mean'].values
measured_std = df_p['std'].values
flag = np.zeros(true.size, dtype=np.bool)
flag[np.abs(true - 2500).argsort()[:5]] = True
x = true[flag]
y = measured[flag]
y_err = measured_std[flag]
p = polyfit(x, y, [1], w=1 / y_err)
ff_c, ff_m = p
d_list.append(dict(
pixel=pix,
ff_c=ff_c,
ff_m=ff_m,
))
if pix == poi:
print("{:.3f}".format(ff_m))
p_fit = FitPlotter()
p_fit.plot(true, measured, true_err, measured_std, flag, p)
p_fit.save(plot_path)
df_calib = pd.DataFrame(d_list)
df_calib = df_calib.sort_values('pixel')
with HDF5Writer(output_path) as writer:
writer.write(data=df_calib)
writer.write_mapping(mapping)
writer.write_metadata(n_pixels=n_pixels)