def process(file):
runlist_path = file.runlist_path
output_path = file.charge_averages_path
df_runs = open_runlist_dl1(runlist_path)
df_runs['transmission'] = 1 / df_runs['fw_atten']
n_runs = df_runs.index.size
mapping = df_runs.iloc[0]['reader'].mapping
n_pixels = df_runs.iloc[0]['reader'].n_pixels
cs = ChargeStatistics()
desc0 = "Looping over files"
it = enumerate(df_runs.iterrows())
for i, (_, row) in tqdm(it, total=n_runs, desc=desc0):
reader = row['reader']
transmission = row['transmission']
n_rows = n_pixels * 1000
pixel, charge = reader.select_columns(['pixel', 'charge'], stop=n_rows)
cs.add(pixel, transmission, charge)
reader.store.close()
df_pixel, df_camera = cs.finish()
df = df_pixel[["pixel", "amplitude", "mean", "std"]].copy()
df = df.rename(columns={"amplitude": "transmission"})
df_runs2 = df_runs[['transmission', 'pe_expected', 'fw_pos']].copy()
df_runs2['run_number'] = df_runs2.index
df = pd.merge(df, df_runs2, on='transmission')
with HDF5Writer(output_path) as writer:
writer.write(data=df)
writer.write_mapping(mapping)
writer.write_metadata(n_pixels=n_pixels)
def process(file):
runlist_path = file.runlist_path
fw_path = file.fw_path
ff_path = file.ff_path
output_path = file.charge_resolution_path
df_runs = open_runlist_dl1(runlist_path)
df_runs['transmission'] = 1 / df_runs['fw_atten']
n_runs = df_runs.index.size
mapping = df_runs.iloc[0]['reader'].mapping
n_pixels = df_runs.iloc[0]['reader'].n_pixels
with HDF5Reader(fw_path) as reader:
df = reader.read("data")
fw_m = df['fw_m'].values
fw_merr = df['fw_merr'].values
with HDF5Reader(ff_path) as reader:
df = reader.read("data")
ff_m = df['ff_m'].values
ff_c = df['ff_c'].values
cr = ChargeResolution()
cs = ChargeStatistics()
desc0 = "Looping over files"
it = enumerate(df_runs.iterrows())
for i, (_, row) in tqdm(it, total=n_runs, desc=desc0):
reader = row['reader']
transmission = row['transmission']
n_rows = n_pixels * 1000
pixel, charge = reader.select_columns(['pixel', 'charge'], stop=n_rows)
true = transmission * fw_m[pixel]
measured = (charge - ff_c[pixel]) / ff_m[pixel]
cr.add(pixel, true, measured)
cs.add(pixel, true, measured)
reader.store.close()
df_cr_pixel, df_cr_camera = cr.finish()
df_cs_pixel, df_cs_camera = cs.finish()
def add_error(df):
df['true_err'] = df['true'] / fw_m[df['pixel']] * fw_merr[df['pixel']]
add_error(df_cr_pixel)
with HDF5Writer(output_path) as writer:
writer.write(
charge_resolution_pixel=df_cr_pixel,
charge_resolution_camera=df_cr_camera,
charge_statistics_pixel=df_cs_pixel,
charge_statistics_camera=df_cs_camera,
)
writer.write_mapping(mapping)
writer.write_metadata(n_pixels=n_pixels)
def main():
description = 'Create a new HDFStore file containing the time corrections'
parser = argparse.ArgumentParser(description=description,
formatter_class=Formatter)
parser.add_argument('-f', '--file', dest='input_path',
action='store', required=True,
help='path to the runlist txt file')
parser.add_argument('-o', '--output', dest='output_path',
action='store', required=True,
help='path to store the output HDFStore file')
args = parser.parse_args()
input_path = args.input_path
output_path = args.output_path
df_runs = open_runlist_dl1(input_path)
if os.path.exists(output_path):
os.remove(output_path)
print("Created HDFStore file: {}".format(output_path))
with pd.HDFStore(output_path) as store:
store['mapping'] = df_runs.iloc[0]['reader'].store['mapping']
store.get_storer('mapping').attrs.metadata = df_runs.iloc[0]['reader'].store.get_storer('mapping').attrs.metadata
mean_list = []
desc = "Looping over files"
n_rows = df_runs.index.size
for index, row in tqdm(df_runs.iterrows(), total=n_rows, desc=desc):
reader = row['reader']
amplitude = row['illumination']
attenuation = row['attenuation']
df = reader.load_entire_table()
df_mean = df.groupby('pixel').mean().reset_index()
df_mean['attenuation'] = attenuation
df_mean['amplitude'] = amplitude
mean_list.append(df_mean)
reader.store.close()
store['mean'] = pd.concat(mean_list, ignore_index=True)
print("Filled file: {}".format(output_path))
def process(file):
runlist_path = file.runlist_path
fw_path = file.fw_path
ff_path = file.ff_path
output_path = file.stats_path
df_runs = open_runlist_dl1(runlist_path)
df_runs['transmission'] = 1 / df_runs['fw_atten']
n_runs = df_runs.index.size
mapping = df_runs.iloc[0]['reader'].mapping
n_pixels = df_runs.iloc[0]['reader'].n_pixels
with HDF5Reader(fw_path) as reader:
df = reader.read("data")
fw_m = df['fw_m'].values
fw_merr = df['fw_merr'].values
with HDF5Reader(ff_path) as reader:
df = reader.read("data")
ff_m = df['ff_m'].values
ff_c = df['ff_c'].values
df_list = []
desc0 = "Looping over files"
it = enumerate(df_runs.iterrows())
for i, (run, row) in tqdm(it, total=n_runs, desc=desc0):
reader = row['reader']
transmission = row['transmission']
fw_pos = row['fw_pos']
n_rows = n_pixels * 1000
pixel, charge = reader.select_columns(['pixel', 'charge'], stop=n_rows)
true = transmission * fw_m[pixel]
df = pd.DataFrame(
dict(
pixel=pixel,
charge=charge,
measured=(charge - ff_c[pixel]) / ff_m[pixel],
run=run,
transmission=transmission,
fw_pos=fw_pos,
true=true,
))
trans = df.groupby('pixel').transform('mean')
df['charge_mean'] = trans['charge']
df['measured_mean'] = trans['measured']
gb = df.groupby('pixel')
df_stats = gb.agg({
'charge': ['mean', 'std'],
'measured': ['mean', 'std']
})
df_stats['run'] = run
df_stats['transmission'] = transmission
df_stats['fw_pos'] = fw_pos
df_stats['true'] = transmission * fw_m
df_stats['pixel'] = df_stats.index
df_stats.loc[:, ('measured',
'res')] = gb.apply(charge_resolution_df).values
df_stats.loc[:, ('charge', 'rms')] = gb.apply(rms_charge_df).values
df_stats.loc[:, ('measured', 'rms')] = gb.apply(rms_measured_df).values
df_list.append(df_stats)
reader.store.close()
df = pd.concat(df_list, ignore_index=True)
with HDF5Writer(output_path) as writer:
writer.write(data=df)
writer.write_mapping(mapping)
writer.write_metadata(n_pixels=n_pixels)
def process(file):
runlist_path = file.spe_runlist_path
spe_path = file.spe_path
profile_path = file.illumination_profile_path
dead = file.dead
fw_path = file.fw_path
plot_dir = file.fw_plot_dir
pde = file.pde
df_runs = open_runlist_dl1(runlist_path, False)
df_runs['transmission'] = 1/df_runs['fw_atten']
store_spe = pd.HDFStore(spe_path)
df_spe = store_spe['coeff_pixel']
df_spe_err = store_spe['errors_pixel']
mapping = store_spe['mapping']
with warnings.catch_warnings():
warnings.simplefilter('ignore', UserWarning)
mapping.metadata = store_spe.get_storer('mapping').attrs.metadata
meta_spe = store_spe.get_storer('metadata').attrs.metadata
n_spe_illuminations = meta_spe['n_illuminations']
spe_files = meta_spe['files']
n_pixels = meta_spe['n_pixels']
mean_opct = df_spe['opct'].mean()
if pde is None:
pe2photons = PE2Photons().convert(mean_opct)
else:
pe2photons = 1/pde
print("PDE = {:.3f}".format(1/pe2photons))
print("OPCT = {:.3f}".format(mean_opct))
spe_transmission = []
pattern = '(.+?)/Run(.+?)_dl1.h5'
for path in spe_files:
try:
reg_exp = re.search(pattern, path)
if reg_exp:
run = int(reg_exp.group(2))
spe_transmission.append(df_runs.loc[run]['transmission'])
except AttributeError:
print("Problem with Regular Expression, "
"{} does not match patten {}".format(path, pattern))
pix_lambda = np.zeros((n_spe_illuminations, n_pixels))
pix_lambda_err = np.zeros((n_spe_illuminations, n_pixels))
for ill in range(n_spe_illuminations):
key = "lambda_" + str(ill)
lambda_ = df_spe[['pixel', key]].sort_values('pixel')[key].values * pe2photons
lambda_err = df_spe_err[['pixel', key]].sort_values('pixel')[key].values
pix_lambda[ill] = lambda_
pix_lambda_err[ill] = lambda_err
if profile_path:
with HDF5Reader(profile_path) as reader:
correction = reader.read("correction")['correction']
else:
correction = np.ones(n_pixels)
df_list = []
for i in range(n_spe_illuminations):
df_list.append(pd.DataFrame(dict(
pixel=np.arange(n_pixels),
correction=correction,
transmission=spe_transmission[i],
lambda_=pix_lambda[i],
lambda_err=pix_lambda_err[i],
)))
df = pd.concat(df_list)
# Obtain calibration
dead_mask = np.zeros(n_pixels, dtype=np.bool)
dead_mask[dead] = True
transmission = np.unique(df['transmission'].values)
lambda_ = []
lambda_err = []
corrections = []
for i in range(len(transmission)):
df_t = df.loc[df['transmission'] == transmission[i]]
lambda_.append(df_t['lambda_'].values)
lambda_err.append(df_t['lambda_err'].values)
corrections.append(df_t['correction'].values)
correction = corrections[0]
lambda_ = np.array(lambda_)
lambda_err = np.array(lambda_err)
c_list = []
m_list = []
merr_list = []
for pix in range(n_pixels):
x = transmission
y = lambda_[:, pix]
yerr = lambda_err[:, pix]
w = 1/yerr
cp, mp = polyfit(x, y, 1, w=w)
c_list.append(cp)
m_list.append(mp)
w2 = w**2
merrp = np.sqrt(np.sum(w2)/(np.sum(w2)*np.sum(w2*x**2) - (np.sum(w2*x))**2))
merr_list.append(merrp)
c = np.array(c_list)
m = np.array(m_list)
merr = np.array(merr_list)
# Exlude low gradients (dead pixels)
# dead_mask[m < 1000] = True
merr_corrected = merr / correction
merr_corrected_d = merr_corrected[~dead_mask]
m_corrected = m / correction
m_corrected_d = m_corrected[~dead_mask]
w = 1/merr_corrected_d
m_avg = np.average(m_corrected_d, weights=w)
m_pix = m_avg * correction
m_avg_std = np.sqrt(np.average((m_corrected_d - m_avg) ** 2, weights=w))
m_pix_std = m_avg_std * correction
print("{:.3f} ± {:.3f}".format(m_avg, m_avg_std))
df_calib = pd.DataFrame(dict(
pixel=np.arange(n_pixels),
fw_m=m_pix,
fw_merr=m_pix_std,
))
df_calib = df_calib.sort_values('pixel')
with HDF5Writer(fw_path) as writer:
writer.write(data=df_calib)
writer.write_mapping(mapping)
writer.write_metadata(
n_pixels=n_pixels,
fw_m_camera=m_avg,
fw_merr_camera=m_avg_std,
)
p_fit = FitPlotter()
l = np.s_[:5]
p_fit.plot(transmission, lambda_[:, l], lambda_err[:, l], c[l], m[l])
p_fit.save(os.path.join(plot_dir, "fw_calibration_fit.pdf"))
p_line = LinePlotter()
p_line.plot(m_avg, m_pix, m_avg_std)
p_line.save(os.path.join(plot_dir, "fw_calibration.pdf"))
p_hist = HistPlotter()
p_hist.plot(m_corrected[~dead_mask])
p_hist.save(os.path.join(plot_dir, "relative_pde.pdf"))
def process(file):
runlist_path = file.runlist_path
output_path = file.saturation_recovery_path
fw_path = file.fw_path
plot_path = file.saturation_recovery_plot_path
poi = file.poi
df_runs = open_runlist_dl1(runlist_path)
df_runs['transmission'] = 1 / df_runs['fw_atten']
n_runs = df_runs.index.size
mapping = df_runs.iloc[0]['reader'].mapping
n_pixels = df_runs.iloc[0]['reader'].n_pixels
cs = ChargeStatistics()
desc0 = "Looping over files"
it = enumerate(df_runs.iterrows())
for i, (_, row) in tqdm(it, total=n_runs, desc=desc0):
reader = row['reader']
transmission = row['transmission']
n_rows = n_pixels * 1000
pixel, charge = reader.select_columns(['pixel', 'saturation_coeff'],
stop=n_rows)
cs.add(pixel, transmission, charge)
reader.store.close()
df_pixel, df_camera = cs.finish()
df = df_pixel[["pixel", "amplitude", "mean", "std"]].copy()
df = df.rename(columns={"amplitude": "transmission"})
df_runs2 = df_runs[['transmission', 'pe_expected', 'fw_pos']].copy()
df_runs2['run_number'] = df_runs2.index
df = pd.merge(df, df_runs2, on='transmission')
with HDF5Reader(fw_path) as reader:
df_fw = reader.read("data")
fw_m = df_fw['fw_m'].values
fw_merr = df_fw['fw_merr'].values
pixel = df['pixel'].values
transmission = df['transmission'].values
df['illumination'] = transmission * fw_m[pixel]
df['illumination_err'] = transmission * fw_merr[pixel]
d_list = []
for pix in np.unique(df['pixel']):
df_p = df.loc[df['pixel'] == pix]
true = df_p['illumination'].values
true_err = df_p['illumination_err'].values
measured = df_p['mean'].values
measured_std = df_p['std'].values
flag = np.zeros(true.size, dtype=np.bool)
flag[np.abs(true - 2500).argsort()[:5]] = True
x = true[flag]
y = measured[flag]
y_err = measured_std[flag]
p = polyfit(x, y, [1], w=1 / y_err)
ff_c, ff_m = p
d_list.append(dict(
pixel=pix,
ff_c=ff_c,
ff_m=ff_m,
))
if pix == poi:
print("{:.3f}".format(ff_m))
p_fit = FitPlotter()
p_fit.plot(true, measured, true_err, measured_std, flag, p)
p_fit.save(plot_path)
df_calib = pd.DataFrame(d_list)
df_calib = df_calib.sort_values('pixel')
with HDF5Writer(output_path) as writer:
writer.write(data=df_calib)
writer.write_mapping(mapping)
writer.write_metadata(n_pixels=n_pixels)
def main():
description = 'Extract the charge resolution from a dynamic range dataset'
parser = argparse.ArgumentParser(description=description,
formatter_class=Formatter)
parser.add_argument('-f',
'--file',
dest='input_path',
action='store',
required=True,
help='path to the runlist.txt file for '
'a dynamic range run')
parser.add_argument('-s',
'--spe',
dest='spe_path',
action='store',
required=True,
help='path to the spe file to use for '
'the calibration of the measured '
'and true charges')
parser.add_argument('-o',
'--output',
dest='output_path',
action='store',
help='path to store the output HDF5 file '
'(OPTIONAL, will be automatically set if '
'not specified)')
args = parser.parse_args()
input_path = args.input_path
spe_path = args.spe_path
output_path = args.output_path
df_runs = open_runlist_dl1(input_path)
df_runs['transmission'] = 1 / df_runs['fw_atten']
n_runs = df_runs.index.size
dead = [677, 293, 27, 1925]
spe_handler = SPEHandler(df_runs, spe_path)
cr = ChargeResolution()
cs = ChargeStatistics()
if not output_path:
output_dir = os.path.dirname(input_path)
output_path = os.path.join(output_dir, "charge_res.h5")
output_dir = os.path.dirname(output_path)
if not os.path.exists(output_dir):
os.makedirs(output_dir)
print("Created directory: {}".format(output_dir))
if os.path.exists(output_path):
os.remove(output_path)
with pd.HDFStore(output_path) as store:
desc0 = "Looping over files"
it = enumerate(df_runs.iterrows())
for i, (_, row) in tqdm(it, total=n_runs, desc=desc0):
reader = row['reader']
transmission = row['transmission']
# for df in reader.iterate_over_chunks():
df = reader.get_first_n_events(1000)
df = df.loc[~df['pixel'].isin(dead)]
pixel = df['pixel'].values
true = spe_handler.calibrate_true(pixel, transmission)
measured = df['charge'].values
measured = spe_handler.calibrate_measured(pixel, measured)
cr.add(pixel, true, measured)
cs.add(pixel, true, measured)
reader.store.close()
df_pixel, df_camera = cr.finish()
store['charge_resolution_pixel'] = df_pixel
store['charge_resolution_camera'] = df_camera
df_pixel, df_camera = cs.finish()
store['charge_statistics_pixel'] = df_pixel
store['charge_statistics_camera'] = df_camera