def process(file):
runlist_path = file.runlist_path
output_path = file.charge_averages_path
df_runs = open_runlist_dl1(runlist_path)
df_runs['transmission'] = 1 / df_runs['fw_atten']
n_runs = df_runs.index.size
mapping = df_runs.iloc[0]['reader'].mapping
n_pixels = df_runs.iloc[0]['reader'].n_pixels
cs = ChargeStatistics()
desc0 = "Looping over files"
it = enumerate(df_runs.iterrows())
for i, (_, row) in tqdm(it, total=n_runs, desc=desc0):
reader = row['reader']
transmission = row['transmission']
n_rows = n_pixels * 1000
pixel, charge = reader.select_columns(['pixel', 'charge'], stop=n_rows)
cs.add(pixel, transmission, charge)
reader.store.close()
df_pixel, df_camera = cs.finish()
df = df_pixel[["pixel", "amplitude", "mean", "std"]].copy()
df = df.rename(columns={"amplitude": "transmission"})
df_runs2 = df_runs[['transmission', 'pe_expected', 'fw_pos']].copy()
df_runs2['run_number'] = df_runs2.index
df = pd.merge(df, df_runs2, on='transmission')
with HDF5Writer(output_path) as writer:
writer.write(data=df)
writer.write_mapping(mapping)
writer.write_metadata(n_pixels=n_pixels)
def process(file):
runlist_path = file.runlist_path
fw_path = file.fw_path
ff_path = file.ff_path
output_path = file.charge_resolution_path
df_runs = open_runlist_dl1(runlist_path)
df_runs['transmission'] = 1 / df_runs['fw_atten']
n_runs = df_runs.index.size
mapping = df_runs.iloc[0]['reader'].mapping
n_pixels = df_runs.iloc[0]['reader'].n_pixels
with HDF5Reader(fw_path) as reader:
df = reader.read("data")
fw_m = df['fw_m'].values
fw_merr = df['fw_merr'].values
with HDF5Reader(ff_path) as reader:
df = reader.read("data")
ff_m = df['ff_m'].values
ff_c = df['ff_c'].values
cr = ChargeResolution()
cs = ChargeStatistics()
desc0 = "Looping over files"
it = enumerate(df_runs.iterrows())
for i, (_, row) in tqdm(it, total=n_runs, desc=desc0):
reader = row['reader']
transmission = row['transmission']
n_rows = n_pixels * 1000
pixel, charge = reader.select_columns(['pixel', 'charge'], stop=n_rows)
true = transmission * fw_m[pixel]
measured = (charge - ff_c[pixel]) / ff_m[pixel]
cr.add(pixel, true, measured)
cs.add(pixel, true, measured)
reader.store.close()
df_cr_pixel, df_cr_camera = cr.finish()
df_cs_pixel, df_cs_camera = cs.finish()
def add_error(df):
df['true_err'] = df['true'] / fw_m[df['pixel']] * fw_merr[df['pixel']]
add_error(df_cr_pixel)
with HDF5Writer(output_path) as writer:
writer.write(
charge_resolution_pixel=df_cr_pixel,
charge_resolution_camera=df_cr_camera,
charge_statistics_pixel=df_cs_pixel,
charge_statistics_camera=df_cs_camera,
)
writer.write_mapping(mapping)
writer.write_metadata(n_pixels=n_pixels)
def process(file):
dl1_paths = file.dl1_paths
pde = file.pde
mc_calib_path = file.mc_calib_path
output_path = file.intensity_resolution_path
n_runs = len(dl1_paths)
reader_list = [DL1Reader(p) for p in dl1_paths]
mapping = reader_list[0].mapping
n_pixels = reader_list[0].n_pixels
n_rows = n_pixels * 1000
with HDF5Reader(mc_calib_path) as reader:
df = reader.read("data")
mc_m = df['mc_m'].values
cr = ChargeResolution(mc_true=True)
cs = ChargeStatistics()
desc0 = "Looping over files"
for reader in tqdm(reader_list, total=n_runs, desc=desc0):
pixel, charge, true = reader.select_columns(
['pixel', 'charge', 'mc_true'], stop=n_rows)
true_photons = true / pde
measured = charge / mc_m[pixel]
f = true > 0
true_photons = true_photons[f]
measured = measured[f]
cr.add(pixel, true_photons, measured)
cs.add(pixel, true_photons, measured)
reader.store.close()
df_cr_pixel, df_cr_camera = cr.finish()
df_cs_pixel, df_cs_camera = cs.finish()
with HDF5Writer(output_path) as writer:
writer.write(
charge_resolution_pixel=df_cr_pixel,
charge_resolution_camera=df_cr_camera,
charge_statistics_pixel=df_cs_pixel,
charge_statistics_camera=df_cs_camera,
)
writer.write_mapping(mapping)
writer.write_metadata(n_pixels=n_pixels)
# signal *= norm
# pe_signal = np.nansum(signal, 0)
#
# s = ped_signal + pe_signal
# s /= s.sum()
#
# return s
opct = 0.38
lambda_ = 1
cr = ChargeResolution()
cs = ChargeStatistics()
desc = "Looping over lambdas"
true_arr = np.geomspace(0.1, 400, 100)
for lambda_ in tqdm(true_arr, desc=desc):
pixel = 1920
dist = get_distribution(opct, lambda_)
n = 100000
measured = np.random.choice(x, p=dist, size=n)
true = lambda_
cr.add(pixel, true, measured)
cs.add(pixel, true, measured)
df_pixel, _ = cr.finish()
x = df_pixel['true']
cr = df_pixel['rmse']
df_pixel, _ = cs.finish()
mean = df_pixel['mean']
np.savez("opct_limit.npz", x=x, cr=cr, mean=mean)
def process(file):
runlist_path = file.runlist_path
output_path = file.saturation_recovery_path
fw_path = file.fw_path
plot_path = file.saturation_recovery_plot_path
poi = file.poi
df_runs = open_runlist_dl1(runlist_path)
df_runs['transmission'] = 1 / df_runs['fw_atten']
n_runs = df_runs.index.size
mapping = df_runs.iloc[0]['reader'].mapping
n_pixels = df_runs.iloc[0]['reader'].n_pixels
cs = ChargeStatistics()
desc0 = "Looping over files"
it = enumerate(df_runs.iterrows())
for i, (_, row) in tqdm(it, total=n_runs, desc=desc0):
reader = row['reader']
transmission = row['transmission']
n_rows = n_pixels * 1000
pixel, charge = reader.select_columns(['pixel', 'saturation_coeff'],
stop=n_rows)
cs.add(pixel, transmission, charge)
reader.store.close()
df_pixel, df_camera = cs.finish()
df = df_pixel[["pixel", "amplitude", "mean", "std"]].copy()
df = df.rename(columns={"amplitude": "transmission"})
df_runs2 = df_runs[['transmission', 'pe_expected', 'fw_pos']].copy()
df_runs2['run_number'] = df_runs2.index
df = pd.merge(df, df_runs2, on='transmission')
with HDF5Reader(fw_path) as reader:
df_fw = reader.read("data")
fw_m = df_fw['fw_m'].values
fw_merr = df_fw['fw_merr'].values
pixel = df['pixel'].values
transmission = df['transmission'].values
df['illumination'] = transmission * fw_m[pixel]
df['illumination_err'] = transmission * fw_merr[pixel]
d_list = []
for pix in np.unique(df['pixel']):
df_p = df.loc[df['pixel'] == pix]
true = df_p['illumination'].values
true_err = df_p['illumination_err'].values
measured = df_p['mean'].values
measured_std = df_p['std'].values
flag = np.zeros(true.size, dtype=np.bool)
flag[np.abs(true - 2500).argsort()[:5]] = True
x = true[flag]
y = measured[flag]
y_err = measured_std[flag]
p = polyfit(x, y, [1], w=1 / y_err)
ff_c, ff_m = p
d_list.append(dict(
pixel=pix,
ff_c=ff_c,
ff_m=ff_m,
))
if pix == poi:
print("{:.3f}".format(ff_m))
p_fit = FitPlotter()
p_fit.plot(true, measured, true_err, measured_std, flag, p)
p_fit.save(plot_path)
df_calib = pd.DataFrame(d_list)
df_calib = df_calib.sort_values('pixel')
with HDF5Writer(output_path) as writer:
writer.write(data=df_calib)
writer.write_mapping(mapping)
writer.write_metadata(n_pixels=n_pixels)
def main():
description = 'Extract the charge resolution from a dynamic range dataset'
parser = argparse.ArgumentParser(description=description,
formatter_class=Formatter)
parser.add_argument('-f',
'--file',
dest='input_path',
action='store',
required=True,
help='path to the runlist.txt file for '
'a dynamic range run')
parser.add_argument('-s',
'--spe',
dest='spe_path',
action='store',
required=True,
help='path to the spe file to use for '
'the calibration of the measured '
'and true charges')
parser.add_argument('-o',
'--output',
dest='output_path',
action='store',
help='path to store the output HDF5 file '
'(OPTIONAL, will be automatically set if '
'not specified)')
args = parser.parse_args()
input_path = args.input_path
spe_path = args.spe_path
output_path = args.output_path
df_runs = open_runlist_dl1(input_path)
df_runs['transmission'] = 1 / df_runs['fw_atten']
n_runs = df_runs.index.size
dead = [677, 293, 27, 1925]
spe_handler = SPEHandler(df_runs, spe_path)
cr = ChargeResolution()
cs = ChargeStatistics()
if not output_path:
output_dir = os.path.dirname(input_path)
output_path = os.path.join(output_dir, "charge_res.h5")
output_dir = os.path.dirname(output_path)
if not os.path.exists(output_dir):
os.makedirs(output_dir)
print("Created directory: {}".format(output_dir))
if os.path.exists(output_path):
os.remove(output_path)
with pd.HDFStore(output_path) as store:
desc0 = "Looping over files"
it = enumerate(df_runs.iterrows())
for i, (_, row) in tqdm(it, total=n_runs, desc=desc0):
reader = row['reader']
transmission = row['transmission']
# for df in reader.iterate_over_chunks():
df = reader.get_first_n_events(1000)
df = df.loc[~df['pixel'].isin(dead)]
pixel = df['pixel'].values
true = spe_handler.calibrate_true(pixel, transmission)
measured = df['charge'].values
measured = spe_handler.calibrate_measured(pixel, measured)
cr.add(pixel, true, measured)
cs.add(pixel, true, measured)
reader.store.close()
df_pixel, df_camera = cr.finish()
store['charge_resolution_pixel'] = df_pixel
store['charge_resolution_camera'] = df_camera
df_pixel, df_camera = cs.finish()
store['charge_statistics_pixel'] = df_pixel
store['charge_statistics_camera'] = df_camera