def obtain_calibrated_df(path_dynrange, path_spe):
with HDF5Reader(path_dynrange) as reader:
df_dynrange = reader.read("data")
with HDF5Reader(path_spe) as reader:
df_spe_values = reader.read("values")
df_spe_errors = reader.read("errors")
spe_scale = reader.get_metadata()["spe_scale"]
expected_illuminations = reader.get_metadata()["illuminations"]
n_illuminations = len(expected_illuminations)
def apply_calibration(df):
ipix = int(df.iloc[0]['pixel'])
df_spe_values_poi = df_spe_values.loc[ipix]
# Calibrate Illumination
measured_illuminations = np.array(
[df_spe_values_poi[f'lambda_{i}'] for i in range(n_illuminations)])
c_ill = polyfit(expected_illuminations, measured_illuminations, 1)
df['true_illumination_pe'] = polyval(df['expected_illumination_pe'],
c_ill)
# Calibration Charge
# scale = df_spe_values_poi['pe'] * spe_scale / (1 - df_spe_values_poi['opct'])
# df['charge_pe'] = df['charge'] / scale
pedestal = 0 #df_spe_values_poi['eped']
df_bright = df.loc[(df['true_illumination_pe'] >= 40)
& (df['true_illumination_pe'] <= 100)]
c_charge = polyfit(df_bright['charge'] - pedestal,
df_bright['true_illumination_pe'], [1])
df['charge_pe'] = polyval(df['charge'] - pedestal, c_charge)
return df
return df_dynrange.groupby('pixel').apply(apply_calibration)
def main():
paths = dict(
self=get_data(
"d191118_pedestal_temperature/d191118/residuals_self.h5"),
single_31degree=get_data(
"d191118_pedestal_temperature/d191118/residuals_single_30.h5"),
lookup=get_data(
"d191118_pedestal_temperature/d191118/residuals_lookup.h5"),
interp=get_data(
"d191118_pedestal_temperature/d191118/residuals_interp.h5"),
pchip=get_data(
"d191118_pedestal_temperature/d191118/residuals_pchip.h5"),
)
p_mean = MeanVsTemperature()
p_std = StdVsTemperature()
p_relstd = RelativeStdVsTemperature()
with HDF5Reader(paths['self']) as reader:
df = reader.read("data")
df = df.set_index("temperature_r0_primary").sort_index()
ref_std = df['std'].values
for label, path in paths.items():
with HDF5Reader(path) as reader:
df = reader.read("data")
df = df.set_index("temperature_r0_primary").sort_index()
temperature = df.index.values
mean = df['mean'].values
std = df['std'].values
p_mean.plot(temperature, mean, label)
p_std.plot(temperature, std, label)
p_relstd.plot(temperature, std, ref_std, label)
p_mean.save(get_plot(f"d191118_pedestal_temperature/d191118/mean.pdf"))
p_std.save(get_plot(f"d191118_pedestal_temperature/d191118/std.pdf"))
p_relstd.save(
get_plot(f"d191118_pedestal_temperature/d191118/rel_std.pdf"))
with HDF5Reader(paths['self']) as reader:
df = reader.read("data")
df = df.set_index("temperature_r0_chamber").sort_index()
chamber = df.index.values
primary = df['temperature_r0_primary'].values
p_temp = TemperatureComparison()
p_temp.plot(chamber, primary)
p_temp.save(get_plot(f"d191118_pedestal_temperature/d191118/temp.pdf"))
def main():
# path = "/Users/Jason/Downloads/tempdata/alpha/mc/gamma_1deg.h5"
path = "/Volumes/gct-jason/astri_onsky_archive/d2019-05-15_simulations/gamma_1deg/run1_hillas.h5"
with HDF5Reader(path) as reader:
df_hillas = reader.read("data")
df_source = reader.read("source")
mapping = reader.get_mapping()
# mapping = get_clp_mapping_from_version("1.1.0")
p_image = ImagePlotter(mapping)
order = np.argsort(df_source['alpha90'])
with PdfPages(get_plot("d190717_alpha/mc_alpha_test.pdf")) as pdf:
for index in order:
row_hillas = df_hillas.iloc[index]
row_source = df_source.iloc[index]
cog_x = row_hillas['x']
cog_y = row_hillas['y']
length = row_hillas['length']
width = row_hillas['width']
psi = row_hillas['psi']
src_x = row_source['source_x']
src_y = row_source['source_y']
alpha90 = row_source['alpha90']
p_image.update(cog_x, cog_y, length, width, psi, src_x, src_y)
p_image.ax.set_title(np.rad2deg(alpha90))
# pdf.savefig(p_image.fig)
plt.pause(0.3)
def main():
pm = PixelMasks()
dead = np.where(np.logical_or(pm.dead, np.repeat(pm.bad_hv, 4)))[0]
bright_path = get_astri_2019("d2019-04-23_nudges/bright_50pe/charge.h5")
with HDF5Reader(bright_path) as reader:
df = reader.read("data").groupby(['nudge',
'pixel']).mean().reset_index()
mapping = reader.get_mapping()
df_0 = df.loc[df['nudge'] == 0]
isin_dead = df_0['pixel'].isin(dead)
avg = df_0.loc[~isin_dead].mean()
ff = avg['onsky_calib'] / df_0['onsky_calib'].values
ff[dead] = 1
np.testing.assert_allclose(ff.mean(), 1, rtol=1e-2)
df = pd.DataFrame(dict(
pixel=np.arange(ff.size),
ff=ff,
))
output_dir = get_astri_2019("d2019-04-23_nudges/results/extract_ff")
cm = CameraImage.from_mapping(mapping)
cm.image = ff
cm.add_colorbar()
cm.highlight_pixels(pm.dead, 'red')
cm.highlight_pixels(np.repeat(pm.bad_hv, 4), 'black')
cm.highlight_pixels(pm.low, 'blue')
cm.save(join(output_dir, "ff_camera.pdf"))
outpath = get_calib_data("ff_coeff.dat")
df.to_csv(outpath, sep='\t', index=False)
print(f"Created ff_coeff file: {outpath}")
def process(path, dead, output):
with HDF5Reader(path) as reader:
df = reader.read("data")
df = df.loc[~df['pixel'].isin(dead)]
d_list = []
extractors = set(df.columns) - {'iobs', 'iev', 'pixel', 'true'}
for extractor in extractors:
gt5 = df['true'].values > 5
true = df['true'].values[gt5]
measured = df[extractor].values[gt5]
coeff = polyfit(true, measured, [1])
_, calib = coeff
df[extractor] /= calib
for true, group in df.groupby("true"):
n = group.index.size
d_list.append(
dict(
extractor=extractor,
true=true,
n=n,
mean=group[extractor].mean(),
std=group[extractor].std(),
rmse=np.sqrt(np.sum((group[extractor] - true)**2) / n),
res=np.sqrt(
np.sum((group[extractor] - true)**2) / n + true),
))
df_cr = pd.DataFrame(d_list)
with HDF5Writer(output) as writer:
writer.write(data=df_cr)
def main():
file = Test()
path_reduced = file.reduced_cell_info
path_tcal = file.tcal
with HDF5Reader(path_reduced) as reader:
df = reader.read("data")
poi = reader.get_metadata()['poi']
df = setup_cells(df)
gb = df.groupby(['fblock', 'fbpisam'])['adc']
pedestal_reduced = gb.mean().unstack().values
hits_reduced = gb.count().unstack().values
std_reduced = gb.std().unstack().values
tm = poi // 64
tmpix = poi % 64
ped_reader = PedestalArrayReader(path_tcal)
pedestal_tcal = np.array(ped_reader.GetPedestal())[tm, tmpix]
hits_tcal = np.array(ped_reader.GetHits())[tm, tmpix]
std_tcal = np.array(ped_reader.GetStdDev())[tm, tmpix]
header = fits.open(path_tcal)[0].header
n_samplesbp = int(header['MAXSAMPLESBP']) + 1
pedestal_tcal = pedestal_tcal[:, :n_samplesbp]
hits_tcal = hits_tcal[:, :n_samplesbp]
std_tcal = std_tcal[:, :n_samplesbp]
print(f"Pedestal: {np.allclose(pedestal_reduced, pedestal_tcal)}")
print(f"Hits: {np.allclose(hits_reduced, hits_tcal)}")
print(f"StdDev: {np.allclose(std_reduced, std_tcal, rtol=1.e-4)}")
def main():
paths = dict(
self=get_data("d191118_pedestal_temperature/d191118/residuals_self.h5"),
single_31degree=get_data("d191118_pedestal_temperature/d191118/residuals_single_30.h5"),
lookup=get_data("d191118_pedestal_temperature/d191118/residuals_lookup.h5"),
interp=get_data("d191118_pedestal_temperature/d191118/residuals_interp.h5"),
pchip=get_data("d191118_pedestal_temperature/d191118/residuals_pchip.h5"),
)
p_hist = dict()
for label, path in paths.items():
with HDF5Reader(path) as reader:
df = reader.read("data")
df = df.set_index("temperature_r0_primary").sort_index()
for temperature, row in df.iterrows():
hist = row['hist']
edges = row['edges']
mean = row['mean']
std = row['std']
if temperature not in p_hist:
p_hist[temperature] = HistPlot()
p_hist[temperature].ax.set_title(f"TM Primary Temperature = {temperature:.2f} °C")
p_hist[temperature].plot(hist, edges, mean, std, label)
for temperature, plot in p_hist.items():
plot.save(get_plot(f"d191118_pedestal_temperature/d191118/hist/T{temperature:.2f}.pdf"))
def main():
path_gamma = "/Users/Jason/Downloads/tempdata/alpha/mc/gamma_1deg.h5"
path_proton = "/Users/Jason/Downloads/tempdata/alpha/mc/proton.h5"
with HDF5Reader(path_gamma) as reader:
df_gamma = reader.read('source')
with HDF5Reader(path_proton) as reader:
df_proton = reader.read('source')
p_hist = Hist()
p_hist.plot(df_gamma['alpha90'].values, "Gamma")
p_hist.plot(df_proton['alpha90'].values, "CRs")
p_hist.ax.set_xlabel("Alpha (deg)")
p_hist.ax.set_ylabel("N Events")
p_hist.add_legend('best')
p_hist.save(get_plot("d190717_alpha/mc_alpha.pdf"))
def main():
with HDF5Reader(get_data("d190730_pedestal/tcal_std.h5")) as r:
df = r.read("data").iloc[2:]
p_std = ComparisonPlotter()
p_std.plot(df['name'].values, df['mean'].values, df['std'].values,
df['max'].values)
p_std.save(get_plot("d190730_pedestal/tcal_std.pdf"))
def main():
path = get_data(f"d191118_pedestal_temperature/d191118/adc_vs_temperature.h5")
with HDF5Reader(path) as reader:
df = reader.read("data")
df = df.set_index("temperature").sort_index()
temperature = df.index.values
delta_mean = df['delta_mean'].values
delta_std = df['delta_std'].values
delta_channel_mean = df['delta_channel_mean'].values
delta_channel_std = df['delta_channel_std'].values
spead_mean = df['spread_mean'].values
spread_std = df['spread_std'].values
p_delta = ValueVsTemp()
p_delta.plot(temperature, delta_mean, delta_std, "TM")
p_delta.plot(temperature, delta_channel_mean, delta_channel_std, "Channel 0")
p_delta.save(get_plot(f"d191118_pedestal_temperature/d191118/delta_vs_temp.pdf"))
# p_delta = ValueVsTemp()
# p_delta.plot(temperature, delta_channel_mean, delta_channel_std)
# p_delta.save(get_plot(f"d191118_pedestal_temperature/d191118/delta_channel_vs_temp.pdf"))
p_delta = SpreadVsTemp()
p_delta.plot(temperature, spead_mean, spread_std)
p_delta.save(get_plot(f"d191118_pedestal_temperature/d191118/spread_vs_temp.pdf"))
paths = dict(
d191118_60hz=get_data("d191118_pedestal_temperature/d191118/adc_vs_temperature.h5"),
d191119_600hz=get_data("d191118_pedestal_temperature/d191119/adc_vs_temperature.h5"),
d191120_200hz=get_data("d191118_pedestal_temperature/d191120/adc_vs_temperature.h5"),
)
p_delta = ValueVsTemp()
for label, path in paths.items():
with HDF5Reader(path) as reader:
df = reader.read("data")
df = df.set_index("temperature").sort_index()
temperature = df.index.values
delta_mean = df['delta_mean'].values
delta_std = df['delta_std'].values
p_delta.plot(temperature, delta_mean, delta_std, label)
p_delta.save(get_plot(f"d191118_pedestal_temperature/delta_vs_temp_vs_day.pdf"))
def process_one_cell(input_path, output_path, cell):
with HDF5Reader(input_path) as reader:
df = reader.read("data")
df = setup_cells(df)
p_cellwf = CellWaveform()
p_cellwf.plot(df, cell)
p_cellwf.save(output_path.format(cell))
def process(path, output, title, cuts):
with HDF5Reader(path) as reader:
df_on = reader.read("on")
df_off = reader.read("off")
metadata = reader.get_metadata()
alpha_li_ma = metadata['alpha_li_ma']
if cuts is not None:
df_on = df_on.query(cuts)
df_off = df_off.query(cuts)
time = np.linspace(1, 2 * 24 * 60 * 60, 1000)
bins = np.linspace(0, 90, 91)
alpha_on = df_on['alpha'].values
weights_on = df_on['weights'].values
rate_on_bins, rate_on_err_bins, edges_on = calculate_rates(
alpha_on, weights_on, bins)
print(f"Total Rate ON: {rate_on_bins.sum()}")
mask_on = alpha_on <= REGION_SIZE
rate_on, rate_on_err, _ = calculate_rates(alpha_on[mask_on],
weights_on[mask_on], 1)
rate_on = rate_on.sum()
print(f"Alpha Rate ON: {rate_on.sum()}")
rate_on_err = rate_on_err.sum()
n_on = rate_on * time
alpha_off = df_off['alpha'].values
weights_off = df_off['weights'].values
rate_off_bins, rate_off_err_bins, edges_off = calculate_rates(
alpha_off, weights_off, bins)
print(f"Total Rate OFF: {rate_off_bins.sum()}")
mask_off = alpha_off <= REGION_SIZE
rate_off, rate_off_err, _ = calculate_rates(alpha_off[mask_off],
weights_off[mask_off], 1)
rate_off = rate_off.sum()
print(f"Alpha Rate OFF: {rate_off.sum()}")
rate_off_err = rate_off_err.sum()
n_off = rate_off * time
significance = li_ma(n_on, n_off, alpha_li_ma)
_, sig_per_sqrthour = polyfit(np.sqrt(time / 3600), significance, [1])
rate_off_bins *= alpha_li_ma
rate_off_err_bins *= alpha_li_ma
p_hist = Hist()
p_hist.plot(rate_off_bins, rate_off_err_bins, edges_off, "OFF")
p_hist.plot(rate_on_bins, rate_on_err_bins, edges_on, "ON")
p_hist.annotate_region(REGION_SIZE)
p_hist.annotate_li_ma(sig_per_sqrthour, f"{rate_on:.2f}±{rate_on_err:.2f}",
f"{rate_off:.2f}±{rate_off_err:.2f}", REGION_SIZE,
alpha_li_ma)
p_hist.ax.set_title(title)
p_hist.add_legend(1)
p_hist.save(output)
def main():
path = Lab_bright().reduced_cell_info
# path = d20190614_pedestal_3().reduced_cell_info
with HDF5Reader(path) as r:
df = r.read("data")
df = setup_cells(df)
df = df.groupby("iev").first().reset_index()
embed()
def main():
# sql = ASTRISQLQuerier()
# start = pd.Timestamp("2019-04-29 00:00")
# end = pd.Timestamp("2019-05-13 00:00")
# ra = sql.get_table_between_datetimes("TCU_ACTUAL_RA", start, end)
# dec = sql.get_table_between_datetimes("TCU_ACTUAL_DEC", start, end)
# alt = sql.get_table_between_datetimes("TCU_ELACTPOS", start, end)
# az = sql.get_table_between_datetimes("TCU_AZACTPOS", start, end)
path = get_astri_2019("astri_db.h5")
with HDF5Reader(path) as reader:
ra = reader.read("TCU_ACTUAL_RA")
dec = reader.read("TCU_ACTUAL_DEC")
alt = reader.read("TCU_ELACTPOS")
az = reader.read("TCU_AZACTPOS")
df = pd.merge(pd.merge(
ra, dec,
on='timestamp').rename(columns=dict(value_x="ra", value_y="dec")),
pd.merge(alt, az, on='timestamp').rename(
columns=dict(value_x="alt", value_y="az")),
on='timestamp')
location = EarthLocation.from_geodetic(lon=14.974609,
lat=37.693267,
height=1750)
altaz_frame = AltAz(location=location, obstime=df['timestamp'])
telescope_pointing = SkyCoord(
alt=df['alt'].values,
az=df['az'].values,
unit='deg',
frame=altaz_frame,
)
telescope_pointing_icrs = telescope_pointing.icrs
df['ra_calc'] = telescope_pointing_icrs.ra.deg
df['dec_calc'] = telescope_pointing_icrs.dec.deg
d_ra = np.arctan2(np.sin(df['ra'] - df['ra_calc']),
np.cos(df['ra'] - df['ra_calc']))
d_dec = np.arctan2(np.sin(df['dec'] - df['dec_calc']),
np.cos(df['dec'] - df['dec_calc']))
p = Plotter()
p.ax.plot(df['timestamp'], d_ra)
p.ax.set_xlabel("Timestamp")
p.ax.set_ylabel("RA_database - RA_calculated (deg)")
p.fig.autofmt_xdate()
p.save(get_plot("d190527_astri_sql/ra.pdf"))
p = Plotter()
p.ax.plot(df['timestamp'], d_dec)
p.ax.set_xlabel("Timestamp")
p.ax.set_ylabel("DEC_database - DEC_calculated (deg)")
p.fig.autofmt_xdate()
p.save(get_plot("d190527_astri_sql/dec.pdf"))
def process_all_cells(input_path, output_path):
with HDF5Reader(input_path) as reader:
df = reader.read("data")
df = setup_cells(df)
cells = np.unique(df['cell'].values)[:128]
for c in cells:
p_cellwf = CellWaveform()
p_cellwf.plot(df, c)
p_cellwf.save(output_path.format(c))
def process(path, output, title, cuts):
with HDF5Reader(path) as reader:
df_on = reader.read("on")
df_off = reader.read("off")
metadata = reader.get_metadata()
alpha_li_ma = metadata['alpha_li_ma']
# obstime = metadata['obstime'].total_seconds() / 3600
if cuts is not None:
df_on = df_on.query(cuts)
df_off = df_off.query(cuts)
# obstime = df_on.
bins = np.linspace(0, 90, 91)
alpha_on = df_on['alpha'].values
n_on_bins, edges_on = np.histogram(alpha_on, bins=bins)
n_on_err_bins = np.sqrt(n_on_bins)
n_on = (alpha_on <= REGION_SIZE).sum()
alpha_off = df_off['alpha'].values
n_off_bins, edges_off = np.histogram(alpha_off, bins=bins)
n_off_err_bins = np.sqrt(n_off_bins)
n_off = (alpha_off <= REGION_SIZE).sum()
significance = li_ma(n_on, n_off, alpha_li_ma)
n_off_bins = n_off_bins * alpha_li_ma
n_off_err_bins = n_off_err_bins * alpha_li_ma
p_hist = Hist()
p_hist.plot(n_off_bins, n_off_err_bins, edges_off, "OFF")
p_hist.plot(n_on_bins, n_on_err_bins, edges_on, "ON")
p_hist.annotate_region(REGION_SIZE)
p_hist.ax.text(
0.95, 0.05,
# r"$T_{{obs}}$ = "f"{obstime:.2f} hours \n"
r"$\sigma_{{Li&Ma}}$ = "f"{significance:.2f}\n"
r"$N_{on}$ = "f"{n_on} events\n"
r"$N_{off}$ = "f"{n_off} events ({n_off*alpha_li_ma:.2f})\n"
r"Cut = "f"{REGION_SIZE:.2f} deg\n"
r"$\alpha$ = "f"{alpha_li_ma:.2f}",
transform=p_hist.ax.transAxes,
horizontalalignment='right',
verticalalignment='bottom',
bbox=dict(
boxstyle="round", fc='blue', alpha=0.7
)
)
p_hist.ax.set_title(title)
p_hist.add_legend(1)
p_hist.save(output)
def main():
description = ('Convert hillas file to h5py.')
parser = argparse.ArgumentParser(description=description,
formatter_class=Formatter)
parser.add_argument('-f',
'--files',
dest='input_path',
help='path to the HDF5 hillas files')
parser.add_argument('-o',
dest='output_path',
required=True,
help='output path to store the h5py file')
args = parser.parse_args()
input_path = args.input_path
output_path = args.output_path
with HDF5Reader(input_path) as reader:
tel = reader.read('data')
arr = reader.read('mc')
pnt = reader.read('pointing')
run = reader.read('mcheader')
arr = arr.rename(
columns={
'energy': 'mc_energy',
'alt': 'mc_alt',
'az': 'mc_az',
'core_x': 'mc_core_x',
'core_y': 'mc_core_y',
'h_first_int': 'mc_h_first_int',
'shower_primary_id': 'mc_shower_primary_id',
'x_max': 'mc_x_max',
#'iobs':'run_id',
#'iev': 'array_event_id'
})
pnt = pnt.drop(columns=['t_cpu'])
arr = pd.merge(arr, pnt, on=['iobs', 'iev'])
arr = arr.rename(columns={'iobs': 'run_id', 'iev': 'array_event_id'})
tel['array_event_id'] = tel.iev.values
tel = tel.rename(columns={'iev': 'telescope_event_id', 'iobs': 'run_id'})
tel = tel.drop(columns=['t_cpu'])
plate_scale = 37.56
tel.x = tel.x * plate_scale
tel.y = tel.y * plate_scale
run = run.rename(columns={'iobs': 'run_id'})
to_h5py(tel, output_path, key='telescope_events', mode='w')
to_h5py(arr, output_path, key='array_events', mode='a')
to_h5py(run, output_path, key='runs', mode='a')
def main():
base = get_astri_2019("")
runlist_paths = [
join(base, "d2019-04-30_cosmicray/runlist.txt"),
join(base, "d2019-05-01_cosmicray/runlist.txt"),
join(base, "d2019-05-01_mrk501/runlist.txt"),
join(base, "d2019-05-02_PG1553+113/runlist.txt"),
join(base, "d2019-05-02_mrk421/runlist.txt"),
join(base, "d2019-05-02_mrk501/runlist.txt"),
join(base, "d2019-05-06_mrk501/runlist.txt"),
join(base, "d2019-05-07_cosmicray/runlist.txt"),
join(base, "d2019-05-08_cosmicray/runlist.txt"),
join(base, "d2019-05-09_mrk421/runlist.txt"),
]
output_path = get_data("d190522_hillas_over_campaign/hillas.h5")
with HDF5Writer(output_path) as writer:
n_investigations = len(runlist_paths)
mapping = None
for iinv, path in enumerate(runlist_paths):
print(f"PROGRESS: Processing inv {iinv+1}/{n_investigations}")
if not exists(path):
raise ValueError(f"Missing runlist file: {path}")
directory = abspath(dirname(path))
investigation = directory.split('/')[-1]
df_runlist = pd.read_csv(path, sep='\t')
for _, row in df_runlist.iterrows():
run = int(row['run'])
hillas_path = join(directory, f"Run{run:05d}_hillas.h5")
with HDF5Reader(hillas_path) as reader:
if mapping is None:
mapping = reader.get_mapping()
keys = ['data', 'pointing', 'mc', 'mcheader']
for key in keys:
if key not in reader.dataframe_keys:
continue
it = enumerate(reader.iterate_over_chunks(key, 1000))
for ientry, df in it:
df['iinv'] = iinv
df['investigation'] = investigation
writer.append(df, key=key)
writer.add_mapping(mapping)
def main():
path = get_data("d190507_check_amplitude_calib/data.h5")
output_path = get_plot("d190507_check_amplitude_calib/plot.pdf")
with HDF5Reader(path) as reader:
df = reader.read('data')
df = df.sort_values('t_cpu')
datetime = df['t_cpu'].values
nudge = df['nudge'].values
temperature = df['temperature'].values
p = Plot()
p.plot(datetime, nudge, temperature)
p.save(output_path)
def main():
path = get_astri_2019("astri_db.h5")
with HDF5Reader(path) as reader:
ra = reader.read("TCU_ACTUAL_RA")
dec = reader.read("TCU_ACTUAL_DEC")
alt = reader.read("TCU_ELACTPOS")
az = reader.read("TCU_AZACTPOS")
df = pd.merge(pd.merge(
ra, dec,
on='timestamp').rename(columns=dict(value_x="ra", value_y="dec")),
pd.merge(alt, az, on='timestamp').rename(
columns=dict(value_x="alt", value_y="az")),
on='timestamp')
df = df.query(" (timestamp > '2019-06-12 18:00')"
"&(timestamp < '2019-06-13 05:00')")
embed()
altaz_frame = AltAz(location=LOCATION, obstime=df['timestamp'].values)
telescope_pointing = SkyCoord(
alt=df['alt'].values,
az=df['az'].values,
unit='deg',
frame=altaz_frame,
)
telescope_pointing_icrs = telescope_pointing.icrs
df['ra_calc'] = telescope_pointing_icrs.ra.deg
df['dec_calc'] = telescope_pointing_icrs.dec.deg
d_ra = np.arctan2(np.sin(df['ra'] - df['ra_calc']),
np.cos(df['ra'] - df['ra_calc']))
d_dec = np.arctan2(np.sin(df['dec'] - df['dec_calc']),
np.cos(df['dec'] - df['dec_calc']))
p = Plotter()
p.ax.plot(df['timestamp'], d_ra)
p.ax.set_xlabel("Timestamp")
p.ax.set_ylabel("RA_database - RA_calculated (deg)")
p.fig.autofmt_xdate()
p.save(get_plot("d190918_alpha/pointing/ra.pdf"))
p = Plotter()
p.ax.plot(df['timestamp'], d_dec)
p.ax.set_xlabel("Timestamp")
p.ax.set_ylabel("DEC_database - DEC_calculated (deg)")
p.fig.autofmt_xdate()
p.save(get_plot("d190918_alpha/pointing/dec.pdf"))
def get_dataframe(path, x_onoff, y_onoff, norm_func):
with HDF5Reader(path) as reader:
df_hillas = reader.read('data')
df_mc = reader.read('mc')
iobs = df_mc['iobs'].values
mcheader = reader.read('mcheader')
n_simulated = (mcheader['num_showers'].values *
mcheader['shower_reuse'].values).sum()
mcheader = mcheader.set_index('iobs').loc[iobs]
x_cog = df_hillas['x'].values
y_cog = df_hillas['y'].values
psi = df_hillas['psi'].values
for iangle in range(len(x_onoff)):
alpha, alpha90 = calculate_alpha(x_onoff[iangle], y_onoff[iangle],
x_cog, y_cog, psi)
df_hillas[f'alpha{iangle}'] = np.rad2deg(alpha90)
energies = df_mc['energy'].values * u.TeV
index = mcheader['spectral_index'].values
e_min = mcheader['energy_range_min'].values * u.TeV
e_max = mcheader['energy_range_max'].values * u.TeV
area = (mcheader['max_scatter_range'].values * u.m)**2 * np.pi
solid_angle = (mcheader['max_viewcone_radius'].values -
mcheader['min_viewcone_radius'].values) * u.deg
# embed()
weights, t_norm = norm_func(e_min, e_max, area, solid_angle, index,
energies, n_simulated)
print(f"tnorm={t_norm}")
df_hillas['energies'] = energies
df_hillas['index'] = index
df_hillas['e_min'] = e_min
df_hillas['e_max'] = e_max
df_hillas['area'] = area
df_hillas['solid_angle'] = solid_angle
df_hillas['weights'] = weights / t_norm
df_hillas['diffuse'] = mcheader['diffuse'].values
plate_scale = FOCAL_LENGTH.to_value(u.m) * np.pi / 180
df_hillas['r'] = df_hillas['r'].values / plate_scale
df_hillas['width'] = df_hillas['width'].values / plate_scale
df_hillas['length'] = df_hillas['length'].values / plate_scale
return df_hillas
def get_dataframe(path, x_onoff, y_onoff):
with HDF5Reader(path) as reader:
df_hillas = reader.read('data')
df_mc = reader.read('mc')
df_mcheader = reader.read('mcheader')
iobs = df_mc['iobs'].values
mcheader = df_mcheader.copy()
mcheader = mcheader.set_index('iobs').loc[iobs]
x_cog = df_hillas['x'].values
y_cog = df_hillas['y'].values
psi = df_hillas['psi'].values
for iangle in range(len(x_onoff)):
alpha, alpha90 = calculate_alpha(
x_onoff[iangle], y_onoff[iangle], x_cog, y_cog, psi
)
df_hillas[f'alpha_{iangle}'] = np.rad2deg(alpha90)
df_hillas[f'xpos_{iangle}'] = x_onoff[iangle]
df_hillas[f'ypos_{iangle}'] = y_onoff[iangle]
energies = df_mc['energy'].values * u.TeV
index = mcheader['spectral_index'].values
e_min = mcheader['energy_range_min'].values * u.TeV
e_max = mcheader['energy_range_max'].values * u.TeV
area = (mcheader['max_scatter_range'].values * u.m) ** 2 * np.pi
solid_angle = (mcheader['max_viewcone_radius'].values -
mcheader['min_viewcone_radius'].values) * u.deg
df_hillas['weights'] = get_weights(df_mc, df_mcheader)
df_hillas['energies'] = energies
df_hillas['index'] = index
df_hillas['e_min'] = e_min
df_hillas['e_max'] = e_max
df_hillas['area'] = area
df_hillas['solid_angle'] = solid_angle
df_hillas['diffuse'] = mcheader['diffuse'].values
plate_scale = FOCAL_LENGTH.to_value(u.m) * np.pi/180
df_hillas['r'] = df_hillas['r'].values / plate_scale
df_hillas['width'] = df_hillas['width'].values / plate_scale
df_hillas['length'] = df_hillas['length'].values / plate_scale
return df_hillas
def main():
input_paths = glob(
"/Volumes/gct-jason/astri_onsky_archive/d2019-11-19_monitor/*_mon.h5")
input_paths = sort_file_list(input_paths)
output_path = "/Volumes/gct-jason/astri_onsky_archive/d2019-11-19_monitor/monitor.h5"
n_files = len(input_paths)
with HDF5Writer(output_path) as writer:
for ipath, input_path in enumerate(input_paths):
print("PROGRESS: Processing file {}/{}".format(ipath + 1, n_files))
with HDF5Reader(input_path) as reader:
keys = reader.dataframe_keys
for key in keys:
df = reader.read(key)
df = df.rename(columns={"icomp": "iunit"})
writer.append(df, key=key)
def process(path, output_dir):
with HDF5Reader(path) as reader:
df = reader.read('data')
dtack = []
stale = []
fci = []
for _, group in df.groupby("ipath"):
dtack.append(np.diff(group['tack']))
stale.append(group['stale'][1:])
fci.append(group['fci'][1:])
dtack = np.concatenate(dtack)
stale = np.concatenate(stale).astype(np.bool)
fci = np.concatenate(fci)
istale = np.where(stale)[0]
embed()
def main():
path = get_data("d190918_alpha/onsky_hillas.h5")
n_off = 5
angles = np.linspace(0, 360, n_off + 2)[:-1] * u.deg
with HDF5Reader(path) as reader:
df_hillas = reader.read('data')
# Convert hillas to degrees
plate_scale = FOCAL_LENGTH.to_value(u.m) * np.pi / 180
df_hillas['r'] = df_hillas['r'].values / plate_scale
df_hillas['width'] = df_hillas['width'].values / plate_scale
df_hillas['length'] = df_hillas['length'].values / plate_scale
df_list = []
for angle in angles:
df_copy = df_hillas.copy()
df_copy['angle'] = angle
df_list.append(df_copy)
df = pd.concat(df_list, ignore_index=True)
print("Defining SkyCoords")
telescope_pointing = get_telescope_pointing(df)
engineering_frame = get_engineering_frame(telescope_pointing)
source_ra = df['source_ra'].values
source_dec = df['source_dec'].values
source_skycoord = SkyCoord(source_ra, source_dec, unit='deg', frame='icrs')
x_cog = df['x'].values
y_cog = df['y'].values
psi = df['psi'].values
embed()
print("Transforming to engineering frame")
position_angle = telescope_pointing.position_angle(source_skycoord).to(
u.deg) - df['angle'].values * u.deg
seperation = telescope_pointing.separation(source_skycoord)
off_skycoord = telescope_pointing.directional_offset_by(
position_angle=position_angle, separation=seperation)
off_cam = off_skycoord.transform_to(engineering_frame)
x_angle = off_cam.x.value
y_angle = off_cam.y.value
df['seperation'] = seperation
df['x_angle'] = x_angle
df['y_angle'] = y_angle
def main():
path = "/Volumes/gct-jason/astri_onsky_archive/hillas_over_campaign.h5"
output = get_plot("d190524_time_gradient/correlations/data")
with HDF5Reader(path) as reader:
df = reader.read("data")
tgradient = df['tgradient'].values
length = df['length'].values
width = df['width'].values
notnoisy = ~np.logical_and(length > 0.1, width > 0.1)
tgradient = tgradient[notnoisy]
length = length[notnoisy]
width = width[notnoisy]
p = Hist2D("Time Gradient (ns/degree)", "Length (degree)")
p.plot(tgradient, length)
p.save(join(output, "tgradvslength.pdf"))
def process(path, output):
with HDF5Reader(path) as reader:
df = reader.read("data")
d_list = []
for extractor, group in df.groupby("extractor"):
params = dict(extractor=extractor)
for key, group_key in group.groupby("key"):
charge = group_key['charge'].values
params[f'mean_{key}'] = np.mean(charge)
params[f'std_{key}'] = np.std(charge)
d_list.append(params)
df_output = pd.DataFrame(d_list)
df_output['sn_on_50'] = df_output['mean_on_50'] / df_output['std_off']
df_output['sn_on_3'] = df_output['mean_on_3'] / df_output['std_off']
with HDF5Writer(output) as writer:
writer.write(data=df_output)
def main():
with HDF5Reader(get_data("d190730_pedestal/ped.h5")) as reader:
df = reader.read('data')
df = setup_cells(df)
std = df.groupby(['fblock', 'fbpisam'])['adc'].std()
print(f"Min hits: {count.min()}")
p_count = LookupTable()
p_count.plot(count, "Counts")
p_count.save(get_plot("d190730_pedestal/count.pdf"))
p_mean = LookupTable()
p_mean.plot(mean, "Mean")
p_mean.save(get_plot("d190730_pedestal/mean.pdf"))
p_std = LookupTable()
p_std.plot(std, "Stddev")
p_std.save(get_plot("d190730_pedestal/std.pdf"))
def calculate_charge_resolution(path):
with HDF5Reader(path) as reader:
df = reader.read("data")
n_channels = np.unique(df['channel']).size
n_true = np.unique(df['amplitude']).size
print(np.unique(df['amplitude']))
true = np.zeros((n_channels, n_true))
resolution = np.zeros((n_channels, n_true))
average = np.zeros((n_channels, n_true))
std = np.zeros((n_channels, n_true))
for channel, group in df.groupby('channel'):
# Calibrate
cal_amplitude = 20
assumed_gain = 4 # mV/p.e.
cal_charge = group.loc[group['amplitude'] == cal_amplitude,
'charge'].values
cal_average = cal_charge.mean()
factor = (cal_amplitude / assumed_gain) / cal_average
group = group.copy()
group['measured'] = group['charge'].values * factor
group['true'] = group['amplitude'].values / assumed_gain
group['n'] = 1
group['sum'] = np.power(group['measured'] - group['true'], 2)
summed = group.groupby(['true']).sum()
true_channel = summed.index.values
sum_channel = summed['sum'].values
n_channel = summed['n'].values
true[channel] = true_channel
resolution[channel] = np.sqrt(
sum_channel / n_channel) / np.abs(true_channel)
# resolution[channel] = np.sqrt((sum_channel / n_channel) + true_channel) / np.abs(true_channel)
average_df = group.groupby(['true']).mean()
average[channel] = average_df['measured'].values
std_df = group.groupby(['true']).std()
std[channel] = std_df['measured'].values
return true, resolution, average, std
def main():
output_path = get_data(f"d190918_alpha/onsky_hillas.h5")
path_list = [
glob(get_astri_2019("d2019-05-01_mrk501/*_hillas.h5")),
glob(get_astri_2019("d2019-05-02_PG1553+113/*_hillas.h5")),
glob(get_astri_2019("d2019-05-02_mrk421/*_hillas.h5")),
glob(get_astri_2019("d2019-05-02_mrk501/*_hillas.h5")),
glob(get_astri_2019("d2019-05-06_mrk501/*_hillas.h5")),
glob(get_astri_2019("d2019-05-09_mrk421/*_hillas.h5")),
glob(get_astri_2019("d2019-06-12_mrk421_moonlight/*_hillas.h5")),
glob(get_astri_2019("d2019-06-12_mrk501/*_hillas.h5")),
glob(get_astri_2019("d2019-06-12_mrk501_moonlight/*_hillas.h5")),
]
with HDF5Writer(output_path) as writer:
for iinv, hillas_paths in enumerate(path_list):
hillas_paths = sort_file_list(hillas_paths)
for ifile, input_path in enumerate(hillas_paths):
with HDF5Reader(input_path) as reader:
if ifile == 0:
writer.add_mapping(reader.get_mapping())
writer.add_metadata(**reader.get_metadata())
df = reader.read("data")
df_pointing = reader.read("pointing")
df_source = reader.read("source")
source_name = reader.get_metadata("source")['source_name']
df['iinv'] = iinv
df['ifile'] = ifile
df['source_ra'] = df_source['source_ra'].values
df['source_dec'] = df_source['source_dec'].values
df['altitude_raw'] = df_pointing['altitude_raw'].values
df['altitude_cor'] = df_pointing['altitude_cor'].values
df['azimuth_raw'] = df_pointing['azimuth_raw'].values
df['azimuth_cor'] = df_pointing['azimuth_cor'].values
df['source_name'] = source_name
writer.append(df, 'data')