def obtain_calibrated_df(path_dynrange, path_spe): with HDF5Reader(path_dynrange) as reader: df_dynrange = reader.read("data") with HDF5Reader(path_spe) as reader: df_spe_values = reader.read("values") df_spe_errors = reader.read("errors") spe_scale = reader.get_metadata()["spe_scale"] expected_illuminations = reader.get_metadata()["illuminations"] n_illuminations = len(expected_illuminations) def apply_calibration(df): ipix = int(df.iloc[0]['pixel']) df_spe_values_poi = df_spe_values.loc[ipix] # Calibrate Illumination measured_illuminations = np.array( [df_spe_values_poi[f'lambda_{i}'] for i in range(n_illuminations)]) c_ill = polyfit(expected_illuminations, measured_illuminations, 1) df['true_illumination_pe'] = polyval(df['expected_illumination_pe'], c_ill) # Calibration Charge # scale = df_spe_values_poi['pe'] * spe_scale / (1 - df_spe_values_poi['opct']) # df['charge_pe'] = df['charge'] / scale pedestal = 0 #df_spe_values_poi['eped'] df_bright = df.loc[(df['true_illumination_pe'] >= 40) & (df['true_illumination_pe'] <= 100)] c_charge = polyfit(df_bright['charge'] - pedestal, df_bright['true_illumination_pe'], [1]) df['charge_pe'] = polyval(df['charge'] - pedestal, c_charge) return df return df_dynrange.groupby('pixel').apply(apply_calibration)
def main(): paths = dict( self=get_data( "d191118_pedestal_temperature/d191118/residuals_self.h5"), single_31degree=get_data( "d191118_pedestal_temperature/d191118/residuals_single_30.h5"), lookup=get_data( "d191118_pedestal_temperature/d191118/residuals_lookup.h5"), interp=get_data( "d191118_pedestal_temperature/d191118/residuals_interp.h5"), pchip=get_data( "d191118_pedestal_temperature/d191118/residuals_pchip.h5"), ) p_mean = MeanVsTemperature() p_std = StdVsTemperature() p_relstd = RelativeStdVsTemperature() with HDF5Reader(paths['self']) as reader: df = reader.read("data") df = df.set_index("temperature_r0_primary").sort_index() ref_std = df['std'].values for label, path in paths.items(): with HDF5Reader(path) as reader: df = reader.read("data") df = df.set_index("temperature_r0_primary").sort_index() temperature = df.index.values mean = df['mean'].values std = df['std'].values p_mean.plot(temperature, mean, label) p_std.plot(temperature, std, label) p_relstd.plot(temperature, std, ref_std, label) p_mean.save(get_plot(f"d191118_pedestal_temperature/d191118/mean.pdf")) p_std.save(get_plot(f"d191118_pedestal_temperature/d191118/std.pdf")) p_relstd.save( get_plot(f"d191118_pedestal_temperature/d191118/rel_std.pdf")) with HDF5Reader(paths['self']) as reader: df = reader.read("data") df = df.set_index("temperature_r0_chamber").sort_index() chamber = df.index.values primary = df['temperature_r0_primary'].values p_temp = TemperatureComparison() p_temp.plot(chamber, primary) p_temp.save(get_plot(f"d191118_pedestal_temperature/d191118/temp.pdf"))
def main(): # path = "/Users/Jason/Downloads/tempdata/alpha/mc/gamma_1deg.h5" path = "/Volumes/gct-jason/astri_onsky_archive/d2019-05-15_simulations/gamma_1deg/run1_hillas.h5" with HDF5Reader(path) as reader: df_hillas = reader.read("data") df_source = reader.read("source") mapping = reader.get_mapping() # mapping = get_clp_mapping_from_version("1.1.0") p_image = ImagePlotter(mapping) order = np.argsort(df_source['alpha90']) with PdfPages(get_plot("d190717_alpha/mc_alpha_test.pdf")) as pdf: for index in order: row_hillas = df_hillas.iloc[index] row_source = df_source.iloc[index] cog_x = row_hillas['x'] cog_y = row_hillas['y'] length = row_hillas['length'] width = row_hillas['width'] psi = row_hillas['psi'] src_x = row_source['source_x'] src_y = row_source['source_y'] alpha90 = row_source['alpha90'] p_image.update(cog_x, cog_y, length, width, psi, src_x, src_y) p_image.ax.set_title(np.rad2deg(alpha90)) # pdf.savefig(p_image.fig) plt.pause(0.3)
def main(): pm = PixelMasks() dead = np.where(np.logical_or(pm.dead, np.repeat(pm.bad_hv, 4)))[0] bright_path = get_astri_2019("d2019-04-23_nudges/bright_50pe/charge.h5") with HDF5Reader(bright_path) as reader: df = reader.read("data").groupby(['nudge', 'pixel']).mean().reset_index() mapping = reader.get_mapping() df_0 = df.loc[df['nudge'] == 0] isin_dead = df_0['pixel'].isin(dead) avg = df_0.loc[~isin_dead].mean() ff = avg['onsky_calib'] / df_0['onsky_calib'].values ff[dead] = 1 np.testing.assert_allclose(ff.mean(), 1, rtol=1e-2) df = pd.DataFrame(dict( pixel=np.arange(ff.size), ff=ff, )) output_dir = get_astri_2019("d2019-04-23_nudges/results/extract_ff") cm = CameraImage.from_mapping(mapping) cm.image = ff cm.add_colorbar() cm.highlight_pixels(pm.dead, 'red') cm.highlight_pixels(np.repeat(pm.bad_hv, 4), 'black') cm.highlight_pixels(pm.low, 'blue') cm.save(join(output_dir, "ff_camera.pdf")) outpath = get_calib_data("ff_coeff.dat") df.to_csv(outpath, sep='\t', index=False) print(f"Created ff_coeff file: {outpath}")
def process(path, dead, output): with HDF5Reader(path) as reader: df = reader.read("data") df = df.loc[~df['pixel'].isin(dead)] d_list = [] extractors = set(df.columns) - {'iobs', 'iev', 'pixel', 'true'} for extractor in extractors: gt5 = df['true'].values > 5 true = df['true'].values[gt5] measured = df[extractor].values[gt5] coeff = polyfit(true, measured, [1]) _, calib = coeff df[extractor] /= calib for true, group in df.groupby("true"): n = group.index.size d_list.append( dict( extractor=extractor, true=true, n=n, mean=group[extractor].mean(), std=group[extractor].std(), rmse=np.sqrt(np.sum((group[extractor] - true)**2) / n), res=np.sqrt( np.sum((group[extractor] - true)**2) / n + true), )) df_cr = pd.DataFrame(d_list) with HDF5Writer(output) as writer: writer.write(data=df_cr)
def main(): file = Test() path_reduced = file.reduced_cell_info path_tcal = file.tcal with HDF5Reader(path_reduced) as reader: df = reader.read("data") poi = reader.get_metadata()['poi'] df = setup_cells(df) gb = df.groupby(['fblock', 'fbpisam'])['adc'] pedestal_reduced = gb.mean().unstack().values hits_reduced = gb.count().unstack().values std_reduced = gb.std().unstack().values tm = poi // 64 tmpix = poi % 64 ped_reader = PedestalArrayReader(path_tcal) pedestal_tcal = np.array(ped_reader.GetPedestal())[tm, tmpix] hits_tcal = np.array(ped_reader.GetHits())[tm, tmpix] std_tcal = np.array(ped_reader.GetStdDev())[tm, tmpix] header = fits.open(path_tcal)[0].header n_samplesbp = int(header['MAXSAMPLESBP']) + 1 pedestal_tcal = pedestal_tcal[:, :n_samplesbp] hits_tcal = hits_tcal[:, :n_samplesbp] std_tcal = std_tcal[:, :n_samplesbp] print(f"Pedestal: {np.allclose(pedestal_reduced, pedestal_tcal)}") print(f"Hits: {np.allclose(hits_reduced, hits_tcal)}") print(f"StdDev: {np.allclose(std_reduced, std_tcal, rtol=1.e-4)}")
def main(): paths = dict( self=get_data("d191118_pedestal_temperature/d191118/residuals_self.h5"), single_31degree=get_data("d191118_pedestal_temperature/d191118/residuals_single_30.h5"), lookup=get_data("d191118_pedestal_temperature/d191118/residuals_lookup.h5"), interp=get_data("d191118_pedestal_temperature/d191118/residuals_interp.h5"), pchip=get_data("d191118_pedestal_temperature/d191118/residuals_pchip.h5"), ) p_hist = dict() for label, path in paths.items(): with HDF5Reader(path) as reader: df = reader.read("data") df = df.set_index("temperature_r0_primary").sort_index() for temperature, row in df.iterrows(): hist = row['hist'] edges = row['edges'] mean = row['mean'] std = row['std'] if temperature not in p_hist: p_hist[temperature] = HistPlot() p_hist[temperature].ax.set_title(f"TM Primary Temperature = {temperature:.2f} °C") p_hist[temperature].plot(hist, edges, mean, std, label) for temperature, plot in p_hist.items(): plot.save(get_plot(f"d191118_pedestal_temperature/d191118/hist/T{temperature:.2f}.pdf"))
def main(): path_gamma = "/Users/Jason/Downloads/tempdata/alpha/mc/gamma_1deg.h5" path_proton = "/Users/Jason/Downloads/tempdata/alpha/mc/proton.h5" with HDF5Reader(path_gamma) as reader: df_gamma = reader.read('source') with HDF5Reader(path_proton) as reader: df_proton = reader.read('source') p_hist = Hist() p_hist.plot(df_gamma['alpha90'].values, "Gamma") p_hist.plot(df_proton['alpha90'].values, "CRs") p_hist.ax.set_xlabel("Alpha (deg)") p_hist.ax.set_ylabel("N Events") p_hist.add_legend('best') p_hist.save(get_plot("d190717_alpha/mc_alpha.pdf"))
def main(): with HDF5Reader(get_data("d190730_pedestal/tcal_std.h5")) as r: df = r.read("data").iloc[2:] p_std = ComparisonPlotter() p_std.plot(df['name'].values, df['mean'].values, df['std'].values, df['max'].values) p_std.save(get_plot("d190730_pedestal/tcal_std.pdf"))
def main(): path = get_data(f"d191118_pedestal_temperature/d191118/adc_vs_temperature.h5") with HDF5Reader(path) as reader: df = reader.read("data") df = df.set_index("temperature").sort_index() temperature = df.index.values delta_mean = df['delta_mean'].values delta_std = df['delta_std'].values delta_channel_mean = df['delta_channel_mean'].values delta_channel_std = df['delta_channel_std'].values spead_mean = df['spread_mean'].values spread_std = df['spread_std'].values p_delta = ValueVsTemp() p_delta.plot(temperature, delta_mean, delta_std, "TM") p_delta.plot(temperature, delta_channel_mean, delta_channel_std, "Channel 0") p_delta.save(get_plot(f"d191118_pedestal_temperature/d191118/delta_vs_temp.pdf")) # p_delta = ValueVsTemp() # p_delta.plot(temperature, delta_channel_mean, delta_channel_std) # p_delta.save(get_plot(f"d191118_pedestal_temperature/d191118/delta_channel_vs_temp.pdf")) p_delta = SpreadVsTemp() p_delta.plot(temperature, spead_mean, spread_std) p_delta.save(get_plot(f"d191118_pedestal_temperature/d191118/spread_vs_temp.pdf")) paths = dict( d191118_60hz=get_data("d191118_pedestal_temperature/d191118/adc_vs_temperature.h5"), d191119_600hz=get_data("d191118_pedestal_temperature/d191119/adc_vs_temperature.h5"), d191120_200hz=get_data("d191118_pedestal_temperature/d191120/adc_vs_temperature.h5"), ) p_delta = ValueVsTemp() for label, path in paths.items(): with HDF5Reader(path) as reader: df = reader.read("data") df = df.set_index("temperature").sort_index() temperature = df.index.values delta_mean = df['delta_mean'].values delta_std = df['delta_std'].values p_delta.plot(temperature, delta_mean, delta_std, label) p_delta.save(get_plot(f"d191118_pedestal_temperature/delta_vs_temp_vs_day.pdf"))
def process_one_cell(input_path, output_path, cell): with HDF5Reader(input_path) as reader: df = reader.read("data") df = setup_cells(df) p_cellwf = CellWaveform() p_cellwf.plot(df, cell) p_cellwf.save(output_path.format(cell))
def process(path, output, title, cuts): with HDF5Reader(path) as reader: df_on = reader.read("on") df_off = reader.read("off") metadata = reader.get_metadata() alpha_li_ma = metadata['alpha_li_ma'] if cuts is not None: df_on = df_on.query(cuts) df_off = df_off.query(cuts) time = np.linspace(1, 2 * 24 * 60 * 60, 1000) bins = np.linspace(0, 90, 91) alpha_on = df_on['alpha'].values weights_on = df_on['weights'].values rate_on_bins, rate_on_err_bins, edges_on = calculate_rates( alpha_on, weights_on, bins) print(f"Total Rate ON: {rate_on_bins.sum()}") mask_on = alpha_on <= REGION_SIZE rate_on, rate_on_err, _ = calculate_rates(alpha_on[mask_on], weights_on[mask_on], 1) rate_on = rate_on.sum() print(f"Alpha Rate ON: {rate_on.sum()}") rate_on_err = rate_on_err.sum() n_on = rate_on * time alpha_off = df_off['alpha'].values weights_off = df_off['weights'].values rate_off_bins, rate_off_err_bins, edges_off = calculate_rates( alpha_off, weights_off, bins) print(f"Total Rate OFF: {rate_off_bins.sum()}") mask_off = alpha_off <= REGION_SIZE rate_off, rate_off_err, _ = calculate_rates(alpha_off[mask_off], weights_off[mask_off], 1) rate_off = rate_off.sum() print(f"Alpha Rate OFF: {rate_off.sum()}") rate_off_err = rate_off_err.sum() n_off = rate_off * time significance = li_ma(n_on, n_off, alpha_li_ma) _, sig_per_sqrthour = polyfit(np.sqrt(time / 3600), significance, [1]) rate_off_bins *= alpha_li_ma rate_off_err_bins *= alpha_li_ma p_hist = Hist() p_hist.plot(rate_off_bins, rate_off_err_bins, edges_off, "OFF") p_hist.plot(rate_on_bins, rate_on_err_bins, edges_on, "ON") p_hist.annotate_region(REGION_SIZE) p_hist.annotate_li_ma(sig_per_sqrthour, f"{rate_on:.2f}±{rate_on_err:.2f}", f"{rate_off:.2f}±{rate_off_err:.2f}", REGION_SIZE, alpha_li_ma) p_hist.ax.set_title(title) p_hist.add_legend(1) p_hist.save(output)
def main(): path = Lab_bright().reduced_cell_info # path = d20190614_pedestal_3().reduced_cell_info with HDF5Reader(path) as r: df = r.read("data") df = setup_cells(df) df = df.groupby("iev").first().reset_index() embed()
def main(): # sql = ASTRISQLQuerier() # start = pd.Timestamp("2019-04-29 00:00") # end = pd.Timestamp("2019-05-13 00:00") # ra = sql.get_table_between_datetimes("TCU_ACTUAL_RA", start, end) # dec = sql.get_table_between_datetimes("TCU_ACTUAL_DEC", start, end) # alt = sql.get_table_between_datetimes("TCU_ELACTPOS", start, end) # az = sql.get_table_between_datetimes("TCU_AZACTPOS", start, end) path = get_astri_2019("astri_db.h5") with HDF5Reader(path) as reader: ra = reader.read("TCU_ACTUAL_RA") dec = reader.read("TCU_ACTUAL_DEC") alt = reader.read("TCU_ELACTPOS") az = reader.read("TCU_AZACTPOS") df = pd.merge(pd.merge( ra, dec, on='timestamp').rename(columns=dict(value_x="ra", value_y="dec")), pd.merge(alt, az, on='timestamp').rename( columns=dict(value_x="alt", value_y="az")), on='timestamp') location = EarthLocation.from_geodetic(lon=14.974609, lat=37.693267, height=1750) altaz_frame = AltAz(location=location, obstime=df['timestamp']) telescope_pointing = SkyCoord( alt=df['alt'].values, az=df['az'].values, unit='deg', frame=altaz_frame, ) telescope_pointing_icrs = telescope_pointing.icrs df['ra_calc'] = telescope_pointing_icrs.ra.deg df['dec_calc'] = telescope_pointing_icrs.dec.deg d_ra = np.arctan2(np.sin(df['ra'] - df['ra_calc']), np.cos(df['ra'] - df['ra_calc'])) d_dec = np.arctan2(np.sin(df['dec'] - df['dec_calc']), np.cos(df['dec'] - df['dec_calc'])) p = Plotter() p.ax.plot(df['timestamp'], d_ra) p.ax.set_xlabel("Timestamp") p.ax.set_ylabel("RA_database - RA_calculated (deg)") p.fig.autofmt_xdate() p.save(get_plot("d190527_astri_sql/ra.pdf")) p = Plotter() p.ax.plot(df['timestamp'], d_dec) p.ax.set_xlabel("Timestamp") p.ax.set_ylabel("DEC_database - DEC_calculated (deg)") p.fig.autofmt_xdate() p.save(get_plot("d190527_astri_sql/dec.pdf"))
def process_all_cells(input_path, output_path): with HDF5Reader(input_path) as reader: df = reader.read("data") df = setup_cells(df) cells = np.unique(df['cell'].values)[:128] for c in cells: p_cellwf = CellWaveform() p_cellwf.plot(df, c) p_cellwf.save(output_path.format(c))
def process(path, output, title, cuts): with HDF5Reader(path) as reader: df_on = reader.read("on") df_off = reader.read("off") metadata = reader.get_metadata() alpha_li_ma = metadata['alpha_li_ma'] # obstime = metadata['obstime'].total_seconds() / 3600 if cuts is not None: df_on = df_on.query(cuts) df_off = df_off.query(cuts) # obstime = df_on. bins = np.linspace(0, 90, 91) alpha_on = df_on['alpha'].values n_on_bins, edges_on = np.histogram(alpha_on, bins=bins) n_on_err_bins = np.sqrt(n_on_bins) n_on = (alpha_on <= REGION_SIZE).sum() alpha_off = df_off['alpha'].values n_off_bins, edges_off = np.histogram(alpha_off, bins=bins) n_off_err_bins = np.sqrt(n_off_bins) n_off = (alpha_off <= REGION_SIZE).sum() significance = li_ma(n_on, n_off, alpha_li_ma) n_off_bins = n_off_bins * alpha_li_ma n_off_err_bins = n_off_err_bins * alpha_li_ma p_hist = Hist() p_hist.plot(n_off_bins, n_off_err_bins, edges_off, "OFF") p_hist.plot(n_on_bins, n_on_err_bins, edges_on, "ON") p_hist.annotate_region(REGION_SIZE) p_hist.ax.text( 0.95, 0.05, # r"$T_{{obs}}$ = "f"{obstime:.2f} hours \n" r"$\sigma_{{Li&Ma}}$ = "f"{significance:.2f}\n" r"$N_{on}$ = "f"{n_on} events\n" r"$N_{off}$ = "f"{n_off} events ({n_off*alpha_li_ma:.2f})\n" r"Cut = "f"{REGION_SIZE:.2f} deg\n" r"$\alpha$ = "f"{alpha_li_ma:.2f}", transform=p_hist.ax.transAxes, horizontalalignment='right', verticalalignment='bottom', bbox=dict( boxstyle="round", fc='blue', alpha=0.7 ) ) p_hist.ax.set_title(title) p_hist.add_legend(1) p_hist.save(output)
def main(): description = ('Convert hillas file to h5py.') parser = argparse.ArgumentParser(description=description, formatter_class=Formatter) parser.add_argument('-f', '--files', dest='input_path', help='path to the HDF5 hillas files') parser.add_argument('-o', dest='output_path', required=True, help='output path to store the h5py file') args = parser.parse_args() input_path = args.input_path output_path = args.output_path with HDF5Reader(input_path) as reader: tel = reader.read('data') arr = reader.read('mc') pnt = reader.read('pointing') run = reader.read('mcheader') arr = arr.rename( columns={ 'energy': 'mc_energy', 'alt': 'mc_alt', 'az': 'mc_az', 'core_x': 'mc_core_x', 'core_y': 'mc_core_y', 'h_first_int': 'mc_h_first_int', 'shower_primary_id': 'mc_shower_primary_id', 'x_max': 'mc_x_max', #'iobs':'run_id', #'iev': 'array_event_id' }) pnt = pnt.drop(columns=['t_cpu']) arr = pd.merge(arr, pnt, on=['iobs', 'iev']) arr = arr.rename(columns={'iobs': 'run_id', 'iev': 'array_event_id'}) tel['array_event_id'] = tel.iev.values tel = tel.rename(columns={'iev': 'telescope_event_id', 'iobs': 'run_id'}) tel = tel.drop(columns=['t_cpu']) plate_scale = 37.56 tel.x = tel.x * plate_scale tel.y = tel.y * plate_scale run = run.rename(columns={'iobs': 'run_id'}) to_h5py(tel, output_path, key='telescope_events', mode='w') to_h5py(arr, output_path, key='array_events', mode='a') to_h5py(run, output_path, key='runs', mode='a')
def main(): base = get_astri_2019("") runlist_paths = [ join(base, "d2019-04-30_cosmicray/runlist.txt"), join(base, "d2019-05-01_cosmicray/runlist.txt"), join(base, "d2019-05-01_mrk501/runlist.txt"), join(base, "d2019-05-02_PG1553+113/runlist.txt"), join(base, "d2019-05-02_mrk421/runlist.txt"), join(base, "d2019-05-02_mrk501/runlist.txt"), join(base, "d2019-05-06_mrk501/runlist.txt"), join(base, "d2019-05-07_cosmicray/runlist.txt"), join(base, "d2019-05-08_cosmicray/runlist.txt"), join(base, "d2019-05-09_mrk421/runlist.txt"), ] output_path = get_data("d190522_hillas_over_campaign/hillas.h5") with HDF5Writer(output_path) as writer: n_investigations = len(runlist_paths) mapping = None for iinv, path in enumerate(runlist_paths): print(f"PROGRESS: Processing inv {iinv+1}/{n_investigations}") if not exists(path): raise ValueError(f"Missing runlist file: {path}") directory = abspath(dirname(path)) investigation = directory.split('/')[-1] df_runlist = pd.read_csv(path, sep='\t') for _, row in df_runlist.iterrows(): run = int(row['run']) hillas_path = join(directory, f"Run{run:05d}_hillas.h5") with HDF5Reader(hillas_path) as reader: if mapping is None: mapping = reader.get_mapping() keys = ['data', 'pointing', 'mc', 'mcheader'] for key in keys: if key not in reader.dataframe_keys: continue it = enumerate(reader.iterate_over_chunks(key, 1000)) for ientry, df in it: df['iinv'] = iinv df['investigation'] = investigation writer.append(df, key=key) writer.add_mapping(mapping)
def main(): path = get_data("d190507_check_amplitude_calib/data.h5") output_path = get_plot("d190507_check_amplitude_calib/plot.pdf") with HDF5Reader(path) as reader: df = reader.read('data') df = df.sort_values('t_cpu') datetime = df['t_cpu'].values nudge = df['nudge'].values temperature = df['temperature'].values p = Plot() p.plot(datetime, nudge, temperature) p.save(output_path)
def main(): path = get_astri_2019("astri_db.h5") with HDF5Reader(path) as reader: ra = reader.read("TCU_ACTUAL_RA") dec = reader.read("TCU_ACTUAL_DEC") alt = reader.read("TCU_ELACTPOS") az = reader.read("TCU_AZACTPOS") df = pd.merge(pd.merge( ra, dec, on='timestamp').rename(columns=dict(value_x="ra", value_y="dec")), pd.merge(alt, az, on='timestamp').rename( columns=dict(value_x="alt", value_y="az")), on='timestamp') df = df.query(" (timestamp > '2019-06-12 18:00')" "&(timestamp < '2019-06-13 05:00')") embed() altaz_frame = AltAz(location=LOCATION, obstime=df['timestamp'].values) telescope_pointing = SkyCoord( alt=df['alt'].values, az=df['az'].values, unit='deg', frame=altaz_frame, ) telescope_pointing_icrs = telescope_pointing.icrs df['ra_calc'] = telescope_pointing_icrs.ra.deg df['dec_calc'] = telescope_pointing_icrs.dec.deg d_ra = np.arctan2(np.sin(df['ra'] - df['ra_calc']), np.cos(df['ra'] - df['ra_calc'])) d_dec = np.arctan2(np.sin(df['dec'] - df['dec_calc']), np.cos(df['dec'] - df['dec_calc'])) p = Plotter() p.ax.plot(df['timestamp'], d_ra) p.ax.set_xlabel("Timestamp") p.ax.set_ylabel("RA_database - RA_calculated (deg)") p.fig.autofmt_xdate() p.save(get_plot("d190918_alpha/pointing/ra.pdf")) p = Plotter() p.ax.plot(df['timestamp'], d_dec) p.ax.set_xlabel("Timestamp") p.ax.set_ylabel("DEC_database - DEC_calculated (deg)") p.fig.autofmt_xdate() p.save(get_plot("d190918_alpha/pointing/dec.pdf"))
def get_dataframe(path, x_onoff, y_onoff, norm_func): with HDF5Reader(path) as reader: df_hillas = reader.read('data') df_mc = reader.read('mc') iobs = df_mc['iobs'].values mcheader = reader.read('mcheader') n_simulated = (mcheader['num_showers'].values * mcheader['shower_reuse'].values).sum() mcheader = mcheader.set_index('iobs').loc[iobs] x_cog = df_hillas['x'].values y_cog = df_hillas['y'].values psi = df_hillas['psi'].values for iangle in range(len(x_onoff)): alpha, alpha90 = calculate_alpha(x_onoff[iangle], y_onoff[iangle], x_cog, y_cog, psi) df_hillas[f'alpha{iangle}'] = np.rad2deg(alpha90) energies = df_mc['energy'].values * u.TeV index = mcheader['spectral_index'].values e_min = mcheader['energy_range_min'].values * u.TeV e_max = mcheader['energy_range_max'].values * u.TeV area = (mcheader['max_scatter_range'].values * u.m)**2 * np.pi solid_angle = (mcheader['max_viewcone_radius'].values - mcheader['min_viewcone_radius'].values) * u.deg # embed() weights, t_norm = norm_func(e_min, e_max, area, solid_angle, index, energies, n_simulated) print(f"tnorm={t_norm}") df_hillas['energies'] = energies df_hillas['index'] = index df_hillas['e_min'] = e_min df_hillas['e_max'] = e_max df_hillas['area'] = area df_hillas['solid_angle'] = solid_angle df_hillas['weights'] = weights / t_norm df_hillas['diffuse'] = mcheader['diffuse'].values plate_scale = FOCAL_LENGTH.to_value(u.m) * np.pi / 180 df_hillas['r'] = df_hillas['r'].values / plate_scale df_hillas['width'] = df_hillas['width'].values / plate_scale df_hillas['length'] = df_hillas['length'].values / plate_scale return df_hillas
def get_dataframe(path, x_onoff, y_onoff): with HDF5Reader(path) as reader: df_hillas = reader.read('data') df_mc = reader.read('mc') df_mcheader = reader.read('mcheader') iobs = df_mc['iobs'].values mcheader = df_mcheader.copy() mcheader = mcheader.set_index('iobs').loc[iobs] x_cog = df_hillas['x'].values y_cog = df_hillas['y'].values psi = df_hillas['psi'].values for iangle in range(len(x_onoff)): alpha, alpha90 = calculate_alpha( x_onoff[iangle], y_onoff[iangle], x_cog, y_cog, psi ) df_hillas[f'alpha_{iangle}'] = np.rad2deg(alpha90) df_hillas[f'xpos_{iangle}'] = x_onoff[iangle] df_hillas[f'ypos_{iangle}'] = y_onoff[iangle] energies = df_mc['energy'].values * u.TeV index = mcheader['spectral_index'].values e_min = mcheader['energy_range_min'].values * u.TeV e_max = mcheader['energy_range_max'].values * u.TeV area = (mcheader['max_scatter_range'].values * u.m) ** 2 * np.pi solid_angle = (mcheader['max_viewcone_radius'].values - mcheader['min_viewcone_radius'].values) * u.deg df_hillas['weights'] = get_weights(df_mc, df_mcheader) df_hillas['energies'] = energies df_hillas['index'] = index df_hillas['e_min'] = e_min df_hillas['e_max'] = e_max df_hillas['area'] = area df_hillas['solid_angle'] = solid_angle df_hillas['diffuse'] = mcheader['diffuse'].values plate_scale = FOCAL_LENGTH.to_value(u.m) * np.pi/180 df_hillas['r'] = df_hillas['r'].values / plate_scale df_hillas['width'] = df_hillas['width'].values / plate_scale df_hillas['length'] = df_hillas['length'].values / plate_scale return df_hillas
def main(): input_paths = glob( "/Volumes/gct-jason/astri_onsky_archive/d2019-11-19_monitor/*_mon.h5") input_paths = sort_file_list(input_paths) output_path = "/Volumes/gct-jason/astri_onsky_archive/d2019-11-19_monitor/monitor.h5" n_files = len(input_paths) with HDF5Writer(output_path) as writer: for ipath, input_path in enumerate(input_paths): print("PROGRESS: Processing file {}/{}".format(ipath + 1, n_files)) with HDF5Reader(input_path) as reader: keys = reader.dataframe_keys for key in keys: df = reader.read(key) df = df.rename(columns={"icomp": "iunit"}) writer.append(df, key=key)
def process(path, output_dir): with HDF5Reader(path) as reader: df = reader.read('data') dtack = [] stale = [] fci = [] for _, group in df.groupby("ipath"): dtack.append(np.diff(group['tack'])) stale.append(group['stale'][1:]) fci.append(group['fci'][1:]) dtack = np.concatenate(dtack) stale = np.concatenate(stale).astype(np.bool) fci = np.concatenate(fci) istale = np.where(stale)[0] embed()
def main(): path = get_data("d190918_alpha/onsky_hillas.h5") n_off = 5 angles = np.linspace(0, 360, n_off + 2)[:-1] * u.deg with HDF5Reader(path) as reader: df_hillas = reader.read('data') # Convert hillas to degrees plate_scale = FOCAL_LENGTH.to_value(u.m) * np.pi / 180 df_hillas['r'] = df_hillas['r'].values / plate_scale df_hillas['width'] = df_hillas['width'].values / plate_scale df_hillas['length'] = df_hillas['length'].values / plate_scale df_list = [] for angle in angles: df_copy = df_hillas.copy() df_copy['angle'] = angle df_list.append(df_copy) df = pd.concat(df_list, ignore_index=True) print("Defining SkyCoords") telescope_pointing = get_telescope_pointing(df) engineering_frame = get_engineering_frame(telescope_pointing) source_ra = df['source_ra'].values source_dec = df['source_dec'].values source_skycoord = SkyCoord(source_ra, source_dec, unit='deg', frame='icrs') x_cog = df['x'].values y_cog = df['y'].values psi = df['psi'].values embed() print("Transforming to engineering frame") position_angle = telescope_pointing.position_angle(source_skycoord).to( u.deg) - df['angle'].values * u.deg seperation = telescope_pointing.separation(source_skycoord) off_skycoord = telescope_pointing.directional_offset_by( position_angle=position_angle, separation=seperation) off_cam = off_skycoord.transform_to(engineering_frame) x_angle = off_cam.x.value y_angle = off_cam.y.value df['seperation'] = seperation df['x_angle'] = x_angle df['y_angle'] = y_angle
def main(): path = "/Volumes/gct-jason/astri_onsky_archive/hillas_over_campaign.h5" output = get_plot("d190524_time_gradient/correlations/data") with HDF5Reader(path) as reader: df = reader.read("data") tgradient = df['tgradient'].values length = df['length'].values width = df['width'].values notnoisy = ~np.logical_and(length > 0.1, width > 0.1) tgradient = tgradient[notnoisy] length = length[notnoisy] width = width[notnoisy] p = Hist2D("Time Gradient (ns/degree)", "Length (degree)") p.plot(tgradient, length) p.save(join(output, "tgradvslength.pdf"))
def process(path, output): with HDF5Reader(path) as reader: df = reader.read("data") d_list = [] for extractor, group in df.groupby("extractor"): params = dict(extractor=extractor) for key, group_key in group.groupby("key"): charge = group_key['charge'].values params[f'mean_{key}'] = np.mean(charge) params[f'std_{key}'] = np.std(charge) d_list.append(params) df_output = pd.DataFrame(d_list) df_output['sn_on_50'] = df_output['mean_on_50'] / df_output['std_off'] df_output['sn_on_3'] = df_output['mean_on_3'] / df_output['std_off'] with HDF5Writer(output) as writer: writer.write(data=df_output)
def main(): with HDF5Reader(get_data("d190730_pedestal/ped.h5")) as reader: df = reader.read('data') df = setup_cells(df) std = df.groupby(['fblock', 'fbpisam'])['adc'].std() print(f"Min hits: {count.min()}") p_count = LookupTable() p_count.plot(count, "Counts") p_count.save(get_plot("d190730_pedestal/count.pdf")) p_mean = LookupTable() p_mean.plot(mean, "Mean") p_mean.save(get_plot("d190730_pedestal/mean.pdf")) p_std = LookupTable() p_std.plot(std, "Stddev") p_std.save(get_plot("d190730_pedestal/std.pdf"))
def calculate_charge_resolution(path): with HDF5Reader(path) as reader: df = reader.read("data") n_channels = np.unique(df['channel']).size n_true = np.unique(df['amplitude']).size print(np.unique(df['amplitude'])) true = np.zeros((n_channels, n_true)) resolution = np.zeros((n_channels, n_true)) average = np.zeros((n_channels, n_true)) std = np.zeros((n_channels, n_true)) for channel, group in df.groupby('channel'): # Calibrate cal_amplitude = 20 assumed_gain = 4 # mV/p.e. cal_charge = group.loc[group['amplitude'] == cal_amplitude, 'charge'].values cal_average = cal_charge.mean() factor = (cal_amplitude / assumed_gain) / cal_average group = group.copy() group['measured'] = group['charge'].values * factor group['true'] = group['amplitude'].values / assumed_gain group['n'] = 1 group['sum'] = np.power(group['measured'] - group['true'], 2) summed = group.groupby(['true']).sum() true_channel = summed.index.values sum_channel = summed['sum'].values n_channel = summed['n'].values true[channel] = true_channel resolution[channel] = np.sqrt( sum_channel / n_channel) / np.abs(true_channel) # resolution[channel] = np.sqrt((sum_channel / n_channel) + true_channel) / np.abs(true_channel) average_df = group.groupby(['true']).mean() average[channel] = average_df['measured'].values std_df = group.groupby(['true']).std() std[channel] = std_df['measured'].values return true, resolution, average, std
def main(): output_path = get_data(f"d190918_alpha/onsky_hillas.h5") path_list = [ glob(get_astri_2019("d2019-05-01_mrk501/*_hillas.h5")), glob(get_astri_2019("d2019-05-02_PG1553+113/*_hillas.h5")), glob(get_astri_2019("d2019-05-02_mrk421/*_hillas.h5")), glob(get_astri_2019("d2019-05-02_mrk501/*_hillas.h5")), glob(get_astri_2019("d2019-05-06_mrk501/*_hillas.h5")), glob(get_astri_2019("d2019-05-09_mrk421/*_hillas.h5")), glob(get_astri_2019("d2019-06-12_mrk421_moonlight/*_hillas.h5")), glob(get_astri_2019("d2019-06-12_mrk501/*_hillas.h5")), glob(get_astri_2019("d2019-06-12_mrk501_moonlight/*_hillas.h5")), ] with HDF5Writer(output_path) as writer: for iinv, hillas_paths in enumerate(path_list): hillas_paths = sort_file_list(hillas_paths) for ifile, input_path in enumerate(hillas_paths): with HDF5Reader(input_path) as reader: if ifile == 0: writer.add_mapping(reader.get_mapping()) writer.add_metadata(**reader.get_metadata()) df = reader.read("data") df_pointing = reader.read("pointing") df_source = reader.read("source") source_name = reader.get_metadata("source")['source_name'] df['iinv'] = iinv df['ifile'] = ifile df['source_ra'] = df_source['source_ra'].values df['source_dec'] = df_source['source_dec'].values df['altitude_raw'] = df_pointing['altitude_raw'].values df['altitude_cor'] = df_pointing['altitude_cor'].values df['azimuth_raw'] = df_pointing['azimuth_raw'].values df['azimuth_cor'] = df_pointing['azimuth_cor'].values df['source_name'] = source_name writer.append(df, 'data')