def query_table(self, query={}, selection={}): """ Get a formatted table of all query results. When multiple results are present for a single value, the best one is picked unless the user specifies a selection. This functionality will be revisited in the future. The output is as a QTable which allows Quantities to be embedded in the results. Parameters ---------- query : dict Query to use in MongoDB query language. Default is an empty dictionary for all results. selection : dict Dictionary with the field value and reference to use for it (otherwise will pick best=1) Returns ------- df : astropy.table.QTable Astropy QTable of results """ results = self.query_db(query=query) # For each entry in result, select best field.value or what the user has specified tab_data = [] for entry in results: out_row = {} for key, val in entry.items(): if not isinstance(val, (list, type(np.array([])))): out_row[key] = val else: # Select best one to return if len(val) > 1: if key in selection.keys(): # If selection listed a key, use the reference information there ind = np.array([x['reference'] for x in val]) == selection[key] else: ind = np.array([x.get('best', 0) for x in val]) == 1 unit = val[ind][0].get('unit') if val[ind][0].get('distribution') is not None: temp_dic = get_values_from_distribution(val[ind][0].get('distribution')) temp_val = temp_dic['value'] else: temp_val = val[ind][0]['value'] out_row[key] = self._store_quantity(temp_val, unit) else: unit = val[0].get('unit') if val[0].get('distribution') is not None: temp_dic = get_values_from_distribution(val[0].get('distribution')) temp_val = temp_dic['value'] else: temp_val = val[0]['value'] out_row[key] = self._store_quantity(temp_val, unit) tab_data.append(out_row) # Convert to QTable temp = pd.DataFrame(tab_data) # use pandas as intermediary format df = QTable.from_pandas(temp) return df
def joinTables1(path, dest): 'Same as before, using if the previous does not work' tables = [] for i, filename in enumerate(os.listdir(path)): df = Table.read(os.path.join(path, filename)).to_pandas() tables.append(df) match = pd.concat(tabels, axis=0, ignore_index=True) del l del df match = QTable.from_pandas(match) match.write(dest)
def read_mc_dl2_to_QTable(filename): """ Read MC DL2 files from lstchain and convert into pyirf internal format - astropy.table.QTable Parameters ---------- filename: path Returns ------- `astropy.table.QTable`, `pyirf.simulations.SimulatedEventsInfo` """ # mapping name_mapping = { "mc_energy": "true_energy", "mc_alt": "true_alt", "mc_az": "true_az", "mc_alt_tel": "pointing_alt", "mc_az_tel": "pointing_az", "gammaness": "gh_score", } unit_mapping = { "true_energy": u.TeV, "reco_energy": u.TeV, "pointing_alt": u.rad, "pointing_az": u.rad, "true_alt": u.rad, "true_az": u.rad, "reco_alt": u.rad, "reco_az": u.rad, } simu_info = read_simu_info_merged_hdf5(filename) pyirf_simu_info = SimulatedEventsInfo( n_showers=simu_info.num_showers * simu_info.shower_reuse, energy_min=simu_info.energy_range_min, energy_max=simu_info.energy_range_max, max_impact=simu_info.max_scatter_range, spectral_index=simu_info.spectral_index, viewcone=simu_info.max_viewcone_radius, ) events = pd.read_hdf( filename, key=dl2_params_lstcam_key).rename(columns=name_mapping) events = QTable.from_pandas(events) for k, v in unit_mapping.items(): events[k] *= v return events, pyirf_simu_info
def table(self): _diam = { 'Name': 'Diameter', 'Value': self.diam, 'Comment': 'Primary mirror diameter', 'Formula': '', 'Symbol': r'$$D_\mathrm{PM}$$' } _cobscuration = { 'Name': 'Central obscuration', 'Value': self.cobscuration, 'Comment': 'pct of primary mirror diameter', 'Formula': '', 'Symbol': r'$$\mathrm{oc}$$' } _sobscuration = { 'Name': 'Strut obscuration', 'Value': self.sobscuration, 'Comment': 'pct of primary mirror area', 'Formula': '', 'Symbol': r'$$\mathrm{os}$$' } _clearApFrac = { 'Name': 'Clear aperture fraction', 'Value': self.clearApFrac, 'Comment': 'After obscuration by secondary mirror and struts', 'Formula': r'$$(1-%s)\times (1-%s^2)$$' % (_sobscuration['Symbol'].replace( '$', ''), _cobscuration['Symbol'].replace('$', '')), 'Symbol': r'$$\mathrm{obsc}$$' } _Aeff = { 'Name': 'Effective aperture', 'Value': self.Aeff, 'Comment': 'After obscuration by secondary mirror and struts', 'Formula': r'$$%s \times%s$$' % (_diam['Symbol'].replace( '$', ''), _clearApFrac['Symbol'].replace('$', '')), 'Symbol': r'$$A_\mathrm{eff}$$' } tab = QTable.from_pandas( pd.DataFrame( [_diam, _cobscuration, _sobscuration, _clearApFrac, _Aeff])) tab.add_index('Name') return tab['Name', 'Value', 'Symbol', 'Formula', 'Comment']
def read_snana_fits_to_sncosmo_table(fname): """Load SNANA formatted data and cast it to sncosmo table Args: fname (str): path + name to PHOT.FITS file Returns: (astropy.Table) table with photometry in sncosmo format """ # load photometry dat = Table.read(fname, format="fits") df_phot = dat.to_pandas() # failsafe if df_phot.MJD.values[-1] == -777.0: df_phot = df_phot.drop(df_phot.index[-1]) if df_phot.MJD.values[0] == -777.0: df_phot = df_phot.drop(df_phot.index[0]) # load header header = Table.read(fname.replace("PHOT", "HEAD"), format="fits") df_header = header.to_pandas() df_header["SNID"] = df_header["SNID"].astype(np.int32) # add SNID to phot for skimming arr_ID = np.zeros(len(df_phot), dtype=np.int32) # New light curves are identified by MJD == -777.0 arr_idx = np.where(df_phot["MJD"].values == -777.0)[0] arr_idx = np.hstack((np.array([0]), arr_idx, np.array([len(df_phot)]))) # Fill in arr_ID for counter in range(1, len(arr_idx)): start, end = arr_idx[counter - 1], arr_idx[counter] # index starts at zero arr_ID[start:end] = df_header.SNID.iloc[counter - 1] df_phot["SNID"] = arr_ID df_phot = df_phot[df_phot.MJD != -777.000] df_tmp = pd.DataFrame() df_tmp["SNID"] = df_phot["SNID"] df_tmp["time"] = df_phot["MJD"] df_tmp["band"] = "des" + df_phot["FLT"].str.decode("utf-8").str.strip(" ") df_tmp["zp"] = df_phot["ZEROPT"] # FLUXCAL is zp = 27.5 df_tmp["flux"] = df_phot["FLUXCAL"] df_tmp["fluxerr"] = df_phot["FLUXCALERR"] df_tmp["zp"] = np.repeat(27.5, len(df_phot)) df_tmp["zpsys"] = np.repeat("ab", len(df_phot)) at_phot = QTable.from_pandas(df_tmp) return at_phot
def read_data_dl2_to_QTable(filename, srcdep_pos=None): """ Read data DL2 files from lstchain and return QTable format Parameters ---------- filename: path to the lstchain DL2 file srcdep_pos: assumed source position for source-dependent analysis Returns ------- `astropy.table.QTable` """ # Mapping name_mapping = { "gammaness": "gh_score", "alt_tel": "pointing_alt", "az_tel": "pointing_az", } unit_mapping = { "reco_energy": u.TeV, "pointing_alt": u.rad, "pointing_az": u.rad, "reco_alt": u.rad, "reco_az": u.rad, "dragon_time": u.s, } # add alpha for source-dependent analysis srcdep_flag = dl2_params_src_dep_lstcam_key in get_dataset_keys(filename) if srcdep_flag: unit_mapping['alpha'] = u.deg data = pd.read_hdf(filename, key=dl2_params_lstcam_key) if srcdep_flag: data_srcdep = get_srcdep_params(filename, srcdep_pos) data = pd.concat([data, data_srcdep], axis=1) data = data.rename(columns=name_mapping) data = QTable.from_pandas(data) # Make the columns as Quantity for k, v in unit_mapping.items(): data[k] *= v return data
def loadCGs(filename=None): ''' Load coronagraph tables ''' def p2f(x): return float(x.strip('%')) / 100 if filename is None: fname = path + '/Inputs/ETC/CGs.csv' else: fname = filename c = pd.read_csv(fname, converters={'BW': p2f}, skip_blank_lines=True) cgs = QTable.from_pandas(c) cgs.add_index('CG') return cgs
def loadPlanets(filename=None): ''' Load planet database ''' if filename is None: fname = path + '/Inputs/ETC/Planets.csv' else: fname = filename pl = pd.read_csv(fname, skip_blank_lines=True) planets = QTable.from_pandas(pl) planets['V'].unit = u.mag planets['A'].unit = u.au planets['DIST'].unit = u.pc planets['Rp / R_J'].unit = u.R_jup planets.add_index(['NAME']) return planets
def loadCoronagraph(name): ''' Load single coronagraph description ''' if not isinstance(name, basestring): log.error('Need a string for the coronagraph name') raise else: coron = pd.read_csv(path + '/Inputs/ETC/' + name + '.csv', skip_blank_lines=True) coron = coron.dropna() corono = QTable.from_pandas(coron) corono.add_index('r(arcsec)') corono['r(arcsec)'].unit = u.arcsec corono['area(sq_arcsec'].unit = u.arcsec**2 return corono
def loadScenarios(filename=None): ''' Load various observing scenarios ''' if filename is None: fname = path + '/Inputs/ETC/Scenarios.csv' else: fname = filename scens = pd.read_csv(fname, skip_blank_lines=True) scenarios = QTable.from_pandas(scens) # two indexes scenarios.add_index('Scenario') scenarios.add_index('Coronagraph') scenarios['Center lam'].unit = u.nm scenarios['t integ, hrs'].unit = u.h scenarios['Years at L2'].unit = u.year scenarios['Ref Dmag'].unit = u.mag return scenarios
def loadSpectra(filename=None): ''' Load stellar spectra database ''' if filename is None: fname = path + '/Inputs/ETC/Spectra.csv' else: fname = filename st = pd.read_csv( fname, skip_blank_lines=True, ) stars = QTable.from_pandas(st) stars['Wavelength (m) '].unit = u.m stars.rename_column('Wavelength (m) ', 'Wavelength') stars['E_ph (J)'].unit = u.J stars.rename_column('E_ph (J)', 'E_ph') cols = ['a0v', 'a5v', 'f5v', 'g0v', 'g5v', 'k0v', 'k5v', 'm0v', 'm5v'] for col in cols: stars[col].unit = u.W / u.m**2 / u.m return stars
def read_data_dl2_to_QTable(filename): """ Read data DL2 files from lstchain and return QTable format Parameters ---------- filename: path to the lstchain DL2 file Returns ------- `astropy.table.QTable` """ # Mapping name_mapping = { "gammaness": "gh_score", "alt_tel": "pointing_alt", "az_tel": "pointing_az", } unit_mapping = { "reco_energy": u.TeV, "pointing_alt": u.rad, "pointing_az": u.rad, "reco_alt": u.rad, "reco_az": u.rad, "dragon_time": u.s, } data = pd.read_hdf(filename, key=dl2_params_lstcam_key).rename(columns=name_mapping) data = QTable.from_pandas(data) # Make the columns as Quantity for k, v in unit_mapping.items(): data[k] *= v return data
def main(outdir, data, source, cuts_file, n_offs, n_jobs): gh_cuts = QTable.read(cuts_file, hdu='GH_CUTS') theta_cuts = QTable.read(cuts_file, hdu='THETA_CUTS_OPT') src = SkyCoord.from_name(source) if n_jobs == -1: n_jobs = cpu_count() with Pool(n_jobs) as pool: results = np.array( pool.starmap( calculation.read_run_calculate_thetas, # applies gh cut [(run, columns, gh_cuts, src, n_offs) for run in data] ), dtype=object ) dfs = results[:,0] ontimes = results[:,1] thetas = results[:,2] df5s = results[:,3] # only necessary for theta_off (not currently included) theta_offs = results[:,4] # not currently included observations = [] index = [] for df, ontime, theta, df5, theta_off in zip(dfs, ontimes, thetas, df5s, theta_offs): tstart = df.dragon_time.min() tstop = df.dragon_time.max() df['theta_on'] = theta.deg df.dropna( subset=['gamma_energy_prediction', 'gammaness', 'source_ra_prediction', 'source_dec_prediction'], inplace=True ) # Apply theta cuts here? #theta_mask = evaluate_binned_cut( # theta, df.gamma_energy_prediction.to_numpy() * u.TeV, theta_cuts, operator.le #) df_events = df[event_map.keys()] df_events.rename(columns=event_map, inplace=True) df_pointings = df[pointing_map.keys()] df_pointings.rename(columns=pointing_map, inplace=True) events = QTable.from_pandas(df_events, units=unit_map) events['RA'] = events['RA'].to(u.deg) events['DEC'] = events['DEC'].to(u.deg) pointings = QTable.from_pandas(df_pointings, units=unit_map) pointings['RA_PNT'] = pointings['RA_PNT'].to(u.deg) pointings['DEC_PNT'] = pointings['DEC_PNT'].to(u.deg) event_header = DEFAULT_HEADER.copy() event_header['HDUCLAS1'] = 'EVENTS' event_header['OBS_ID'] = df.obs_id.iloc[0] event_header['TSTART'] = tstart event_header['TSTOP'] = tstop event_header['ONTIME'] = ontime.to_value(u.s) event_header['LIVETIME'] = event_header['ONTIME'] # Fix this? event_header['DEADC'] = 1.0 event_header['RA_PNT'] = np.mean(pointings['RA_PNT']).to_value(u.deg) event_header['DEC_PNT'] = np.mean(pointings['DEC_PNT']).to_value(u.deg) event_header['EQUINOX'] = 2000.0 event_header['RADECSYS'] = 'ICRS' event_header['ORIGIN'] = 'CTA' event_header['TELESCOP'] = 'LST1' event_header['INSTRUME'] = 'LST1' gtis = QTable( [[tstart] * u.s, [tstop] * u.s], names=('START', 'STOP') ) gti_header = DEFAULT_HEADER.copy() gti_header['MJDREFI'] = 40587 # ref time is this correct? 01.01.1970? gti_header['MJDREFF'] = .0 gti_header['TIMEUNIT'] = 's' gti_header['TIMESYS'] = 'UTC' # ? gti_header['TIMEREF'] = 'TOPOCENTER' # ? pointing_header = gti_header.copy() pointing_header['OBSGEO-L'] = -17.89139 pointing_header['OBSGEO-B'] = 28.76139 pointing_header['OBSGEO-H'] = 2184.0 hdus = [ fits.PrimaryHDU(), fits.BinTableHDU(events, header=event_header, name="EVENTS"), fits.BinTableHDU(pointings, header=pointing_header, name="POINTING"), fits.BinTableHDU(gtis, header=gti_header, name="GTI") ] fits.HDUList(hdus).writeto(f'{outdir}/{df.obs_id.iloc[0]}.fits.gz', overwrite=True) observations.append(( df.obs_id.iloc[0], (df_pointings.RA_PNT.mean(axis=0) * u.rad).to_value(u.deg), (df_pointings.DEC_PNT.mean(axis=0) * u.rad).to_value(u.deg), tstart, tstop, 1.0 )) index.append(( df.obs_id.iloc[0], 'events', 'events', '.', f'{df.obs_id.iloc[0]}.fits.gz', 'EVENTS' )) index.append(( df.obs_id.iloc[0], 'gti', 'gti', '.', f'{df.obs_id.iloc[0]}.fits.gz', 'GTI' )) index.append(( df.obs_id.iloc[0], 'aeff', 'aeff_2d', '.', cuts_file.split('/')[-1], 'EFFECTIVE_AREA' )) index.append(( df.obs_id.iloc[0], 'psf', 'psf_table', '.', cuts_file.split('/')[-1], 'PSF' )) index.append(( df.obs_id.iloc[0], 'edisp', 'edisp_2d', '.', cuts_file.split('/')[-1], 'ENERGY_DISPERSION' )) index.append(( df.obs_id.iloc[0], 'bkg', 'bkg_2d', '.', cuts_file.split('/')[-1], 'BACKGROUND' )) observations_table = QTable( rows=observations, names=['OBS_ID', 'RA_PNT', 'DEC_PNT', 'TSTART', 'TSTOP', 'DEADC'], units=['', 'deg', 'deg', 's', 's', ''] ) obs_header = DEFAULT_HEADER.copy() obs_header['HDUCLAS1'] = 'INDEX' obs_header['HDUCLAS2'] = 'OBS' obs_header['MJDREFI'] = 40587 # ref time is this correct? 01.01.1970? obs_header['MJDREFF'] = .0 obs_header['TIMEUNIT'] = 's' obs_header['TIMESYS'] = 'UTC' # ? obs_header['TIMEREF'] = 'TOPOCENTER' # ? obs_header['OBSGEO-L'] = -17.89139 obs_header['OBSGEO-B'] = 28.76139 obs_header['OBSGEO-H'] = 2184.0 hdus = [ fits.PrimaryHDU(), fits.BinTableHDU(observations_table, header=obs_header, name="OBS_INDEX") ] fits.HDUList(hdus).writeto(f'{outdir}/obs-index.fits.gz', overwrite=True) index_table = QTable( rows=index, names=['OBS_ID', 'HDU_TYPE', 'HDU_CLASS', 'FILE_DIR', 'FILE_NAME', 'HDU_NAME'], ) index_header = DEFAULT_HEADER.copy() index_header['HDUCLAS1'] = 'INDEX' index_header['HDUCLAS2'] = 'HDU' hdus = [ fits.PrimaryHDU(), fits.BinTableHDU(index_table, header=index_header, name="HDU_INDEX"), ] fits.HDUList(hdus).writeto(f'{outdir}/hdu-index.fits.gz', overwrite=True)
def searchCadidates(self, days=15): alerce = AlerceArchive() alerceGoodCandidates = alerce.getCandidates(days) lasairbroker = LasairArchive() lasairGoodCandidates = lasairbroker.getLastDetections(days) # alerceGoodCandidates = [r["result"][target] for target in r["result"]] alerceTable = QTable.from_pandas(alerceGoodCandidates) lasairTable = QTable.from_pandas(lasairGoodCandidates) print("candidates found lasair{0} alerce{1} ".format( len(lasairGoodCandidates), len(alerceGoodCandidates))) meanaler_val = { "ramean": np.nan_to_num( np.concatenate((alerceGoodCandidates["meanra"].values, lasairGoodCandidates["meanra"].values), axis=0)), "decmean": np.nan_to_num( np.concatenate((alerceGoodCandidates["meandec"].values, lasairGoodCandidates["meandec"].values), axis=0)), "maggmax": np.nan_to_num( np.concatenate((alerceGoodCandidates["max_magap_g"].values, lasairGoodCandidates["maggmax"].values), axis=0)), "maggmin": np.nan_to_num( np.concatenate((alerceGoodCandidates["min_magap_g"].values, lasairGoodCandidates["maggmin"].values), axis=0)), "magrmax": np.nan_to_num( np.concatenate((alerceGoodCandidates["max_magap_r"].values, lasairGoodCandidates["magrmax"].values), axis=0)), "magrmin": np.nan_to_num( np.concatenate((alerceGoodCandidates["min_magap_r"].values, lasairGoodCandidates["magrmin"].values), axis=0)), "id": np.concatenate((alerceGoodCandidates["oid"].values, lasairGoodCandidates["oid"].values), axis=0) } bigdata = pd.DataFrame(meanaler_val) bigdata_drop = bigdata.drop_duplicates(subset="id", keep=False) bigtable = bigdata if bigdata_drop.size > 0: bigtable = bigdata_drop table_candidates = QTable(QTable.from_pandas(bigtable), masked=False) return table_candidates, alerceGoodCandidates, lasairGoodCandidates
def check_ltc_residuals(dataframe, ccd, rawy_range=(1, 200), binned=None, filter_select=None, plot_it=True, png_file=None, title=''): """Validation for the LTC correction by means of checking the residuals Parameters ---------- dataframe : dataframe, mandatory the pandas dataframe with the Cu Kalpha fit results (from Michael Smith monitoring run). Produced by `ff_monitoring_work2.ipynb` ccd : int, mandatory the EPIC-pn CCD number (from 1 to 12) rawy_range : list the RAWY range to select, can be (1,200) and then can be (x,x+19) for x in range(1,201,20) binned: float, optional bin the data grouping with binned years, if None, then no binning filter_select : str if not None, then a selection on filter wheel is requested, can be one of 'CalClosed', 'CalMedium', 'CalThick', 'CalThin1', 'Closed', 'Medium', 'Thick', 'Thin1', 'Thin2'. If None, then all are selected. plot_it : bool if set, will plot the results. png_file : str is set, then the plotting results will be saved to this file. title : str Text to append to the end of the plot title, e.g. the version or other comment to apper on the plot title Output ------ output: dict {'ccd': []} Method ------ Modification history -------------------- Created 17 Mar 2021, Ivan Valchanov, XMM SOC """ # ntot = np.count_nonzero((dataframe.ccd == ccd) & (dataframe.rawy0 == rawy_range[0]) & (dataframe.rawy1 == rawy_range[1])) # xtab = select_data(dataframe, ccd, rawy_range=rawy_range, filter_select=filter_select) ntab = len(xtab) xmode = xtab.xmode if (filter_select is not None): print( f"CCD {ccd}, {xmode} mode, filter {filter_select}: filtered {ntab} results out of {ntot}" ) else: print( f"CCD {ccd}, {xmode} mode: filtered {ntab} results out of {ntot}") # #line = dataframe.line # xin = xtab.delta_time residual = (xtab.energy - line0) # in eV residual_err = (xtab.energy_err1 + xtab.energy_err2) / 2.0 # in eV qmean = np.mean(residual) qstd = np.std(residual) xstat = stats.sigma_clipped_stats(residual, sigma=3, maxiters=3) # if (binned is not None): # add those as columns in the dataframe qt = QTable.from_pandas(xtab) qt['residual'] = residual qt['residual_err'] = residual_err # year_bin = np.trunc(qt['delta_time'] / binned) year_run = np.unique(year_bin) #year_bin = np.trunc(qt['delta_time']/binned) dat_grouped = qt.group_by(year_bin) # dat_binned = dat_grouped.groups.aggregate(np.median) dat_binned_std = dat_grouped.groups.aggregate(mad) xin_bin = dat_binned['delta_time'] yin_bin = dat_binned['residual'] yin_bin_err = dat_binned_std['residual'] # # prepare the output # output = [ ccd, xmode, rawy_range[0], rawy_range[1], xstat[0], xstat[2], ntab, year_run.data, xin_bin.data, yin_bin.data, yin_bin_err.data ] # # plotting # if (plot_it): fig, ax = plt.subplots(figsize=(10, 6)) ax.errorbar(xin, residual, yerr=residual_err, fmt='o', label=f'CCDNR {ccd}', zorder=0) if (binned is not None): ax.step(year_run, yin_bin, where='pre', zorder=2, color='cyan', label='Per bin median') #ax.step(xin_bin,yin_bin,where='mid',zorder=2,color='cyan',label='Per bin median') ax.errorbar(xin_bin, yin_bin, yerr=yin_bin_err, fmt='o', color='cyan', zorder=1) ax.axhline(0.0, linestyle='dashed', linewidth=3, color='red', zorder=1) ax.axhline(20.0, linestyle='dotted', linewidth=2, color='red', zorder=1) ax.axhline(-20.0, linestyle='dotted', linewidth=2, color='red', zorder=1) ax.text(0.1, 0.9, fr'mean={qmean:.1f} eV, st.dev.={qstd:.1f} eV', fontsize=14, transform=ax.transAxes) ax.text( 0.1, 0.8, fr'mean={xstat[0]:.1f} eV, st.dev.={xstat[2]:.1f} eV (3-$\sigma$ clipped)', fontsize=14, transform=ax.transAxes) ax.set_xlim((0.0, 22.0)) ax.set_ylim((-100.0, 100.0)) ax.grid(True) ax.legend(loc=3) ax.set_title( f"Cu-Ka data for EPIC-PN CCD{ccd:02}, mode={xmode}, RAWY in [{rawy_range[0]},{rawy_range[1]}], {title}" ) ax.set_ylabel(r"E$_{corr}$ - E$_{lab}$ (eV)") ax.set_xlabel("Time since 2000-01-01 (years)") if (png_file is not None): plt.savefig(png_file, dpi=100) plt.show() plt.close() return output
def checkLastDetections(**kwargs): # try: allrecords=0 collection=current_collection days_ago=15 if "collection" in kwargs.keys() and kwargs["collection"]!="": collection = kwargs["collection"] if "days_ago" in kwargs.keys() and kwargs["days_ago"]!="": days_ago=kwargs["days_ago"] if "IDpipeLine" in kwargs.keys() and kwargs["IDpipeLine"]!="": updatePipeline(kwargs["IDpipeLine"],"checkLastDetections",STATE_RUNNING) logger.info("checkLastDetections:: getting the last ZTF detections from brokers...") lasairarchive = LasairArchive() #coneecto to DATABASE db = MongodbManager() config=Config() dbconfig=config.getDatabase("mongodb") db.setDatabase(dbconfig["dbname"]) db.setCollection(collection) #Get last candidates and update previews detection and light curves bestCandidates = BestCandidates() table_candidates, alerceDF, lasairDF = bestCandidates.searchCadidates(days_ago) #check if the new candidates is already in DB #get all zft id in and array to validate if exist into ddatabase and filter by listcandidates=table_candidates["id"] filter={"oid":{"$in":listcandidates.data.tolist()}} projection={"oid":1 ,"lastmjd":1 ,"last_update":1} current_data = db.getData(filter=filter, projection=projection) for remove_data in current_data: oid=remove_data["oid"] print("get info for ",oid) table_candidates.remove_rows(table_candidates["id"] == oid) #get desi photoz dataarchive = SussexArchive() desi_targetsvo, desi_targetstable = dataarchive.getDesiPhotoZfromTable(table_candidates) alerceTable = QTable.from_pandas(alerceDF) lasairTable = QTable.from_pandas(lasairDF) alerceTable.rename_column("oid","id") lasairTable.rename_column("oid", "id") alerceTable["id"] = alerceTable["id"].astype(str) lasairTable["id"] = lasairTable["id"].astype(str) desi_targetstable["id"] =desi_targetstable["id"].astype(str) desi_targetstable["desidec"].mask = False desi_targetstable["desira"].mask = False #calc separation desi source ra_ref = desi_targetstable["ramean"].tolist() dec_ref = desi_targetstable["decmean"].tolist() cref = SkyCoord(ra_ref, dec_ref, frame='icrs', unit='deg') ra_desi = desi_targetstable["desira"].tolist() dec_desi = desi_targetstable["desidec"].tolist() c1 = SkyCoord(ra_desi, dec_desi, frame='icrs', unit='deg') desi_distance = cref.separation(c1).arcsec desi_targetstable["separation"] = desi_distance #merge all table in one json to save in mongo desi_targetstable = Table(desi_targetstable, masked=False) alerceTable = Table(alerceTable, masked=False) lasairTable = Table(lasairTable, masked=False) alerceTable["broker"] = "alerce" lasairTable["broker"] = "lasair" update_alerce_table = join(alerceTable, lasairTable, join_type='outer', keys='id') merge_table = join(update_alerce_table, desi_targetstable, join_type='outer', keys='id') merge_table["desiid"] = merge_table["desiid"].astype(str) merge_table["field"] = merge_table["field"].astype(str) lastItems= merge_table.to_pandas() newItems = lastItems.fillna('', axis=1) dic_result = newItems.to_dict('records') newCandidates=0 logger.info("checkLastDetections:: Ingested {0} candidates".format(str(len(dic_result)))) allrecords=len(dic_result) for index,row in enumerate(dic_result): id=row["id"] print("saving candidate",id) row["comments"]={} row["snh_score"] = 0.0 if row["broker_1"] != "": #alerce #row["pclassearly"]=row["pclassearly_1"] if row["broker_2"]!="": row["broker"]=row["broker_1"]+"/"+row["broker_2"] else: row["broker"] = row["broker_1"] row["meanra"]=row["meanra_1"] row["meandec"]=row["meandec_1"] row["lastmjd"]=row["lastmjd_1"] else: #lasair #row["pclassearly"] = row["pclassearly_2"] row["broker"] = row["broker_2"] row["meanra"] = row["meanra_2"] row["meandec"] = row["meandec_2"] row["lastmjd"] = row["lastmjd_2"] try: #remove duplicate fields #del row["pclassearly_1"] #del row["pclassearly_2"] del row["broker_1"] del row["broker_2"] del row["meanra_1"] del row["meandec_1"] del row["meanra_2"] del row["meandec_2"] del row["lastmjd_2"] del row["lastmjd_1"] except KeyError as er: print("key error",er,id) # check if already exist this candidate, if exist update light curve and run check list to alerts currentdata = db.getData(filter={"id": id}, projection={"nobs": 1, "last_update": 1, "id": 1}) now = datetime.now().timestamp() rowupdated={} if len(currentdata) > 0: currentdata = currentdata[0] days_from_update = ((now - float(currentdata["last_update"])) / 3600) / 24 if days_from_update < 0.6: print("last detections is the same, not getting enough to services update classify",id) logger.info("checkLastDetections:: {0} last detections is the same, not getting enough to services update classify".format(id)) continue classification = getClassification(id) #peak = lasairarchive.getPeakLightCurve(classification["light_curve"]["candidates"]) rowupdated["ra"] = row["meanra"] rowupdated["dec"] = row["meandec"] rowupdated["lasair_clas"]=classification["lasair_clas"] rowupdated["alerce_clas"]=classification["alerce_clas"] rowupdated["alerce_early_class"] = classification["alerce_early_class"] rowupdated["alerce_late_class"] = classification["alerce_late_class"] rowupdated["crossmatch"]={"lasair":classification["light_curve"]["crossmatches"],"check":False} rowupdated["lightcurve"] = classification["light_curve"]["candidates"] rowupdated["report"] = row rowupdated["broker"] = row["broker"] rowupdated["nobs"] = row["nobs"] rowupdated["lastmjd"] = row["lastmjd"] rowupdated["sigmara"] = row["sigmara"] rowupdated["sigmadec"] = row["sigmadec"] rowupdated["last_magpsf_g"] = row["last_magpsf_g"] rowupdated["last_magpsf_r"] = row["last_magpsf_r"] rowupdated["first_magpsf_g"] = row["first_magpsf_g"] rowupdated["first_magpsf_r"] = row["first_magpsf_r"] rowupdated["sigma_magpsf_g"] = row["sigma_magpsf_g"] rowupdated["sigma_magpsf_r"] = row["sigma_magpsf_r"] rowupdated["max_magpsf_g"] = row["max_magpsf_g"] rowupdated["max_magpsf_r"] = row["max_magpsf_r"] rowupdated["id"] = row["id"] #check if already exist this candidate, if exist update light curve and run check list to alerts currentdata=db.getData(filter={"id":id},projection={"nobs":1,"last_update":1,"id":1}) now = datetime.now().timestamp() if len(currentdata)>0 : #update current data try: if currentdata[0]["nobs"] < rowupdated["nobs"]: peak = lasairarchive.getPeakLightCurve(classification["light_curve"]["candidates"]) rowupdated["lightpeak"] = peak update_query={"last_update":now,"lightcurve":rowupdated["lightcurve"],"lightpeak":peak,"lasair_clas":rowupdated["lasair_clas"],"alerce_clas":rowupdated["alerce_clas"],"nobs":rowupdated["nobs"],"state":"updated"} update_id = db.update(filter={"id":id}, query={"$set":update_query}) print("updated source",id,update_id.raw_result) else: print("last detections is the same, not getting enough to services update classify",id) except Exception as err: print("Error updated",id,currentdata[0]["nobs"],rowupdated["nobs"]) logger.error("checkLastDetections:: {0} Error updated..".format(str(id))) else: peak = lasairarchive.getPeakLightCurve(classification["light_curve"]["candidates"]) rowupdated["lightpeak"] = peak #insert new candidate print("save new candidate") rowupdated["state"]="new" rowupdated["last_update"] = now db.saveData(rowupdated) logger.info("checkLastDetections:: {0} Saved candidate with {1} observations".format(id,rowupdated["nobs"])) newCandidates+=1 logger.info("checkLastDetections:: {0} candidates stored..".format(str(len(dic_result)))) logger.info("checkLastDetections:: alerce table detections {0}".format(str(len(alerceTable)))) logger.info("checkLastDetections:: lasair table detections {0}".format(str(len(lasairTable)))) logger.info("checkLastDetections:: desi detections {0}".format(str(len(desi_targetstable)))) logger.info("checkLastDetections:: new Candidates {0}".format(str(newCandidates))) db.saveData(data={"date":now,"newcandidates":newCandidates,"allrecords":allrecords,"alerce_records":len(alerceTable),"lasair_records":len(lasairTable),"desi_matchs":len(desi_targetstable),"process":"lastdetections"},collection="tasks") if "IDpipeLine" in kwargs.keys() and kwargs["IDpipeLine"]!="": updatePipeline(kwargs["IDpipeLine"],"checkLastDetections",STATE_COMPLETED)
def read_mc_dl2_to_QTable(filename): """ Read MC DL2 files from lstchain and convert into pyirf internal format - astropy.table.QTable Parameters ---------- filename: path Returns ------- `astropy.table.QTable`, `pyirf.simulations.SimulatedEventsInfo` """ # mapping name_mapping = { "mc_energy": "true_energy", "mc_alt": "true_alt", "mc_az": "true_az", "mc_alt_tel": "pointing_alt", "mc_az_tel": "pointing_az", "gammaness": "gh_score", } unit_mapping = { "true_energy": u.TeV, "reco_energy": u.TeV, "pointing_alt": u.rad, "pointing_az": u.rad, "true_alt": u.rad, "true_az": u.rad, "reco_alt": u.rad, "reco_az": u.rad, } # add alpha for source-dependent analysis srcdep_flag = dl2_params_src_dep_lstcam_key in get_dataset_keys(filename) if srcdep_flag: unit_mapping['alpha'] = u.deg simu_info = read_simu_info_merged_hdf5(filename) pyirf_simu_info = SimulatedEventsInfo( n_showers=simu_info.num_showers * simu_info.shower_reuse, energy_min=simu_info.energy_range_min, energy_max=simu_info.energy_range_max, max_impact=simu_info.max_scatter_range, spectral_index=simu_info.spectral_index, viewcone=simu_info.max_viewcone_radius, ) events = pd.read_hdf(filename, key=dl2_params_lstcam_key) if srcdep_flag: events_srcdep = get_srcdep_params(filename, 'on') events = pd.concat([events, events_srcdep], axis=1) events = events.rename(columns=name_mapping) events = QTable.from_pandas(events) for k, v in unit_mapping.items(): events[k] *= v return events, pyirf_simu_info
def query_function(params): con = ea.connect('desdr') DF = con.query_to_pandas(query_string(params)) DF = QTable.from_pandas(DF) return DF