def matching(master, cat, masteridskey=None, angular=False, radius=1.5, masked=False): """Function to match stars between frames. """ if masteridskey is None: masterids = np.arange(len(master)) master['masterindex'] = masterids idkey = 'masterindex' else: idkey = masteridskey if angular: masterRaDec = np.empty((len(master), 2), dtype=np.float64) try: masterRaDec[:, 0] = master['RA'] masterRaDec[:, 1] = master['Dec'] except KeyError: masterRaDec[:, 0] = master['ra'] masterRaDec[:, 1] = master['dec'] imRaDec = np.empty((len(cat), 2), dtype=np.float64) try: imRaDec[:, 0] = cat['RA'] imRaDec[:, 1] = cat['Dec'] except KeyError: imRaDec[:, 0] = cat['ra'] imRaDec[:, 1] = cat['dec'] radius2 = radius/3600. dist, ind = cx.crossmatch_angular(masterRaDec, imRaDec, max_distance=radius2/2.) dist_, ind_ = cx.crossmatch_angular(imRaDec, masterRaDec, max_distance=radius2/2.) else: masterXY = np.empty((len(master), 2), dtype=np.float64) masterXY[:, 0] = master['x'] masterXY[:, 1] = master['y'] imXY = np.empty((len(cat), 2), dtype=np.float64) imXY[:, 0] = cat['x'] imXY[:, 1] = cat['y'] dist, ind = cx.crossmatch(masterXY, imXY, max_distance=radius) dist_, ind_ = cx.crossmatch(imXY, masterXY, max_distance=radius) IDs = np.zeros_like(ind_) - 13133 for i in range(len(ind_)): if dist_[i] != np.inf: # dist_o = dist_[i] ind_o = ind_[i] if dist[ind_o] != np.inf: # dist_s = dist[ind_o] ind_s = ind[ind_o] if ind_s == i: IDs[i] = master[idkey][ind_o] # print((len(IDs), len(ind_), len(ind))) # print(("Matching result:: IDs > 0. => {}".format(sum(IDs > 0)))) if masked: mask = IDs > 0 return(IDs, mask) return(IDs)
def _matching( master, cat, masteridskey=None, angular=False, radius=1.5, masked=False ): """ Function to match stars between frames. """ if masteridskey is None: masterids = np.arange(len(master)) master["masterindex"] = masterids idkey = "masterindex" else: idkey = masteridskey if angular: masterRaDec = np.empty((len(master), 2), dtype=np.float64) try: masterRaDec[:, 0] = master["RA"] masterRaDec[:, 1] = master["Dec"] except KeyError: masterRaDec[:, 0] = master["ra"] masterRaDec[:, 1] = master["dec"] imRaDec = np.empty((len(cat), 2), dtype=np.float64) try: imRaDec[:, 0] = cat["RA"] imRaDec[:, 1] = cat["Dec"] except KeyError: imRaDec[:, 0] = cat["ra"] imRaDec[:, 1] = cat["dec"] radius2 = radius / 3600.0 dist, ind = cx.crossmatch_angular( masterRaDec, imRaDec, max_distance=radius2 / 2.0 ) dist_, ind_ = cx.crossmatch_angular( imRaDec, masterRaDec, max_distance=radius2 / 2.0 ) else: masterXY = np.empty((len(master), 2), dtype=np.float64) masterXY[:, 0] = master["x"] masterXY[:, 1] = master["y"] imXY = np.empty((len(cat), 2), dtype=np.float64) imXY[:, 0] = cat["x"] imXY[:, 1] = cat["y"] dist, ind = cx.crossmatch(masterXY, imXY, max_distance=radius) dist_, ind_ = cx.crossmatch(imXY, masterXY, max_distance=radius) IDs = np.zeros_like(ind_) - 13133 for i in range(len(ind_)): if dist_[i] != np.inf: ind_o = ind_[i] if dist[ind_o] != np.inf: ind_s = ind[ind_o] if ind_s == i: IDs[i] = master[idkey][ind_o] if masked: mask = IDs > 0 return (IDs, mask) return IDs
def do_crossmatch(ra_im, dec_im, ra_std, dec_std): ''' Do the crossmatching params: ra,dec of data set 1 and 2 returns: index of the match ''' # get imaging data try: imX = np.empty((len(ra_im), 2), dtype=np.float64) except TypeError: imX = np.empty((1, 2), dtype=np.float64) imX[:, 0] = ra_im imX[:, 1] = dec_im # get standard stars stX = np.empty((len(ra_std), 2), dtype=np.float64) stX[:, 0] = ra_std stX[:, 1] = dec_std # crossmatch catalogs max_radius = 1.5 / 3600 # 1 arcsec dist, ind_im = crossmatch_angular(imX, stX, max_radius) match_im = ~np.isinf(dist) return ind_im, match_im
def crossmatch_skybot(sources, moving_objects, radius=5): """ crossmatch list of detected sources with list of moving objects in this field from skybot parameters: sources, moving_objects, radius: sources: astropy.table containing list of unknown sources detected moving_objects: astropy.table containing list of moving objects from skybot in the same field of view radius in arcsecond returns: astropy.table object NOT WORKING WITH PYTHON2.7, seems ok WITH PYTHON3 """ #sources=ascii.read('/home/corre/codes/Tests/%s.fits.oc' % filename) #moving_objects=ascii.read('/home/corre/codes/gmadet/gmadet/moving_objects.dat') cat1 = np.empty((len(sources), 2), dtype=np.float64) cat2 = np.empty((len(moving_objects), 2), dtype=np.float64) cat1[:, 0] = sources['_RAJ2000'] cat1[:, 1] = sources['_DEJ2000'] cat2[:, 0] = moving_objects['RA'] cat2[:, 1] = moving_objects['DEC'] dist, ind = crossmatch_angular(cat1, cat2, radius / 3600) match = ~np.isinf(dist) #print (match) dist_match = dist[match] # Convert in arcseconds dist_match *= 3600 #print (dist_match) if dist_match: mov_match = sources[ind[np.unique(match)]] print(mov_match) candidates = sources[match == False] return candidates
def get_assigned_targets(cat, ps_cat): ra_e = cat[:,0] dec_e = cat[:,1] X_e = np.empty((ra_e.size, 2), dtype = np.float64) X_e[:, 0] = ra_e X_e[:, 1] = dec_e data_m = ps_cat ra_m = data_m[:,0] dec_m = data_m[:,1] X_m = np.empty((ra_m.size, 2), dtype = np.float64) X_m[:, 0] = ra_m X_m[:, 1] = dec_m print "Out of:", ra_m.size max_radius = 0.3 / 3600 # 0.01 arcsec dist, ind = crossmatch_angular(X_m, X_e, max_radius) idx = (~np.isinf(dist)) return data_m[idx], dist, ind
def crossmatch_skybot(sources, moving_objects, radius=10): """ crossmatch list of detected sources with list of moving objects in this field using skybot parameters: sources, moving_objects, radius: sources: astropy.table containing list of unknown sources detected moving_objects: astropy.table containing list of moving objects from skybot in the same field of view radius in arcsecond returns: astropy.table object NOT WORKING WITH PYTHON2.7, seems ok WITH PYTHON3 """ cat1 = np.empty((len(sources), 2), dtype=np.float64) cat2 = np.empty((len(moving_objects), 2), dtype=np.float64) cat1[:, 0] = sources["_RAJ2000"] cat1[:, 1] = sources["_DEJ2000"] cat2[:, 0] = moving_objects["RA"] cat2[:, 1] = moving_objects["DEC"] dist, ind = crossmatch_angular(cat1, cat2, radius / 3600) match = ~np.isinf(dist) dist_match = dist[match] # Convert in arcseconds dist_match *= 3600 if len(dist_match) > 0: mov_match = moving_objects[ind[match]] movingObjMatch_list = [] movingObjSep_list = [] movingObjName = [] for j in range(len(mov_match)): movingObjMatch_list.append("Y") movingObjSep_list.append(dist_match[j]) movingObjName.append(mov_match["Name"][j]) sources["movingObjMatch"][match] = movingObjMatch_list sources["movingObjSep"][match] = movingObjSep_list sources["movingObjName"][match] = movingObjName return sources
#ra1 = [] #for i in ra: #ra1.append(i/15) first_cat = np.empty((len(ra1), 2), dtype=np.float64) first_cat[:, 0] = ra1 first_cat[:, 1] = dec1 new_cat = np.empty((int(len(ra2)), 2), dtype=np.float64) new_cat[:, 0] = ra2 new_cat[:, 1] = dec2 print(print(new_cat)) print(print(first_cat)) max_radius = 1 / 3600 #.5 arc second dist_between, ind_row = crossmatch_angular(first_cat, new_cat, max_radius) match = ~np.isinf(dist_between) match_table = Table() match_table['matched_true_false'] = match match_table['match_ID'] = ind_row match_table['matched_firstcat_RA'] = first_cat[:, 0] match_table['matched_firstcat_DEC'] = first_cat[:, 1] new_cat_line_ID = range(len(match)) #print('LENGTH OF MATCH: ' + str(len(match))) match_table['match_table_false_id'] = new_cat_line_ID new_cat_match_true = [] new_cat_match_false = [] new_cat_ID_true = []
# get standard stars ##HASH catalog # standards_data = fetch_sdss_S82standards() ## HASH PN stara = ascii.read("ra-pne-HASH.txt") stadec = ascii.read("dec-pne-HASH.txt") c2 = SkyCoord(stara, stadec, unit=(u.hourangle, u.deg)) RA1 = c2.ra.degree.reshape(len(c2.ra.degree), ) DEC1 = c2.dec.degree.reshape(len(c2.dec.degree), ) stX = np.empty((len(c2), 2), dtype=np.float64) stX[:, 0] = RA1 stX[:, 1] = DEC1 #HAST cata # crossmatch catalogs max_radius = 3600. / 3600 # 1 arcsec dist, ind = crossmatch_angular(imX, stX, max_radius) #dist, ind = crossmatch_angular(c1, c2, max_radius) match = ~np.isinf(dist) dist_match = dist[match] dist_match *= 3600 print('Number with match:', np.sum(match)) print('PN:', imX[match]) sys.exit() ax = plt.axes() hist(dist_match, bins='knuth', ax=ax, histtype='stepfilled', ec='k', fc='#AAAAAA')
open("{}.csv".format(trackletPaths[midIdx].split('.')[0]), 'r')) notNAN = np.asarray([ num for num, (v, ve) in enumerate( zip(apass['Johnson_V'].values, apass['Verr'].values)) if (str(v) != 'nan') and (str(ve) != 'nan') ]) apassRA = apass['radeg'].values[notNAN] apassDec = apass['decdeg'].values[notNAN] apassMag = apass['Johnson_V'].values[notNAN] apassMagerr = abs(apass['Verr'].values[notNAN]) # Cross-match the sources between SExtractor catalogs and APASS catalogs. apassTable = np.column_stack((apassRA, apassDec)) sexTable = np.column_stack((sexRA, sexDec)) dist, ind = crossmatch_angular(apassTable, sexTable, coaddFWHM * 1.22 * 2 / 3600) match = ~np.isinf(dist) sexMag_mat = sexMag[ind[match]] sexMagerr_mat = sexMagerr[ind[match]] # Calculate the zero point and its error. zp = np.mean(apassMag[match] - sexMag_mat) zperr = (np.sum(apassMagerr[match]**2) + np.sum(sexMagerr_mat**2))**(0.5) / len(sexMag_mat) # Store the zero point as new headers. hdu = fits.open(stkPath, mode='append') hdu[0].header.set("S-ZEROPT", str(zp), " sexInstMag+ZP=apassCalibMag") hdu[0].header.set("S-ZEROER", str(zperr), " sexZPerr+apassZPerr")
def crossmatch(a, b, keys, name, sname, arcsec=5., bijective =False, plot=False, union=False): ''' Match targets from catalog c2 into reference catalog c1 in a cone with a maximum search radius. Parameters: ------------ a : DataFrame b : DataFrame keys : list with length 2 Contains extensions into RA, Dec, and PMem in a and b, respectively. arcsec : 5. or float Maximum search radius in arcsec. plot : False or bool Export a plot of matching distances. bijective : True or bool Force 1-on-1 matching Return: -------- DataFrame of length c1 with c2 matched if possible. Otherwise the row is filled with the last entry in c2 and the distance is filled with inf. ''' c1 = copy(a).astype(float) c2 = copy(b).astype(float) # get imaging data imX = np.empty((c1.shape[0], 2), dtype=np.float64) imX[:, 0] = c1['RAJ2000_{}'.format(keys[0])].values imX[:, 1] = c1['DEJ2000_{}'.format(keys[0])].values # get standard stars stX = np.empty((c2.shape[0], 2), dtype=np.float64) stX[:, 0] = c2['RAJ2000_{}'.format(keys[1])].values stX[:, 1] = c2['DEJ2000_{}'.format(keys[1])].values # crossmatch catalogs max_radius = arcsec / 3600 dist, ind = crossmatch_angular(imX, stX, max_radius) match = ~np.isinf(dist)#indices into imX #matching distances dist_match = dist[match] dist_match *= 3600 #merge data frames ind = list(filter(lambda x: x != c2.shape[0], ind))#throw out fill values for colname in c2.columns.values: c1[colname] = np.nan c1.iloc[match, c1.columns.get_loc(colname)] = c2.iloc[ind, c2.columns.get_loc(colname)].tolist() #add distances to results distcol = 'dist_{}_{}'.format(keys[0], keys[1]) c1[distcol] = np.nan c1.iloc[match, c1.columns.get_loc(distcol)] = dist_match #Force 1-on-1 matching if requested if bijective == True: c1 = select_min_dist(c1, keys) #append the rest of c2 if required if union == True: allid = list(range(c2.shape[0])) ids, counts = np.unique(allid+ind, return_counts=True) unmatched_ids = ids[np.where(counts==1)] c1 = c1.append(c2.iloc[unmatched_ids,:],ignore_index=True, sort=False) #plotting optional if plot == True: plot_radius_of_match(dist_match,name, sname, keys, arcsec, match, imX, stX) return c1
found_clusters = get_clusters(galaxy_data, p, z, h, minz, linking, 'found_clusters.txt') f_ra = [x[0] for x in found_clusters if str(x) != 'nan'] f_dec = [x[1] for x in found_clusters if str(x) != 'nan'] f_list = np.transpose([f_ra, f_dec]) n_ra = [x[0] for x in natural_clusters] n_dec = [x[1] for x in natural_clusters] n_list = np.transpose([n_ra, n_dec]) c_list = np.transpose(cluster_locs) tol = 0.1 #tolerance value for matches (degrees) l_dist, l_ind = crossmatch_angular(f_list, c_list, tol) l_matches[run] = len(l_dist[~np.isinf(l_dist)]) n_dist, n_ind = crossmatch_angular(f_list, n_list, tol) n_matches[run] = len(n_dist[~np.isinf(n_dist)]) fc[run] = len(found_clusters) print('done' + str(run + 1)) run += 1 nm_avg = np.mean(n_matches) nm_std = np.std(n_matches) lm_avg = np.mean(l_matches) lm_std = np.std(l_matches)
tmass= pd.read_csv("/users/alex/Data/tmassF.min.db") print("imported 2mass") tmass = tmass.apply(lambda x: pd.to_numeric(x, errors='coerce')).dropna() print("fixed 2mass") vvv = pd.read_csv("/users/alex/Data/vvvEQUF.min.db") print("imported vvv") vvv = vvv.apply(lambda x: pd.to_numeric(x, errors='coerce')).dropna() print("fixed vvv") X1 = vvv[["RA2000", "DEC2000"]].to_numpy() X2 = tmass[["ra", "dec"]].to_numpy() dist, ind = crossmatch_angular(X1, X2, max_distance=(1.0/36000)) print("Done crossmatching") vvv['DIST'] = dist vvv["IND"] = ind vvv = vvv[vvv['DIST']!=np.inf] print(vvv) print("Max of gen ind: ", np.max(vvv["IND"])) print("Length of tmass: ", len(tmass['k_m'])) vvv["K2MASS"] = np.array([tmass['k_m'][i] for i in vvv["IND"]]) print("Subscripted K2MASS vals") del tmass
# Set arrays of MPA-JHU SFG SDSSX = np.empty((len(table_SFG_sdss), 2), dtype=np.float64) SDSSX[:, 0] = table_SFG_sdss['RA'] SDSSX[:, 1] = table_SFG_sdss['DEC'] # Set arrays of MaNGA DR14 Table_drp = Table.read('/home/qliu/MaNGA/drpall-v2_1_2.fits') table_drp = Table(Table_drp.columns["plateifu", "objra", "objdec"]).to_pandas() MANGAX = np.empty((len(table_drp), 2), dtype=np.float64) MANGAX[:, 0] = table_drp['objra'] MANGAX[:, 1] = table_drp['objdec'] # Crossmatch catalogs print "Start Crossmatching..." max_radius = 1. / 3600 # 1 arcsec dist, ind = crossmatch_angular(SDSSX, MANGAX, max_radius) match = ~np.isinf(dist) dist_match = dist[match] dist_match *= 3600 # Plot crossmatching results ax = plt.axes() hist(dist_match, bins='knuth', ax=ax, histtype='stepfilled', ec='k', fc='#AAAAAA') ax.set_xlabel('radius of match (arcsec)') ax.set_ylabel('N(r, r+dr)')
def check(): ''' Check if any median separations with the reference stars is an outlier ''' science_path = os.path.join(os.getcwd(), 'science-imaging') nights = [ n for n in sorted(os.listdir(science_path)) if os.path.isdir(os.path.join(science_path, n)) ] outliers = {} # Median outliers notsolved = {} # Not solved by Astrometry medianseps = {} # Quartiles of separations # Check outliers inside each night using a boxplot for n in tqdm(nights, desc='Checking calibrations'): img_path = os.path.join(science_path, n) name = [] mediansep = [] notsol = [] sci_path = [ sci for sci in sorted(os.listdir(img_path)) if os.path.isdir(os.path.join(img_path, sci)) and sci.startswith('science-imaging') ] for science in sci_path: folder = os.path.join(science_path, n, science) if not os.path.isfile(os.path.join(folder, 'xxx.new')): notsol.append(science) continue else: dat_df = pd.read_csv(os.path.join(folder, 'full_1.cat'), delim_whitespace=True, header=None, comment='#') ref_data = dat_df[dat_df[1] == 0] ref = np.empty((len(ref_data), 2), dtype=np.float64) ref[:, 0] = ref_data[10] ref[:, 1] = ref_data[11] sci_data = dat_df[dat_df[1] == 1] sci = np.empty((len(sci_data), 2), dtype=np.float64) sci[:, 0] = sci_data[10] sci[:, 1] = sci_data[11] max_radius = 1. / 3600 # 1 arcsec dist, ind = crossmatch_angular(sci, ref, max_radius) match = ~np.isinf(dist) dist_match = dist[match] dist_match *= 3600 name.append(science) with warnings.catch_warnings(): warnings.simplefilter("ignore", category=RuntimeWarning) mediansep.append(np.median(dist_match)) sep = pd.DataFrame([name, mediansep]).T out = ['NaN ' + str(i) for i in list(sep[0][np.isnan(list(sep[1]))])] sep_aux = sep.dropna().reset_index(drop=True) sep_aux[1] = [float("%.50f" % elem) for elem in list(sep_aux[1])] fl = plt.boxplot(sep_aux[1], showfliers=True) plt.close() fl_data = [i.get_data() for i in fl['fliers']] for i in [float("%.50f" % elem) for elem in fl_data[0][1]]: if i > fl['medians'][0].get_ydata()[0]: out.append( str(i) + ' ' + str(sep_aux[0][list(sep_aux[1]).index(i)])) outliers[n] = out notsolved[n] = notsol medianseps[n] = 'Median_of_seps: ' + str( fl['medians'][0].get_ydata()[0]) + ' | Q1: ' + str( fl['whiskers'][0].get_ydata()[0]) + ' | Q3: ' + str( fl['whiskers'][1].get_ydata()[0]) if any(notsolved.values()): print("\n Astrometry couldn't find astrometric solution for:") NS = pd.DataFrame(notsolved.items(), columns=['Night', 'Images']) NS.to_csv(os.path.join(os.getcwd(), "list_notsolved.csv"), index=False, header=True) pprint(notsolved, width=1) else: print(" - Astrometry solved all the images.") if any(outliers.values()): print('\n Bad astrometric solutions found in:') OL = pd.DataFrame(outliers.items(), columns=['Night', 'Images']) OL.to_csv(os.path.join(os.getcwd(), "list_outliers.csv"), index=False, header=True) pprint(outliers, width=1) else: print(" - There are no outliers.") pprint(medianseps, width=1) return [outliers, notsolved, medianseps]
def correct_photom_SkyMapper(year, month, field, exp_time, verbose=False, debugmode=False): input_path = '/fred/oz100/NOAO_archive/archive_NOAO_data/data_outputs/'+year+'/'+month+'/'+field+'/g_band/single/*/SE_cats/' input_cats = '/fred/oz100/NOAO_archive/archive_NOAO_data/scripts/correct_photom/skymapper/shortlisted_stars_for_photom/'+field+'_SM_NEW.csv_SHORTLISTED.ascii' print('\n#########################') print('# Corrections to SkyMapper have started #') print('#########################\n') path_list = glob.glob(input_path) for i in path_list: print(i) av_correction = [] odd_differences = [] filenames_good = [] filenames_odd = [] master_filenames = [] num_sources_ccd = [] master_mean_diff = [] master_median_diff = [] num_photom_sources = [] for filename in os.listdir(i): #av_correction = np.zeros(len(os.listdir(i))) #print(av_correction) #print( filename) #filenames_for_av = [] #filenames_for_av.append(filename) #num_sources_ccd = [] if filename.endswith('.cat'): MAG_APER, MAGERR_APER, MAG_AUTO, MAGERR_AUTO, XPEAK_IMAGE, YPEAK_IMAGE, X_IMAGE, Y_IMAGE, ALPHA_J2000, DELTA_J2000 = np.loadtxt(i + filename, unpack = True) master_filenames.append(filename) zp_exp_correction = 2.5*np.log10(exp_time) ccd_SE_cat = Table() num_sources_ccd.append(len(MAG_APER)) print('Filename:' + str(filename)) print('Number sources ccd: '+ str(len(MAG_APER))) ccd_SE_cat['mag_aper'] = MAG_APER + zp_exp_correction ccd_SE_cat['magerr_aper'] = MAGERR_APER ccd_SE_cat['mag_auto'] = MAG_AUTO + zp_exp_correction ccd_SE_cat['mag_auto_err'] = MAGERR_AUTO ccd_SE_cat['xpeak_image'] = XPEAK_IMAGE ccd_SE_cat['ypeak_image'] = YPEAK_IMAGE ccd_SE_cat['x_image'] = X_IMAGE ccd_SE_cat['y_image'] = Y_IMAGE ccd_SE_cat['RA'] = ALPHA_J2000 ccd_SE_cat['DEC'] = DELTA_J2000 DWF_X = np.empty((len(ccd_SE_cat), 2), dtype=np.float64) DWF_X[:, 0] = ALPHA_J2000 DWF_X[:, 1] = DELTA_J2000 ra, dec, ngood, class_star, g_psf, e_g_psf, r_psf, e_r_psf, i_psf, e_i_psf, z_psf, e_z_psf = np.loadtxt(input_cats , unpack = True, skiprows = 1) SM_X = np.empty((len(ra), 6), dtype=np.float64) SM_X[:, 0] = ra SM_X[:, 1] = dec SM_X[:, 2] = g_psf SM_X[:, 3] = r_psf SM_X[:, 4] = i_psf SM_X[:, 5] = z_psf os.chdir(i) os.chdir("..") filename_directory = os.path.abspath(os.curdir) #plt.plot(DWF_X[:, 0], DWF_X[:, 1], 'b') #plt.plot(ra, dec) #plt.show() #break photom_correction_path = filename_directory + '/photom_correction_files/' #print(photom_correction_path) final_source_cat_path = filename_directory + '/final_source_cats/' #print(final_source_cat_path) photom_correction_path_images = photom_correction_path + 'skymapper_checkphotom_plots/' #print(photom_correction_path_images) if not os.path.exists(photom_correction_path): os.makedirs(photom_correction_path, 0o755) else: pass if not os.path.exists(final_source_cat_path): os.makedirs(final_source_cat_path, 0o755) else: pass if not os.path.exists(photom_correction_path_images): os.makedirs(photom_correction_path_images, 0o755) else: pass max_radius = 1./3600 #1 arc second dist_between, ind_row = crossmatch_angular(DWF_X, SM_X, max_radius) match = ~np.isinf(dist_between) if len(match) != 0: match_table = Table() match_table['matched_true_false'] = match match_table['matched_ID'] = ind_row match_table['matched_DWF_data_gmag'] = ccd_SE_cat['mag_auto'] match_table['matched_DWF_RA'] = ccd_SE_cat['RA'] match_table['matched_DWF_DEC'] = ccd_SE_cat['DEC'] SM_match_true = [] SM_row_matched = [] DWF_g_mags_matched = [] DWF_g_mags_error_matched = [] DWF_obs_ra_matched = [] DWF_obs_dec_matched = [] for row in match_table: if row['matched_true_false'] == True: SM_match_true.append(row['matched_true_false']) SM_row_matched.append(row['matched_ID']) DWF_g_mags_matched.append(row['matched_DWF_data_gmag']) DWF_obs_ra_matched.append(row['matched_DWF_RA']) DWF_obs_dec_matched.append(row['matched_DWF_DEC']) SM_RA = [] SM_DEC = [] SM_g_mag = [] SM_r_mag = [] SM_i_mag = [] SM_z_mag = [] for j in SM_row_matched: RA = SM_X[j, 0] DEC = SM_X[j, 1] g_mag = SM_X[j, 2] r_mag = SM_X[j, 3] i_mag = SM_X[j, 4] z_mag = SM_X[j, 5] SM_RA.append(RA) SM_DEC.append(DEC) SM_g_mag.append(g_mag) SM_r_mag.append(r_mag) SM_i_mag.append(i_mag) SM_z_mag.append(z_mag) source_match_table = Table() source_match_table['SM_match_true'] = SM_match_true print('SKY MATCH: ' + str(len(source_match_table['SM_match_true']))) source_match_table['SM_match_row_ID'] = SM_row_matched source_match_table['SM_g_mag'] = SM_g_mag source_match_table['SM_r_mag'] = SM_r_mag source_match_table['SM_i_mag'] = SM_i_mag source_match_table['SM_z_mag'] = SM_z_mag source_match_table['SM_RA'] = SM_RA source_match_table['SM_DEC'] = SM_DEC source_match_table['DWF_gmag'] = DWF_g_mags_matched source_match_table['DWF_RA'] = DWF_obs_ra_matched source_match_table['DWF_DEC'] = DWF_obs_dec_matched diff_mags = [] num_stars = [] for row, row1 in zip(source_match_table['DWF_gmag'], source_match_table['SM_g_mag']): if row < 24: num_stars.append(1) diff = row - row1 if diff < 1.5: diff_mags.append(diff) filenames_good.append(filename) #print(diff) elif diff > 1.5: odd_differences.append(diff) filenames_odd.append(filename) elif row > 24: pass print('num photom stars:' + str(len(num_stars))) num_photom_sources.append(len(num_stars)) if len(diff_mags) > 0: mean_diff = np.mean(diff_mags) median_diff= np.median(diff_mags) print('mean correction (mag):' +str(mean_diff)) print('median correction (mag):' +str(median_diff)) master_mean_diff.append(mean_diff) master_median_diff.append(median_diff) #print('got to step 3: found mean: ' + str(mean_diff)) source_match_table['DWF_g_mag_ZPoffset'] = source_match_table['DWF_gmag'] - mean_diff #a = [10, 23] #b = [10, 23] #plt.scatter(source_match_table['DWF_gmag'], source_match_table['SM_g_mag']) #plt.ylim(10, 23) #plt.xlim(10, 23) #plt.plot(a, b, 'r') #plt.xlabel('DWF_g_mag') #plt.ylabel('SM_g_mag') #plt.title('uncorrected ' + str(filename)) #plt.savefig(photom_correction_path_images + filename + 'skymapper_uncorrection.png', overwrite = True) #plt.show() #plt.close() #plt.scatter(source_match_table['DWF_g_mag_ZPoffset'], source_match_table['SM_g_mag']) #plt.ylim(10, 23) #plt.xlim(10, 23) #plt.plot(a, b, 'r') #plt.xlabel('DWF_g_mag') #plt.ylabel('SM_g_mag') #plt.title('corrected ' + str(filename)) #plt.savefig(photom_correction_path_images + filename + 'skymapper_corrected.png', overwrite = True) #plt.close() corrected_g_mags = ccd_SE_cat['mag_auto'] - mean_diff corrected_g_mags_aper = ccd_SE_cat['mag_aper'] - mean_diff #print(corrected_g_mags) #print('Matching Sky Mapper sources found! Correcting to average deviation for this ccd.') new_cat = Table() new_cat['RA'] = ccd_SE_cat['RA'] new_cat['DEC'] = ccd_SE_cat['DEC'] new_cat['g_mag'] = corrected_g_mags new_cat['g_mag_err'] = ccd_SE_cat['mag_auto_err'] new_cat['g_mag_APER'] = corrected_g_mags_aper new_cat['g_mag_err_APER'] = ccd_SE_cat['magerr_aper'] #print(new_cat) av_correction.append(mean_diff) #print('HELOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOO') #print('average correction ' + str(av_correction)) t = new_cat output= final_source_cat_path + '/'+ filename + '_NEWCORRECTED.ascii' #print(output) t.write(output, format= 'ascii' , overwrite = True) '''new_tab = Table() new_tab['filename'] = str(filename) new_tab['num ccd sources'] = num_sources_ccd new_tab['num SM source'] = len(num_stars) new_tab['mean diff '] = mean_diff new_tab['median diff '] = median_diff output_name = field + '_'+year+'_' +month+'_CHECK_VALUES.ascii' new_tab.write(output_name, format='ascii', overwrite=True)''' print('FILENAME' ) else: print('No gband from SkyMapper for this ccd') if len(av_correction) != 0: mean_of_means = np.mean(av_correction) print('mean_of_means: ' + str(mean_of_means)) if mean_of_means == 0: print('No matching Skymapper sources, NOT correcting for this ccd') corrected_g_mags = ccd_SE_cat['mag_auto'] - mean_of_means corrected_g_mags_aper = ccd_SE_cat['mag_aper'] - mean_of_means new_cat = Table() new_cat['RA'] = ccd_SE_cat['RA'] new_cat['DEC'] = ccd_SE_cat['DEC'] new_cat['g_mag_AUTO'] = corrected_g_mags new_cat['g_mag_err_AUTO'] = ccd_SE_cat['mag_auto_err'] new_cat['g_mag_APER'] = corrected_g_mags_aper new_cat['g_mag_err_APER'] = ccd_SE_cat['magerr_aper'] t = new_cat output= final_source_cat_path + '/' + filename + '_NOT_NEWCORRECTED.ascii' t.write(output, format= 'ascii' , overwrite = True) elif mean_of_means > 0: #print('No matching Skymapper sources, correcting to average of average corrections for this field') corrected_g_mags = ccd_SE_cat['mag_auto'] - mean_of_means corrected_g_mags_aper = ccd_SE_cat['mag_aper'] - mean_of_means new_cat = Table() new_cat['RA'] = ccd_SE_cat['RA'] new_cat['DEC'] = ccd_SE_cat['DEC'] new_cat['g_mag_AUTO'] = corrected_g_mags new_cat['g_mag_err_AUTO'] = ccd_SE_cat['mag_auto_err'] new_cat['g_mag_APER'] = corrected_g_mags_aper new_cat['g_mag_err_APER'] = ccd_SE_cat['magerr_aper'] t = new_cat output= final_source_cat_path + '/' + filename + '_NEWCORRECTED.ascii' t.write(output, format= 'ascii' , overwrite = True) #new_table =Table() #new_table['filename'] = else: print('No matching Skymapper sources, NOT correcting for this ccd') corrected_g_mags = ccd_SE_cat['mag_auto'] corrected_g_mags_aper = ccd_SE_cat['mag_aper'] new_cat = Table() new_cat['RA'] = ccd_SE_cat['RA'] new_cat['DEC'] = ccd_SE_cat['DEC'] new_cat['g_mag_AUTO'] = corrected_g_mags new_cat['g_mag_err_AUTO'] = ccd_SE_cat['mag_auto_err'] new_cat['g_mag_APER'] = corrected_g_mags_aper new_cat['g_mag_err_APER'] = ccd_SE_cat['magerr_aper'] t = new_cat output= final_source_cat_path + '/' + filename + '_NOT_NEWCORRECTED.ascii' t.write(output, format= 'ascii' , overwrite = True) try: av_tab = Table() av_tab['average correction'] = av_correction #av_tab['filenames'] = filenames_good print('HELLOOOOOOOOOOOOOOOOOOOOOOOOOOOO') print(av_tab) #av_tab['filename'] = filenames_for_av output2 = photom_correction_path + '/average_correction_per_ccd.ascii' av_tab.write(output2, format ='ascii', overwrite = True) new_tab = Table() new_tab['filename'] = master_filenames new_tab['num ccd sources'] = num_sources_ccd new_tab['num SM source'] = num_photom_sources new_tab['mean diff '] = master_mean_diff new_tab['median diff '] = master_median_diff output_name = photom_correction_path + filename[0:17]+'_NEW_CHECK_VALUES.ascii' new_tab.write(output_name, format='ascii', overwrite=True) print(output_name) #print(av_tab) except: pass try: odd_tab = Table() odd_tab['odd differences'] = odd_differences odd_tab['filenames'] = filenames_odd output3 = photom_correction_path + '/odd_values_found_per_ccd.ascii' odd_tab.write(output3, format ='ascii', overwrite = True) except: pass filenames_good.clear() av_correction.clear() return print('finshed')
MAG_APP_OBS_band3, MAGERR_APP_OBS_band3, MAG_AUTO_OBS_band3, MAGERR_AUTO_OBS_band3, XPEAK_OBS_band3, YPEAK_OBS_band3, X_IMG_OBS_band3, Y_IMG_OBS_band3, RA_OBS_band3, DEC_OBS_band3 = np.loadtxt(field_name +'_' + filter_band3 + '_band_' + date +'.cat', unpack = True) Coords_obs_band3 = np.empty((len(MAG_APP_OBS_band3), 4), dtype = np.float64) Coords_obs_band3[:, 0] = RA_OBS_band3 Coords_obs_band3[:, 1] = DEC_OBS_band3 Coords_obs_band3[:, 2] = MAG_AUTO_OBS_band3 Coords_obs_band3[:, 3] = MAGERR_AUTO_OBS_band3 #print(Coords_obs_band3) #---------------Cross match between sources between the two bands -------------- # max_radius = 1./3600 # 1 arcsec dist_between, ind_row = crossmatch_angular(Coords_obs_band1, Coords_obs_band2, max_radius) match = ~np.isinf(dist_between) #print(match) match_table = Table() match_table['band_' + filter_band2+ '_match_TF'] = match match_table['band_' + filter_band2+ '_match_ID'] = ind_row match_table['band_' + filter_band1+ '_obs_mag'] = MAG_AUTO_OBS_band1 match_table['band_' + filter_band1+ '_obs_mags_err'] = MAGERR_AUTO_OBS_band1 match_table['band_' + filter_band1+ '_obs_ra'] = RA_OBS_band1 match_table['band_' + filter_band1+ '_obs_dec'] = DEC_OBS_band1 print(match_table) band2_match_true = []
data_table['band_' + filter_band3 + '_mag_err'] = data['band_' + filter_band3 + '_mag_err'] data_table['band_' + filter_band3 + '_ra'] = data['band_' + filter_band3 + '_ra'] data_table['band_' + filter_band3 + '_dec'] = data['band_' + filter_band3 + '_dec'] #----------- make coords_cat for the the first band to cross match with the entred coords ----------------# coords_cat = np.empty((len(data), 12), dtype=np.float64) coords_cat[:, 0] = data['band_' + filter_band1 + '_ra'] coords_cat[:, 1] = data['band_' + filter_band1 + '_dec'] coords_cat[:, 2] = data['band_' + filter_band1 + '_mag'] max_radius = sep_arcsec / 3600. # 5 arcsec dist_between_can, ind_row_can = crossmatch_angular(coords_cat, can_entry, max_radius) match_can = ~np.isinf(dist_between_can) print(match_can) coords_matched_band1_mag = [] coords_matched_band1_mag_err = [] coords_matched_band2_mag = [] coords_matched_band2_mag_err = [] coords_matched_band3_mag = [] coords_matched_band3_mag_err = [] coords_matched_table = Table() coords_matched_table['match_true_false'] = match_can coords_matched_table['band_' + filter_band1 + '_mag'] = coords_cat[:, 2] coords_matched_table['band_' + filter_band1 + '_mag_err'] = data_table['band_' + filter_band1 +
# get imaging data image_data = fetch_imaging_sample() imX = np.empty((len(image_data), 2), dtype=np.float64) imX[:, 0] = image_data['ra'] imX[:, 1] = image_data['dec'] # get standard stars standards_data = fetch_sdss_S82standards() stX = np.empty((len(standards_data), 2), dtype=np.float64) stX[:, 0] = standards_data['RA'] stX[:, 1] = standards_data['DEC'] # crossmatch catalogs max_radius = 1. / 3600 # 1 arcsec dist, ind = crossmatch_angular(imX, stX, max_radius) match = ~np.isinf(dist) dist_match = dist[match] dist_match *= 3600 ax = plt.axes() hist(dist_match, bins='knuth', ax=ax, histtype='stepfilled', ec='k', fc='#AAAAAA') ax.set_xlabel('radius of match (arcsec)') ax.set_ylabel('N(r, r+dr)') ax.text(0.95, 0.95, "Total objects: %i\nNumber with match: %i" % (imX.shape[0], np.sum(match)), ha='right', va='top', transform=ax.transAxes) ax.set_xlim(0, 0.2)
if not os.path.exists(photom_correction_path): os.makedirs(photom_correction_path, 0o755) else: pass if not os.path.exists(final_source_cat_path): os.makedirs(final_source_cat_path, 0o755) else: pass if not os.path.exists(photom_correction_path_images): os.makedirs(photom_correction_path_images, 0o755) else: pass max_radius = 5. / 3600 #1 arc second dist_between, ind_row = crossmatch_angular(DWF_X, SM_X, max_radius) match = ~np.isinf(dist_between) if len(match) != 0: match_table = Table() match_table['matched_true_false'] = match match_table['matched_ID'] = ind_row match_table['matched_DWF_data_gmag'] = ccd_SE_cat['mag_auto'] match_table['matched_DWF_RA'] = ccd_SE_cat['RA'] match_table['matched_DWF_DEC'] = ccd_SE_cat['DEC'] SM_match_true = [] SM_row_matched = [] DWF_g_mags_matched = [] DWF_g_mags_error_matched = [] DWF_obs_ra_matched = [] DWF_obs_dec_matched = [] for row in match_table:
def create_finding_chart_inputs(RA, DEC, source_name, field, template, verbose=False, debugmode=False): print('#############################################') print('# CANVIS Finding Chart version has started #') print('#############################################') print(f'Finding chart image for RA: {RA} DEC: {DEC}') temp_path = '/fred/oz100/pipes/DWF_PIPE/field_stacks/' + template print(f'Template path being used is: {temp_path}') output_path = '/fred/oz100/FINDING_CHARTS/' output_files_path = '/fred/oz100/FINDING_CHARTS/' + field + '/' + source_name + '/' print(f'Your files will be saved here: {output_files_path}') if not os.path.exists(output_files_path): os.makedirs(output_files_path, 0o755) else: pass mydic = SortedDict() with fits.open(temp_path) as hdu: print('## Template opened ##') size = 1370 w = WCS(hdu[0].header) #print(w) head = hdu[0].header date = dt.datetime.strptime(head['DATE'], '%Y-%m-%dT%H:%M:%S') xlim = head['NAXIS1'] ylim = head['NAXIS2'] print(xlim, ylim) pixcrd_im = np.array([[xlim, ylim]], np.float_) world_im = w.wcs_pix2world(pixcrd_im, 1) pixx_im, pixy_im = world_im[0][0], world_im[0][1] pixcrd = np.array([[RA, DEC]], np.float_) worldpix = w.wcs_world2pix(pixcrd, 1) pixx, pixy = worldpix[0][0], worldpix[0][1] cutout = Cutout2D(hdu[0].data, (pixx, pixy), size, wcs=w) hdu[0].data = cutout.data hdu[0].header['CRPIX1'] = cutout.wcs.wcs.crpix[0] hdu[0].header['CRPIX2'] = cutout.wcs.wcs.crpix[1] hdu.writeto(output_files_path + source_name + '.fits', overwrite=True) print('## Large Fits CUTOUT saved ##') small_size = 457 with fits.open(temp_path) as hdu: size = 457 w = WCS(hdu[0].header) #print(w) head = hdu[0].header date = dt.datetime.strptime(head['DATE'], '%Y-%m-%dT%H:%M:%S') xlim = head['NAXIS1'] ylim = head['NAXIS2'] print(xlim, ylim) pixcrd_im = np.array([[xlim, ylim]], np.float_) world_im = w.wcs_pix2world(pixcrd_im, 1) pixx_im, pixy_im = world_im[0][0], world_im[0][1] pixcrd = np.array([[RA, DEC]], np.float_) worldpix = w.wcs_world2pix(pixcrd, 1) pixx, pixy = worldpix[0][0], worldpix[0][1] cutout = Cutout2D(hdu[0].data, (pixx, pixy), size, wcs=w) hdu[0].data = cutout.data hdu[0].header['CRPIX1'] = cutout.wcs.wcs.crpix[0] hdu[0].header['CRPIX2'] = cutout.wcs.wcs.crpix[1] hdu.writeto(output_files_path + source_name + 'SMALL.fits', overwrite=True) print('## Small Fits CUTOUT saved ##') print('## Starting Source Extracting for SMALL image ##') os.system('sex ' + output_files_path + source_name + 'SMALL.fits' + ' -c default_params.sex -CATALOG_NAME ' + output_files_path + source_name + '_SE.cat -DETECT_THRESH 1.5 -MAG_ZEROPOINT 25 ') print('## Source extractor done! ##') print('#############################################') print('## Cross matching with SkyMapper DR2 ##') print('#############################################') try: skymapper_cat_path = '/fred/oz100/SM_cats/' + field + '_SM.csv' sm_cat = pd.read_csv(skymapper_cat_path).to_dict(orient='row') sm_ra = [] sm_dec = [] sm_ngood = [] sm_class_star = [] sm_g_psf = [] sm_g_psf_error = [] sm_r_psf = [] sm_r_psf_error = [] sm_i_psf = [] sm_i_psf_error = [] sm_z_psf = [] sm_z_psf_error = [] if debugmode: print('Numebr of sources in SkyMapper Catalouge: ' + str(len(sm_cat))) for i in range(len(sm_cat)): sm_ra.append(sm_cat[i]['raj2000']) sm_dec.append(sm_cat[i]['dej2000']) sm_ngood.append(sm_cat[i]['ngood']) sm_class_star.append(sm_cat[i]['class_star']) sm_g_psf.append(sm_cat[i]['g_psf']) sm_g_psf_error.append(sm_cat[i]['e_g_psf']) sm_r_psf.append(sm_cat[i]['r_psf']) sm_r_psf_error.append(sm_cat[i]['e_r_psf']) sm_i_psf.append(sm_cat[i]['i_psf']) sm_i_psf_error.append(sm_cat[i]['e_i_psf']) sm_z_psf.append(sm_cat[i]['z_psf']) sm_z_psf_error.append(sm_cat[i]['e_z_psf']) SM_table = Table() SM_table['RA'] = sm_ra SM_table['DEC'] = sm_dec SM_table['ngood'] = sm_ngood SM_table['class_star'] = sm_class_star SM_table['g_psf'] = sm_g_psf SM_table['e_g_psf'] = sm_g_psf_error SM_table['r_psf'] = sm_r_psf SM_table['e_r_psf'] = sm_r_psf_error SM_table['i_psf'] = sm_i_psf SM_table['e_i_psf'] = sm_i_psf_error SM_table['z_psf'] = sm_z_psf SM_table['e_z_psf'] = sm_z_psf_error SM_filtered_ra = [] SM_filtered_dec = [] SM_filtered_ngood = [] SM_filtered_class_star = [] SM_filtered_g_psf = [] SM_filtered_g_psf_err = [] SM_filtered_r_psf = [] SM_filtered_r_psf_err = [] SM_filtered_i_psf = [] SM_filtered_i_psf_err = [] SM_filtered_z_psf = [] SM_filtered_z_psf_err = [] for row in SM_table: if 0.90 <= row['class_star'] <= 1.0: SM_filtered_ra.append(row['RA']) SM_filtered_dec.append(row['DEC']) SM_filtered_ngood.append(row['ngood']) SM_filtered_class_star.append(row['class_star']) SM_filtered_g_psf.append(row['g_psf']) SM_filtered_g_psf_err.append(row['e_g_psf']) SM_filtered_r_psf.append(row['r_psf']) SM_filtered_r_psf_err.append(row['e_r_psf']) SM_filtered_i_psf.append(row['i_psf']) SM_filtered_i_psf_err.append(row['e_i_psf']) SM_filtered_z_psf.append(row['z_psf']) SM_filtered_z_psf_err.append(row['e_z_psf']) if debugmode: print('Number of SkyMapper Sources pass S/G cuts: ' + str(len(SM_filtered_ra))) SM_X = np.empty((len(SM_filtered_ra), 7), dtype=np.float64) SM_X[:, 0] = SM_filtered_ra SM_X[:, 1] = SM_filtered_dec SM_X[:, 2] = SM_filtered_ngood SM_X[:, 3] = SM_filtered_g_psf SM_X[:, 4] = SM_filtered_r_psf SM_X[:, 5] = SM_filtered_i_psf SM_X[:, 6] = SM_filtered_z_psf MAG_APER, MAGERR_APER, MAG_AUTO, MAGERR_AUTO, XPEAK_IMAGE, YPEAK_IMAGE, X_IMAGE, Y_IMAGE, ALPHA_J2000, DELTA_J2000 = np.loadtxt( output_files_path + source_name + '_SE.cat', unpack=True) DWF_X = np.empty((len(MAG_APER), 2), dtype=np.float64) DWF_X[:, 0] = ALPHA_J2000 DWF_X[:, 1] = DELTA_J2000 max_radius = 2 / 3600 #1 arc second dist_between, ind_row = crossmatch_angular(DWF_X, SM_X, max_radius) match = ~np.isinf(dist_between) if debugmode: print('Length of Match Table: ' + str(len(match))) print(match) if len(match) != 0: match_table = Table() match_table['matched_true_false'] = match match_table['matched_ID'] = ind_row SM_match_true = [] SM_row_matched = [] DWF_g_mags_matched = [] DWF_g_mags_error_matched = [] DWF_obs_ra_matched = [] DWF_obs_dec_matched = [] for row in match_table: if row['matched_true_false'] == True: if debugmode: print('Found match source: ' + str(row['matched_ID'])) SM_match_true.append(row['matched_true_false']) SM_row_matched.append(row['matched_ID']) SM_RA = [] SM_DEC = [] SM_g_mag = [] SM_r_mag = [] SM_i_mag = [] SM_z_mag = [] for j in SM_row_matched: RA = SM_X[j, 0] DEC = SM_X[j, 1] g_mag = SM_X[j, 3] r_mag = SM_X[j, 4] i_mag = SM_X[j, 5] z_mag = SM_X[j, 6] SM_RA.append(RA) SM_DEC.append(DEC) SM_g_mag.append(g_mag) SM_r_mag.append(r_mag) SM_i_mag.append(i_mag) SM_z_mag.append(z_mag) SM_final_table = Table() SM_final_table['RA'] = SM_RA SM_final_table['DEC'] = SM_DEC SM_final_table['g_mag'] = SM_g_mag SM_final_table['r_mag'] = SM_r_mag SM_final_table['i_mag'] = SM_i_mag SM_final_table['z_mag'] = SM_z_mag output = output_files_path + source_name + '_skymapper_star_cat.ascii' SM_final_table.write(output, format='ascii', overwrite=True) print('#############################################') print('SkyMapper Sources Found and Saved') print('#############################################') except: print('SKY MAPPER SOURCE CHECK FAILED') print('#############################################') print('Starting to search for GAIA sources') GAIA_DR2 = "I/345" field_RA_DEC = str(f'{RA} {DEC}') result = Vizier.query_region(field_RA_DEC, radius=Angle('120"'), catalog=GAIA_DR2) print(field_RA_DEC) GAIA_DR2 = Table() GAIA_DR2['RA'] = result[0]['RA_ICRS'] GAIA_DR2['DEC'] = result[0]['DE_ICRS'] GAIA_DR2['Gmag'] = result[0]['Gmag'] GAIA_DR2['BPmag'] = result[0]['BPmag'] GAIA_DR2['RPmag'] = result[0]['RPmag'] output = output_files_path + source_name + '_gaia_star_cat.ascii' GAIA_DR2.write(output, format='ascii', overwrite=True) print('#############################################') print('# YOUR FINDING CHART INPUTS ARE DONE#') print(f'# FIND THEM HERE: {output_files_path}') print('#############################################')