def lens_on_color_mag(TGASid, filename): """ Plots the lens with TGASid on a color magnitude diagram with the rest of the TGAS sources - uses 2mass [j-k] colour and absolute G magnitude, saves the plot with file name. Args: TGASid (long) : indentified for lens id in TGAS source_id column filename (string) : output file name Returns: file (.png) : output file of the plot with filename. """ #Get color mag info for lens querystringLens = 'select j_m,k_m,parallax,phot_g_mean_mag from gaia_dr1_aux.gaia_source_2mass_xm where source_id=' + str( TGASid) JmagLens, kMagLens, parallaxLens, gMagLens = sqlutil.get( querystringLens, db='wsdb', host='cappc127.ast.cam.ac.uk', user='******', password='******') #Calculate colour and absolute mag for lens absMagLens = gMagLens + 5 * (np.log10(parallaxLens / 1000.0) + 1) colorLens = JmagLens - kMagLens #Get colour mag for 10000 source form tgas querystring2mass = 'select j_m,k_m,parallax,phot_g_mean_mag from gaia_dr1_aux.gaia_source_2mass_xm where parallax > 0.0001 and j_m > 0.0001 and k_m > 0.0001 limit 100000' Jmag, kMag, parallax, gMag = sqlutil.get(querystring2mass, db='wsdb', host='cappc127.ast.cam.ac.uk', user='******', password='******') #Calculate colour and absolute mags color = Jmag - kMag absmag = gMag + 5 * (np.log10(parallax / 1000.0) + 1) #plot 2dhist with scatter of lens overlaid plt.hist2d(color, absmag, bins=300, range=[[-0.1, 1.25], [-2, 13.5]], cmap='plasma') plt.scatter([colorLens], [absMagLens], color='white', label='TGAS Lens') plt.gca().invert_yaxis() plt.xlabel(r'J-K [2MASS]') plt.ylabel('Absolute G [mag]') plt.legend() plt.savefig(filename, dpi=200) plt.clf()
def get_gaia_source_pos(centerRa, centerDec, size): """ Get all the gaia source positions are epoch 2015.0 in a search radius of size. Args: centerRA (double) : Right acession of the center of the search radius [Degrees] centerDec (souble) : Declination of the center of the the search radius [Degrees] size (double) : size of the search radius [arminutes] Returns: pos (np.array(double)) : Position of gaia sources [ [Ra,Ra],[dec,dec]..] """ query = 'select ra,dec from gaia_dr1.gaia_source where ra BETWEEN ' + str( centerRa) + ' - 0.016 and ' + str( centerRa) + '+ 0.016 AND dec BETWEEN ' + str( centerDec) + '- 0.016 and ' + str(centerDec) + '+ 0.016' ra, dec = sqlutil.get(query, db='wsdb', host='cappc127.ast.cam.ac.uk', user='******', password='******') return [ra, dec]
def getdata(cached=True): if not cached: import sqlutil Q = 'select fieldid, sky_u,sky_g,sky_r,sky_i,sky_z,ramin as ra,decmin as dec,mjd_r as mjd from sdssdr9.field ' D = sqlutil.get(Q, host='cappc127', asDict=True) fieldid, sky_u, sky_g, sky_r, sky_i, sky_z, ra, dec, mjd = D.values() tab = atpy.Table(data=D) tab.write('sdss_sky.fits', format='fits', overwrite=True) else: tab = atpy.Table().read('sdss_sky.fits') fieldid, sky_u, sky_g, sky_r, sky_i, sky_z, ra, dec, mjd = [ tab[_] for _ in 'fieldid,sky_u,sky_g,sky_r,sky_i,sky_z,ra,dec,mjd'.split(',') ] return fieldid, sky_u, sky_g, sky_r, sky_i, sky_z, ra, dec, mjd
def queryDatabase(query, columns, post_query, cache_file='db.pickle'): #cache_file = "reducedPM2.pickle" if not path.exists(cache_file): res = sqlutil.get(query.format(','.join(columns), post_query), db='wsdb', host='cappc127.ast.cam.ac.uk', user=user, password=password, asDict=True) with open(cache_file, "wb") as f: pickle.dump(res, f) else: with open(cache_file, "rb") as f: res = pickle.load(f) return res
def nearest_neighbour(tile, outdir, checkplots=False, saveplots=False): """ """ from astropy.table import Table, hstack import srpylib import match_lists import matplotlib.pyplot as plt import numpy as np db, host, user, password, db_table = rd_config_wsdb(table='wise') t = Table.read(outdir + tile + "_WISEfp_DEScat.fits") ra_min, ra_max, dec_min, dec_max = srpylib.min_max_match( t["RA_CALC_G"], t["DEC_CALC_G"], 1.0, DES_box=True ) query_wise = "select ra, dec FROM allwise.main WHERE ra > " + \ ra_min + " and ra < " + ra_max + " and dec > " + dec_min + \ " and dec < " + dec_max RA, DEC = sqlutil.get(query_wise, db=db, host=host, user=user, password=password) dists, inds = match_lists.match_lists( t["RA_CALC_G"].data, t["DEC_CALC_G"].data, RA, DEC, 1.0, 2 ) match_distance = dists[:,1]*3600.0 if checkplots: plt.axvline(6.1) plt.axvline(6.4) plt.axvline(6.5) plt.axvline(12.0) plt.hist(match_distance, bins=100) plt.xlabel('Match Distance(")') plt.ylabel("Number") plt.title("Nearest Neighbours:" + tile) plt.show() print("Median Distance:", np.median(match_distance))
def get_closes_gaia_source_match(centerRa, centerDec, size): """ Get all the gaia source information for all gaia sources within a small search radius. And have low proper motions Args: centerRA (double) : Right acession of the center of the search radius [Degrees] centerDec (souble) : Declination of the center of the the search radius [Degrees] size (double) : size of the search radius [arminutes] Returns: pos (np.array(double)) : Position of gaia sources [ [[id, id,..],[Ra,Ra],[dec,dec],[gmag,gmag...]..] """ query = 'select ra,dec, source_id, phot_g_mag_mean from gaia_dr1.gaia_source where ra BETWEEN ' + str( centerRa) + ' - 0.026 and ' + str( centerRa) + '+ 0.026 AND dec BETWEEN ' + str( centerDec) + '- 0.026 and ' + str( centerDec) + '+ 0.026 AND hypot(pmra,pmdec) < 50.0' ra, dec, id, gmag = sqlutil.get(query, db='wsdb', host='cappc127.ast.cam.ac.uk', user='******', password='******') return [ra, dec, id, gmag]
if vmin is None: vmin = 0 if vmax is None: vmax = hh.max() if weight_norm: hh = hh * 1. / (hhsum + 1 * (hhsum == 0)) if skip_empty: hh1 = hh[hhsum > 0] else: hh1 = hh coll.set_array(hh1) coll.set_cmap(cmap) coll.set_clim(vmin, vmax) coll.set_edgecolors(coll.cmap(coll.norm(hh1))) coll.set_rasterized(rasterized) tmpcoll.set_clim(vmin, vmax) ax.add_collection(coll) return coll if __name__=='__main__': ras,decs=sqlutil.get('select radeg,dedeg from sdss_phot_qso.main', host='cappc118') doit(ras,decs, ramin=0,ramax=360,decmin=-90, decmax=90, nside=64) plt.savefig('xx.png',dpi=200)
def VHS_match(t, RA_main, DEC_main, c_graph = True, width = 0.0004, w_units = "degrees", survey = "des", bands = ["Y", "J", "H", "K"], DES_box = False): """ Matches to VHS Data from WSDB Tables used are vhs_1504. + survey DES_box returns all objects within a box of about the size of a DES tile centred on the median RA and DEC of the table. Iterates through a table where the RA column is RA_main and the DEC_column is DEC_main Puts NaNs in for no matches c_graph plots diagnostic graphs of the match - crashes if there are no matches Can specify not the full number of bands #29/03/16 - Changed from 1504 to 1603 """ import numpy as np import match_lists import sqlutil from astropy.table import Table, vstack import ConfigParser config_file = "/home/sr525/Python_Code/srpylib/srpylib.cfg" config = ConfigParser.RawConfigParser() config.read(config_file) uname = config.get("wsdb", "uname") pword = config.get("wsdb", "pword") host = config.get("wsdb", "host") if w_units == "arcsecs": width = width / 3600.0 if survey == "des": cols = ["RA_VHS", "DEC_VHS", "J", "J_ERR", "H", "H_ERR", "K", "K_ERR"] elif survey == "atlas": cols = ["RA_VHS", "DEC_VHS", "Y", "Y_ERR", "J", "J_ERR", "H", "H_ERR", \ "K", "K_ERR"] try: for col in cols: t.add_empty_column(col, dtype = np.float64) except AttributeError as e: for col in cols: t[col] = np.zeros(len(t)) ra_min, ra_max, dec_min, dec_max = min_max_match(t[RA_main], t[DEC_main], \ width, DES_box = DES_box) area = (float(ra_max)-float(ra_min))*(float(dec_max)-float(dec_min)) print area if len(t) > 1000 and area < 2000: ra_min, ra_max, dec_min, dec_max = min_max_match(t[RA_main], t[DEC_main], \ width, DES_box = DES_box) print "Retreiving data" if survey == "atlas": query_vhs = "select ra, dec, yapercormag3, yapercormag3_err, " + \ "japercormag3, japercormag3_err, hapercormag3, " + \ "hapercormag3_err, kapercormag3, kapercormag3_err " + \ "FROM vhs_1603." + survey + " WHERE ra > " + ra_min + \ " and ra < " + ra_max + " and dec > " + dec_min + \ " and dec < " + dec_max RA_vhs, DEC_vhs, Y, Yerr, J, Jerr, H, Herr, K, Kerr = \ sqlutil.get(query_vhs, db="wsdb", host=host, user=uname, password=pword) elif survey == "des": query_vhs = "select ra, dec, japercormag3, japercormag3_err, " + \ "hapercormag3, hapercormag3_err, kapercormag3, " + \ "kapercormag3_err FROM vhs_1603." + survey + \ " WHERE ra > " + ra_min + " and ra < " + ra_max + \ " and dec > " + dec_min + " and dec < " + dec_max RA_vhs, DEC_vhs, J, Jerr, H, Herr, K, Kerr = \ sqlutil.get(query_vhs, db="wsdb", host=host, user=uname, password=pword) print "Retreived data" if len(RA_vhs) == 0: for (i, col) in enumerate(cols): t[col] = [np.float64(np.nan)]*len(t) else: dists, inds = match_lists.match_lists(t[RA_main], t[DEC_main], RA_vhs, DEC_vhs, width, 1) if survey == "des": info = [RA_vhs, DEC_vhs, J, Jerr, H, Herr, K, Kerr] elif survey == "atlas": info = [RA_vhs, DEC_vhs, Y, Yerr, J, Jerr, H, Herr, K, Kerr] match = np.where( (inds <> len(RA_vhs)) )[0] no_match = np.where( (inds == len(RA_vhs)) )[0] for (i, col) in enumerate(cols): t[col][match] = info[i][inds[match]] t[col][no_match] = np.float64(np.nan) if len(t) <= 1000: n = 0 while n < len(t): RA = t[RA_main][n] DEC = t[DEC_main][n] if survey == "atlas": query_vhs = "select ra, dec, yapercormag3, yapercormag3_err, " + \ "japercormag3, japercormag3_err, hapercormag3, " + \ "hapercormag3_err, kapercormag3, kapercormag3_err " + \ "FROM vhs_1504." + survey + \ " WHERE q3c_radial_query(ra, dec, " + str(RA) + \ ", " + str(DEC) + ", 0.1102)" RA_vhs, DEC_vhs, Y, Yerr, J, Jerr, H, Herr, K, Kerr = \ sqlutil.get(query_vhs, db="wsdb", host=host, user=uname, password=pword) elif survey == "des": query_vhs = "select ra, dec, japercormag3, japercormag3_err, " + \ "hapercormag3, hapercormag3_err, kapercormag3, " + \ "kapercormag3_err FROM vhs_1504." + survey + \ " WHERE q3c_radial_query(ra, dec, " + str(RA) + \ ", " + str(DEC) + ", 0.1102)" RA_vhs, DEC_vhs, J, Jerr, H, Herr, K, Kerr = \ sqlutil.get(query_vhs, db="wsdb", host=host, user=uname, password=pword) if len(RA_vhs) > 0: dists, inds = match_lists.match_lists([RA], [DEC], RA_vhs, DEC_vhs, width, 1) j = inds[0] if j <> len(RA_vhs): if survey == "des": info = [RA_vhs[j], DEC_vhs[j], J[j], Jerr[j], H[j], Herr[j], \ K[j], Kerr[j]] if survey == "atlas": info = [RA_vhs[j], DEC_vhs[j], Y[j], Yerr[j], J[j], Jerr[j], \ H[j], Herr[j], K[j], Kerr[j]] for (k, col) in enumerate(cols): t[col][n] = info[k] else: for (k, col) in enumerate(cols): t[col][n] = np.float64(np.nan) else: for (k, col) in enumerate(cols): t[col][n] = np.float64(np.nan) n += 1 print "Done VHS" if area > 2000 and len(t) > 1000: t.sort(RA_main) l1 = np.ceil(area/1001.0) l = len(t)/l1 i = 0 while i < l1: if (i+1)*l < len(t): t1 = t[i*l:(i+1)*l] else: t1 = t[i*l:] t1 = VHS_match(t1, RA_main, DEC_main, c_graph = c_graph, width = width, \ w_units = w_units, survey = survey, bands = bands, \ DES_box = DES_box) if i == 0: t_out = t1 else: t_out = vstack([t_out, t1]) i += 1 t = t_out if c_graph: width = width *3600.0 RA_med, DEC_med = check_graph(t, RA_main, DEC_main, "VHS", width) return t
def GALEX_match(t, RA_main, DEC_main, width = 0.0004, w_units = "degrees", c_graph = True, table = "ais"): """ Matches to GALEX table DES_box returns all objects within a box of about the size of a DES tile centred on the median RA and DEC of the table. Puts NaNs in for no matches c_graph plots diagnostic graphs of the match """ import numpy as np import match_lists import sqlutil import ConfigParser config_file = "/home/sr525/Python_Code/srpylib/srpylib.cfg" config = ConfigParser.RawConfigParser() config.read(config_file) uname = config.get("wsdb", "uname") pword = config.get("wsdb", "pword") host = config.get("wsdb", "host") if w_units == "arcsecs": width = width / 3600.0 cols = ["RA_GALEX", "DEC_GALEX", "NUV_MAG", "NUV_MAGERR", "FUV_MAG", \ "FUV_MAGERR", "E_BV", "NUV_CLASS_STAR", "FUV_CLASS_STAR"] try: for col in cols: t.add_empty_column(col, dtype = np.float64) except AttributeError as e: for col in cols: t[col] = np.zeros(len(t)) if len(t) > 1000: ra_min, ra_max, dec_min, dec_max = min_max_match(t[RA_main], t[DEC_main], \ width, DES_box = DES_box) print "Retreiving data" query = "select ra, dec, nuv_mag, nuv_magerr, fuv_mag, fuv_magerr, e_bv, \ nuv_class_star, fuv_class_star FROM galexgr6." + table + "primary \ WHERE ra > " + ra_min + " and ra < " + ra_max + " and dec > " + \ dec_min + " and dec < " + dec_max RA_galex, DEC_galex, nuv_mag, nuv_magerr, fuv_mag, fuv_magerr, e_bv, \ nuv_class_star, fuv_class_star = \ sqlutil.get(query, db = "wsdb", host=host, user=uname, password=pword) print "Retreived data" if len(RA_galex) == 0: for (i, col) in enumerate(cols): t[col] = [np.float64(np.nan)]*len(t) else: dists, inds = match_lists.match_lists(t[RA_main], t[DEC_main], RA_vhs, DEC_vhs, width, 1) info = [ra_galex, dec_galex, nuv_mag, nuv_magerr, fuv_mag, fuv_magerr, \ e_bv, nuv_class_star, fuv_class_star] match = np.where( (inds <> len(RA_galex)) )[0] no_match = np.where( (inds == len(RA_galex)) )[0] for (i, col) in enumerate(cols): t[col][match] = info[i][inds[match]] t[col][no_match] = np.float64(np.nan) elif len(t) <= 1000: n = 0 while n < len(t): RA = t[RA_main][n] DEC = t[DEC_main][n] query_galex = "select ra, dec, nuv_mag, nuv_magerr, fuv_mag, \ fuv_magerr, e_bv, nuv_class_star, fuv_class_star FROM \ galexgr6." + table + "primary WHERE \ q3c_radial_query(ra, dec, " + str(RA) + ", " + str(DEC) + ", 0.1102)" RA_galex, DEC_galex, nuv_mag, nuv_magerr, fuv_mag, fuv_magerr, e_bv, \ nuv_class_star, fuv_class_star = sqlutil.get(query_galex, db = "wsdb", \ host=host, user=uname, password=pword) if len(RA_galex) > 0: dists, inds = match_lists.match_lists([RA], [DEC], RA_galex, DEC_galex, width, 1) j = inds[0] if j <> len(RA_galex): info = [RA_galex[j], DEC_galex[j], nuv_mag[j], nuv_magerr[j], \ fuv_mag[j], fuv_magerr[j], e_bv[j], nuv_class_star[j], \ fuv_class_star[j]] for (k, col) in enumerate(cols): t[col][n] = info[rki] else: for (k, col) in enumerate(cols): t[col][n] = np.float64(np.nan) else: for (k, col) in enumerate(cols): t[col][n] = np.float64(np.nan) n += 1 print "Done GALEX" elif len(t) > 1000000000: print "Splitting by tile" tiles_all = t["TILENAME"] tiles = set(tiles_all) tiles = list(tiles) print len(tiles), "distinct tiles" t.sort("TILENAME") tiles.sort() n = 0 m = 0 rows = [] tile_rows = [] while n < len(t): tile = tiles[m] if t["TILENAME"][n] == tile: rows.append(n) n += 1 else: tile_rows.append(rows) rows = [] m += 1 for rows in tile_rows: t1 = t.rows(rows) print "Matching", t1["TILENAME"][0], "with length", len(t1) t1 = GALEX_match(t1, RA_main, DEC_main, width, c_graph = False) if tile_rows.index(rows) == 0: t_out = t1 else: t_out.append(t1) t = t_out if c_graph: width = width *3600.0 RA_med, DEC_med = check_graph(t, RA_main, DEC_main, "GALEX", width) return t
def VST_match(t, RA_main, DEC_main, c_graph = True, width = 0.0004, w_units = "degrees", DES_box = False): """ Matches to VST Data from WSDB Table used is vst_201310.atlas_rgi DES_box returns all objects within a box of about the size of a DES tile centred on the median RA and DEC of the table. Iterates through a table where the RA column is RA_main and the DEC_column is DEC_main Puts NaNs in for no matches c_graph plots diagnostic graphs of the match - crashes if there are no matches """ import numpy as np import match_lists import sqlutil import ConfigParser config_file = "/home/sr525/Python_Code/srpylib/srpylib.cfg" config = ConfigParser.RawConfigParser() config.read(config_file) uname = config.get("wsdb", "uname") pword = config.get("wsdb", "pword") host = config.get("wsdb", "host") if w_units == "arcsecs": width = width / 3600.0 cols = ["RA_VST", "DEC_VST", "G_VST", "G_VST_ERR", "R_VST", "R_VST_ERR", \ "I_VST", "I_VST_ERR"] try: for col in cols: t.add_empty_column(col, dtype = np.float64) except AttributeError as e: for col in cols: t[col] = np.zeros(len(t)) if len(t) > 1000 and len(t) < 1000000000: ra_min, ra_max, dec_min, dec_max = min_max_match(t[RA_main], t[DEC_main], \ width, DES_box = DES_box) print "Retreiving data" query = "select ra, dec, mag_g, magerr_g, mag_r, magerr_r, mag_i, " + \ "magerr_i FROM vst_201310.atlas_rgi WHERE ra > " + \ ra_min + " and ra < " + ra_max + " and dec > " + \ dec_min + " and dec < " + dec_max RA_vst, DEC_vst, g, gerr, r, rerr, i, ierr = \ sqlutil.get(query, db="wsdb", host=host, user=uname, password=pword) print "Retreived data" dists, inds = match_lists.match_lists(t[RA_main], t[DEC_main], RA_vst, DEC_vst, width, 1) print "Matched lists, updating table" info = [RA_vst, DEC_vst, g, gerr, r, rerr, i, ierr] match = np.where( (inds <> len(RA_vst)) )[0] no_match = np.where( (inds == len(RA_vst)) )[0] for (i, col) in enumerate(cols): t[col][match] = info[i][inds[match]] t[col][no_match] = np.float64(np.nan) print "Table updated" elif len(t) <= 1000: n = 0 while n < len(t): RA = t[RA_main][n] DEC = t[DEC_main][n] query_vhs = "select ra, dec, mag_g, magerr_g, mag_r, magerr_r, " + \ "mag_i, magerr_i FROM vst_201310.atlas_rgi " + \ "WHERE q3c_radial_query(ra, dec, " + str(RA) + ", " + \ str(DEC) + ", 0.1102)" RA_vst, DEC_vst, g, gerr, r, rerr, i, ierr = \ sqlutil.get(query_vhs, db="wsdb", host=host, user=uname, password=pword) if len(RA_vst) > 0: dists, inds = match_lists.match_lists([RA], [DEC], RA_vst, DEC_vst, width, 1) j = inds[0] if j <> len(RA_vst): info = [RA_vst[j], DEC_vst[j], g[j], gerr[j], r[j], rerr[j], \ i[j], ierr[j]] for (k, col) in enumerate(cols): t[col][n] = info[k] else: for (k, col) in enumerate(cols): t[col][n] = np.float64(np.nan) else: for (k, col) in enumerate(cols): t[col][n] = np.float64(np.nan) n += 1 elif len(t) > 1000000000: print "Splitting by tile" tiles_all = t["TILENAME"] tiles = set(tiles_all) tiles = list(tiles) print len(tiles), "distinct tiles" t.sort("TILENAME") tiles.sort() n = 0 m = 0 rows = [] tile_rows = [] while n < len(t): tile = tiles[m] if t["TILENAME"][n] == tile: rows.append(n) n += 1 else: tile_rows.append(rows) rows = [] m += 1 for rows in tile_rows: t1 = t.rows(rows) print "Matching", t1["TILENAME"][0], "with length", len(t1) t1 = VST_match(t1, RA_main, DEC_main, width, c_graph = False) if tile_rows.index(rows) == 0: t_out = t1 else: t_out.append(t1) t = t_out if c_graph: width = width *3600.0 RA_med, DEC_med = check_graph(t, RA_main, DEC_main, "VST", width) return t
def WISE_match(t, RA_main, DEC_main, width = 0.0004, w_units = "degrees", c_graph = True, DES_box = False): """ Matches to WISE Data from WSDB Table used is allwise.main DES_box returns all objects within a box of about the size of a DES tile centred on the median RA and DEC of the table. Iterates through a table where the RA column is RA_main and the DEC_column is DEC_main Puts NaNs in for no matches c_graph plots diagnostic graphs of the match - crashes if there are no matches """ import numpy as np import match_lists import sqlutil import ConfigParser config_file = "/home/sr525/Python_Code/srpylib/srpylib.cfg" config = ConfigParser.RawConfigParser() config.read(config_file) uname = config.get("wsdb", "uname") pword = config.get("wsdb", "pword") host = config.get("wsdb", "host") if w_units == "arcsecs": width = width / 3600.0 cols = ["RA_WISE", "DEC_WISE", "W1", "W1_ERR", "W2", "W2_ERR", "W3", "W3_ERR", "W4", "W4_ERR"] try: for col in cols: t.add_empty_column(col, dtype = np.float64) except AttributeError as e: for col in cols: t[col] = np.zeros(len(t)) n = 0 if len(t) > 5000 and len(t) < 100000000: ra_min, ra_max, dec_min, dec_max = min_max_match(t[RA_main], t[DEC_main], \ width, DES_box = DES_box) print "Retreiving data" query_wise = "select ra, dec, w1mpro, w1sigmpro, w2mpro, w2sigmpro, " + \ "w3mpro, w3sigmpro, w4mpro, w4sigmpro FROM " + \ "allwise.main WHERE ra > " + ra_min + " and ra < " + \ ra_max + " and dec > " + dec_min + " and dec < " + dec_max RA_wise, DEC_wise, W1, W1err, W2, W2err, W3, W3err, W4, W4err = \ sqlutil.get(query_wise, db="wsdb", host=host, user=uname, password=pword) info = [RA_wise, DEC_wise, W1, W1err, W2, W2err, W3, W3err, W4, W4err] if len(RA_wise) == 0: for (i, col) in enumerate(cols): t[col] = [np.float64(np.nan)]*len(t) else: print "Retreived data, matching lists" dists, inds = match_lists.match_lists(t[RA_main], t[DEC_main], RA_wise, DEC_wise, width, 1) print "Matched lists, updating table" match = np.where( (inds <> len(RA_wise)) )[0] no_match = np.where( (inds == len(RA_wise)) )[0] for (i, col) in enumerate(cols): t[col][match] = info[i][inds[match]] t[col][no_match] = np.float64(np.nan) print "Table updated" elif len(t) <= 5000: while n < len(t): RA = t[RA_main][n] DEC = t[DEC_main][n] query = "select ra, dec, w1mpro, w1sigmpro, w2mpro, w2sigmpro, " + \ "w3mpro, w3sigmpro, w4mpro, w4sigmpro FROM allwise.main " + \ "WHERE q3c_radial_query(ra, dec, " + str(RA) + ", " + \ str(DEC) + ", 0.1002)" RA_W, DEC_W, W1, W1err, W2, W2err, W3, W3err, W4, W4err = \ sqlutil.get(query, db='wsdb',host=host, user=uname, password=pword) if len(RA_W) > 0: dists, inds = match_lists.match_lists([RA], [DEC], RA_W, DEC_W, width, 1) j = inds[0] if j <> len(RA_W): info = [RA_W[j], DEC_W[j], W1[j], W1err[j], W2[j], W2err[j], \ W3[j], W3err[j], W4[j], W4err[j]] for (i, col) in enumerate(cols): t[col][n] = info[i] else: for (i, col) in enumerate(cols): t[col][n] = np.float64(np.nan) else: for (i, col) in enumerate(cols): t[col][n] = np.float64(np.nan) n += 1 elif len(t) > 1000000000: print "Splitting by tile" tiles_all = t["TILENAME"] tiles = set(tiles_all) tiles = list(tiles) print len(tiles), "distinct tiles" t.sort("TILENAME") tiles.sort() n = 0 m = 0 rows = [] tile_rows = [] while n < len(t): tile = tiles[m] if t["TILENAME"][n] == tile: rows.append(n) n += 1 else: tile_rows.append(rows) rows = [] m += 1 for rows in tile_rows: t1 = t.rows(rows) print "Matching", t1["TILENAME"][0], "with length", len(t1) t1 = WISE_match(t1, RA_main, DEC_main, width, c_graph = False) if tile_rows.index(rows) == 0: t_out = t1 else: t_out.append(t1) t = t_out if c_graph: width = width *3600.0 RA_med, DEC_med = check_graph(t, RA_main, DEC_main, "WISE", width, gtype = "square") return t
def SDSS_match(t, RA_main, DEC_main, width = 0.0004, w_units = "degrees", c_graph = True, dr = "dr9", DES_box = False): """ Matches to SDSS Data from WSDB Table used is dr + .phototag DES_box returns all objects within a box of about the size of a DES tile centred on the median RA and DEC of the table. Iterates through a table where the RA column is RA_main and the DEC_column is DEC_main Puts NaNs in for no matches c_graph plots diagnostic graphs of the match - crashes if there are no matches """ import numpy as np import match_lists import sqlutil from astropy.table import Table, vstack import ConfigParser config_file = "/home/sr525/Python_Code/srpylib/srpylib.cfg" config = ConfigParser.RawConfigParser() config.read(config_file) uname = config.get("wsdb", "uname") pword = config.get("wsdb", "pword") host = config.get("wsdb", "host") if w_units == "arcsecs": width = width / 3600.0 cols = ["RA_SDSS", "DEC_SDSS", "MAG_PSF_U_SDSS", "MAGERR_PSF_U_SDSS", \ "MAG_PSF_G_SDSS", "MAGERR_PSF_G_SDSS", "MAG_PSF_R_SDSS", \ "MAGERR_PSF_R_SDSS", "MAG_PSF_I_SDSS", "MAGERR_PSF_I_SDSS", \ "MAG_PSF_Z_SDSS", "MAGERR_PSF_Z_SDSS", "MAG_MODEL_U_SDSS", \ "MAGERR_MODEL_U_SDSS", "MAG_MODEL_G_SDSS", "MAGERR_MODEL_G_SDSS", \ "MAG_MODEL_R_SDSS", "MAGERR_MODEL_R_SDSS", "MAG_MODEL_I_SDSS", \ "MAGERR_MODEL_I_SDSS", "MAG_MODEL_Z_SDSS", "MAGERR_MODEL_Z_SDSS", "TYPE"] try: for col in cols: t.add_empty_column(col, dtype = np.float64) except AttributeError as e: for col in cols: t[col] = np.zeros(len(t)) n = 0 if len(t) > 100 and len(t) < 1000000: print "Finding max and mins" ra_min, ra_max, dec_min, dec_max = min_max_match(t[RA_main], t[DEC_main], \ width, DES_box = DES_box) print "Retreiving data" query_sdss = "select ra, dec, psfMag_u, psfMagErr_u, psfMag_g, " + \ "psfMagErr_g, psfMag_r, psfMagErr_r, psfMag_i, psfMagErr_i, " + \ "psfMag_z, psfMagErr_z, modelMag_u, modelMagErr_u, " + \ "modelMag_g, modelMagErr_g, modelMag_r, modelMagErr_r, " + \ "modelMag_i, modelMagErr_i, modelMag_z, modelMagErr_z, " + \ "type FROM sdss" + dr + ".phototag WHERE ra > " + ra_min + \ " and ra < " + ra_max + " and dec > " + dec_min + \ " and dec < " + dec_max RA_sdss, DEC_sdss, psfu, psfuerr, psfg, psfgerr, psfr, psfrerr, psfi, \ psfierr, psfz, psfzerr, modelu, modeluerr, modelg, modelgerr, modelr, \ modelrerr, modeli, modelierr, modelz, modelzerr, types = \ sqlutil.get(query_sdss, db = "wsdb", host = host, user = uname, password = pword) info = [RA_sdss, DEC_sdss, psfu, psfuerr, psfg, psfgerr, psfr, psfrerr, \ psfi, psfierr, psfz, psfzerr, modelu, modeluerr, modelg, modelgerr, \ modelr, modelrerr, modeli, modelierr, modelz, modelzerr, types] if len(RA_sdss) == 0: for (i, col) in enumerate(cols): t[col] = [np.float64(np.nan)]*len(t) else: print "Retreived data, matching lists" dists, inds = match_lists.match_lists(t[RA_main], t[DEC_main], RA_sdss, DEC_sdss, width, 1) print "Matched lists, updating table" match = np.where( (inds <> len(RA_sdss)) )[0] no_match = np.where( (inds == len(RA_sdss)) )[0] for (i, col) in enumerate(cols): t[col][match] = info[i][inds[match]] t[col][no_match] = np.float64(np.nan) print "Table updated" elif len(t) <= 100: while n < len(t): RA = t[RA_main][n] DEC = t[DEC_main][n] print RA, DEC query_sdss = "select ra, dec, psfMag_u, psfMagErr_u, psfMag_g, " + \ "psfMagErr_g, psfMag_r, psfMagErr_r, psfMag_i, " + \ "psfMagErr_i, psfMag_z, psfMagErr_z, modelMag_u, " + \ "modelMagErr_u, modelMag_g, modelMagErr_g, modelMag_r, " + \ "modelMagErr_r, modelMag_i, modelMagErr_i, modelMag_z, " + \ "modelMagErr_z, type FROM sdss" + dr + \ ".phototag WHERE q3c_radial_query(ra, dec, " + str(RA) + \ ", " + str(DEC) + ", 0.1002)" RA_sdss, DEC_sdss, psfu, psfuerr, psfg, psfgerr, psfr, psfrerr, psfi, \ psfierr, psfz, psfzerr, modelu, modeluerr, modelg, modelgerr, modelr, \ modelrerr, modeli, modelierr, modelz, modelzerr, stype = \ sqlutil.get(query_sdss, db = "wsdb", host = host, user = uname, password = pword) if len(RA_sdss) > 0: dist, inds = match_lists.match_lists([RA], [DEC], RA_sdss, DEC_sdss, width, 1) j = inds[0] info = [RA_sdss[j], DEC_sdss[j], psfu[j], psfuerr[j], psfg[j], \ psfgerr[j], psfr[j], psfrerr[j], psfi[j], psfierr[j], \ psfz[j], psfzerr[j], modelu[j], modeluerr[j], modelg[j], \ modelgerr[j], modelr[j], modelrerr[j], modeli[j], \ modelierr[j], modelz[j], modelzerr[j], stype[j]] for (i, col) in enumerate(cols): t[col][n] = info[i] else: for (i, col) in enumerate(cols): t[col][n] = np.float64(np.nan) n += 1 elif len(t) > 1000000: print "Splitting by tile" tiles_all = t["TILENAME"] tiles = set(tiles_all) tiles = list(tiles) print len(tiles), "distinct tiles" t.sort("TILENAME") tiles.sort() n = 0 m = 0 rows = [] tile_rows = [] while n < len(t): tile = tiles[m] if t["TILENAME"][n] == tile: rows.append(n) n += 1 else: print len(rows) tile_rows.append(rows) rows = [] m += 1 for rows in tile_rows: try: t1 = t.rows(rows) except AttributeError: t1 = t[rows] print "Matching", t1["TILENAME"][0], "with length", len(t1) t1 = SDSS_match(t1, RA_main, DEC_main, width, c_graph = False) if tile_rows.index(rows) == 0: t_out = t1 else: try: t_out.append(t1) except AttributeError: t_out = vstack([t_out, t1]) t = t_out if c_graph: width = width *3600.0 RA_med, DEC_med = check_graph(t, RA_main, DEC_main, "SDSS", width, gtype = "square") return t
def DES_match(t, RA_main, DEC_main, c_graph = True, width = 0.0004, w_units = "degrees", release = "Y1A1"): """ Matches to DES data ingested into WSDB Tables used are des_y1a1.coadd_objects and des_y2q1.objects Iterates through a table where the RA column is RA_main and the DEC_column is DEC_main Puts NaNs in for no matches c_graph plots diagnostic graphs of the match - crashes if there are no matches """ import numpy as np import match_lists import sqlutil from astropy.table import Table, vstack import ConfigParser config_file = "/home/sr525/Python_Code/srpylib/srpylib.cfg" config = ConfigParser.RawConfigParser() config.read(config_file) uname = config.get("wsdb", "uname") pword = config.get("wsdb", "pword") host = config.get("wsdb", "host") if w_units == "arcsecs": width = width / 3600.0 if release == "Y1A1": cols = ["COADD_OBJECTS_ID", "RA_DES_Y1A1", "DEC_DES_Y1A1", "TILENAME_Y1A1", \ "RUN_Y1A1", "MPSF_G_Y1A1", "MPSF_ERR_G_Y1A1", "MPSF_R_Y1A1", \ "MPSF_ERR_R_Y1A1", "MPSF_I_Y1A1", "MPSF_ERR_I_Y1A1", "MPSF_Z_Y1A1", \ "MPSF_ERR_Z_Y1A1", "MPSF_Y_Y1A1", "MPSF_ERR_Y_Y1A1", "MAPER3_G_Y1A1", \ "MAPER3_ERR_G_Y1A1", "MAPER3_R_Y1A1", "MAPER3_ERR_R_Y1A1", \ "MAPER3_I_Y1A1", "MAPER3_ERR_I_Y1A1", "MAPER3_Z_Y1A1", \ "MAPER3_ERR_Z_Y1A1", "MAPER3_Y_Y1A1", "MAPER3_ERR_Y_Y1A1"] str_cols = ["TILENAME_Y1A1", "RUN_Y1A1"] elif release == "Y2Q1": cols = ["QUICK_OBJECT_ID", "RA_DES_Y2Q1", "DEC_DES_Y2Q1", "MPSF_G_Y2Q1", \ "MPSF_ERR_G_Y2Q1", "MPSF_R_Y2Q1", "MPSF_ERR_R_Y2Q1", "MPSF_I_Y2Q1", \ "MPSF_ERR_I_Y2Q1", "MPSF_Z_Y2Q1", "MPSF_ERR_Z_Y2Q1", "MPSF_Y_Y2Q1", \ "MPSF_ERR_Y_Y2Q1", "MAUTO_G_Y2Q1", "MAUTO_ERR_G_Y2Q1", \ "MAUTO_R_Y2Q1", "MAUTO_ERR_R_Y2Q1", "MAUTO_I_Y2Q1", \ "MAUTO_ERR_I_Y2Q1", "MAUTO_Z_Y2Q1", "MAUTO_ERR_Z_Y2Q1", \ "MAUTO_Y_Y2Q1", "MAUTO_ERR_Y_Y2Q1"] str_cols = [] try: for col in cols: if col not in str_cols: t.add_empty_column(col, dtype = np.float64) else: t.add_empty_col(col, dtype = np.string) except AttributeError as e: for col in cols: if col not in str_cols: t[col] = np.zeros(len(t)) else: t[col] = ["AAAAAAAAAAAAAAAAAAAAAAAAAA"] ra_min, ra_max, dec_min, dec_max = min_max_match(t[RA_main], t[DEC_main], \ width, DES_box = False) area = (float(ra_max)-float(ra_min))*(float(dec_max)-float(dec_min)) print area if len(t) > 1000 and area < 2000: ra_min, ra_max, dec_min, dec_max = min_max_match(t[RA_main], t[DEC_main], \ width, DES_box = False) print "Retreiving data" if release == "Y1A1": query_des = "select COADD_OBJECTS_ID, RA, DEC, TILENAME, RUN, MAG_PSF_G, \ MAGERR_PSF_G, MAG_PSF_R, MAGERR_PSF_R, MAG_PSF_I, MAGERR_PSF_I, \ MAG_PSF_Z, MAGERR_PSF_Z, MAG_PSF_Y, MAGERR_PSF_Y, MAG_APER_3_G, \ MAGERR_APER_3_G, MAG_APER_3_R, MAGERR_APER_3_R, MAG_APER_3_I, \ MAGERR_APER_3_I, MAG_APER_3_Z, MAGERR_APER_3_Z, MAG_APER_3_Y, \ MAGERR_APER_3_Y FROM des_y1a1.coadd_objects WHERE \ ra > " + ra_min + " and ra < " + ra_max + " and dec > " + \ dec_min + " and dec < " + dec_max co_id, RA_des, DEC_des, tile, run, G, Gerr, R, Rerr, I, Ierr, Z, Zerr, Y, \ Yerr, Ga, Gaerr, Ra, Raerr, Ia, Iaerr, Za, Zaerr, Ya, Yaerr = \ sqlutil.get(query_des, db="wsdb", host=host, user=uname, password=pword) elif release == "Y2Q1": query_des = "select QUICK_OBJECT_ID, RA, DEC, MAG_PSF_G, MAGERR_PSF_G, \ MAG_PSF_R, MAGERR_PSF_R, MAG_PSF_I, MAGERR_PSF_I, MAG_PSF_Z, \ MAGERR_PSF_Z, MAG_PSF_Y, MAGERR_PSF_Y, MAG_AUTO_G, \ MAGERR_AUTO_G, MAG_AUTO_R, MAGERR_AUTO_R, MAG_AUTO_I, \ MAGERR_AUTO_I, MAG_AUTO_Z, MAGERR_AUTO_Z, MAG_AUTO_Y, \ MAGERR_AUTO_Y FROM des_y2q1.objects WHERE ra > " + ra_min \ + " and ra < " + ra_max + " and dec > " + dec_min + \ " and dec < " + dec_max q_id, RA_des, DEC_des, G, Gerr, R, Rerr, I, Ierr, Z, Zerr, Y, Yerr, Ga, \ Gaerr, Ra, Raerr, Ia, Iaerr, Za, Zaerr, Ya, Yaerr = \ sqlutil.get(query_des, db="wsdb", host=host, user=uname, password=pword) print "Retreived data" if len(RA_des) == 0: for (i, col) in enumerate(cols): t[col] = [np.float64(np.nan)]*len(t) else: dists, inds = match_lists.match_lists(t[RA_main], t[DEC_main], RA_des, DEC_des, width, 1) if release == "Y1A1": info = [co_id, RA_des, DEC_des, tile, run, G, Gerr, R, Rerr, I, \ Ierr, Z, Zerr, Y, Yerr, Ga, Gaerr, Ra, Raerr, Ia, Iaerr, Za, Zaerr, Ya, Yaerr] elif release == "Y2Q1": info = [q_id, RA_des, DEC_des, G, Gerr, R, Rerr, I, Ierr, Z, Zerr, \ Y, Yerr, Ga, Gaerr, Ra, Raerr, Ia, Iaerr, Za, Zaerr, Ya, Yaerr] match = np.where( (inds <> len(RA_des)) )[0] no_match = np.where( (inds == len(RA_des)) )[0] for (i, col) in enumerate(cols): t[col][match] = info[i][inds[match]] t[col][no_match] = np.float64(np.nan) if len(t) <= 1000: n = 0 while n < len(t): RA = t[RA_main][n] DEC = t[DEC_main][n] if release == "Y1A1": query_des = "select COADD_OBJECTS_ID, RA, DEC, TILENAME, RUN, MAG_PSF_G, \ MAGERR_PSF_G, MAG_PSF_R, MAGERR_PSF_R, MAG_PSF_I, MAGERR_PSF_I, \ MAG_PSF_Z, MAGERR_PSF_Z, MAG_PSF_Y, MAGERR_PSF_Y, MAG_APER_3_G, \ MAGERR_APER_3_G, MAG_APER_3_R, MAGERR_APER_3_R, MAG_APER_3_I, \ MAGERR_APER_3_I, MAG_APER_3_Z, MAGERR_APER_3_Z, MAG_APER_3_Y, \ MAGERR_APER_3_Y FROM des_y1a1.coadd_objects WHERE \ q3c_radial_query(ra, dec, " + str(RA) + ", " + str(DEC) + ", 0.1102)" co_id, RA_des, DEC_des, tile, run, G, Gerr, R, Rerr, I, Ierr, Z, \ Zerr, Y, Yerr, Ga, Gaerr, Ra, Raerr, Ia, Iaerr, Za, Zaerr, Ya, Yaerr = \ sqlutil.get(query_des, db="wsdb", host=host, user=uname, password=pword) elif release == "Y2Q1": query_des = "select QUICK_OBJECT_ID, RA, DEC, MAG_PSF_G, MAGERR_PSF_G, MAG_PSF_R, \ MAGERR_PSF_R, MAG_PSF_I, MAGERR_PSF_I, MAG_PSF_Z, \ MAGERR_PSF_Z, MAG_PSF_Y, MAGERR_PSF_Y, MAG_AUTO_G, \ MAGERR_AUTO_G, MAG_AUTO_R, MAGERR_AUTO_R, MAG_AUTO_I, \ MAGERR_AUTO_I, MAG_AUTO_Z, MAGERR_AUTO_Z, MAG_AUTO_Y, \ MAGERR_AUTO_Y FROM des_y2q1.objects WHERE \ q3c_radial_query(ra, dec, " + str(RA) + ", " + str(DEC) + ", 0.1102)" q_id, RA_des, DEC_des, G, Gerr, R, Rerr, I, Ierr, Z, Zerr, Y, Yerr, \ Ga, Gaerr, Ra, Raerr, Ia, Iaerr, Za, Zaerr, Ya, Yaerr = \ sqlutil.get(query_des, db="wsdb", host=host, user=uname, password=pword) if len(RA_des) > 0: dists, inds = match_lists.match_lists([RA], [DEC], RA_des, DEC_des, width, 1) j = inds[0] if j <> len(RA_des): if release == "Y1A1": info = [co_id[j], RA_des[j], DEC_des[j], tile[j], run[j], G[j], \ Gerr[j], R[j], Rerr[j], I[j], Ierr[j], Z[j], \ Zerr[j], Y[j], Yerr[j], Ga[j], Gaerr[j], Ra[j], \ Raerr[j], Ia[j], Iaerr[j], Za[j], Zaerr[j], Ya[j], Yaerr[j]] if release == "Y2Q1": info = [q_id[j], RA_des[j], DEC_des[j], G[j], Gerr[j], R[j], Rerr[j], \ I[j], Ierr[j], Z[j], Zerr[j], Y[j], Yerr[j], Ga[j], \ Gaerr[j], Ra[j], Raerr[j], Ia[j], Iaerr[j], Za[j], \ Zaerr[j], Ya[j], Yaerr[j]] for (k, col) in enumerate(cols): t[col][n] = info[k] else: for (k, col) in enumerate(cols): t[col][n] = np.float64(np.nan) else: for (k, col) in enumerate(cols): t[col][n] = np.float64(np.nan) n += 1 print "Done DES" if area > 2000 and len(t) > 1000: t.sort(RA_main) l1 = np.ceil(area/1001.0) l = len(t)/l1 i = 0 while i < l1: if (i+1)*l < len(t): t1 = t[i*l:(i+1)*l] else: t1 = t[i*l:] t1 = DES_match(t1, RA_main, DEC_main, c_graph = False, width = width, \ w_units = w_units, release = release) if i == 0: t_out = t1 else: t_out = vstack([t_out, t1]) i += 1 t = t_out if c_graph: width = width *3600.0 if release == "Y1A1": survey = "DES_Y1A1" elif release == "Y2Q1": survey = "DES_Y2Q1" RA_med, DEC_med = check_graph(t, RA_main, DEC_main, survey, width) return t
def UKIDSS_match(t, RA_main, DEC_main, c_graph = True, width = 0.0004, w_units = "degrees", DES_box = False): """ Matches to VHS Data from WSDB Tables used are vhs_1504. + survey DES_box returns all objects within a box of about the size of a DES tile centred on the median RA and DEC of the table. Iterates through a table where the RA column is RA_main and the DEC_column is DEC_main Puts NaNs in for no matches c_graph plots diagnostic graphs of the match - crashes if there are no matches Can specify not the full number of bands """ import numpy as np import match_lists import sqlutil from astropy.table import Table, vstack import ConfigParser config_file = "srpylib.cfg" uname = config.get("wsdb", uname) pword = config.get("wsdb", pword) host = config.get("wsdb", host) if w_units == "arcsecs": width = width / 3600.0 cols = ["RA_UKIDSS", "DEC_UKIDSS", "Y_UKIDSS", "Y_ERR_UKIDSS", "J_UKIDSS", \ "J_ERR_UKIDSS", "H_UKIDSS", "H_ERR_UKIDSS", "K_UKIDSS", "K_ERR_UKIDSS"] try: for col in cols: t.add_empty_column(col, dtype = np.float64) except AttributeError as e: for col in cols: t[col] = np.zeros(len(t)) ra_min, ra_max, dec_min, dec_max = min_max_match(t[RA_main], t[DEC_main], \ width, DES_box = DES_box) area = (float(ra_max)-float(ra_min))*(float(dec_max)-float(dec_min)) print area if len(t) > 1000 and area < 2000: ra_min, ra_max, dec_min, dec_max = min_max_match(t[RA_main], t[DEC_main], \ width, DES_box = DES_box) print "Retreiving data" query_ukidss = "select ra, dec, yAperMag3, yAperMag3Err, j_1AperMag3, \ j_1AperMag3Err, hAperMag3, hAperMag3Err, kAperMag3, \ kAperMag3Err FROM ukidssdr10.lassource \ WHERE ra > " + ra_min + " and ra < " + ra_max + " and \ dec > " + dec_min + " and dec < " + dec_max RA_u, DEC_u, Y, Yerr, J, Jerr, H, Herr, K, Kerr = \ sqlutil.get(query_ukidss, db="wsdb", host=host, user=uname, password=pword) print "Retreived data" if len(RA_u) == 0: for (i, col) in enumerate(cols): t[col] = [np.float64(np.nan)]*len(t) else: dists, inds = match_lists.match_lists(t[RA_main], t[DEC_main], RA_u, DEC_u, width, 1) info = [RA_u, DEC_u, Y, Yerr, J, Jerr, H, Herr, K, Kerr] match = np.where( (inds <> len(RA_u)) )[0] no_match = np.where( (inds == len(RA_u)) )[0] for (i, col) in enumerate(cols): t[col][match] = info[i][inds[match]] t[col][no_match] = np.float64(np.nan) if len(t) <= 1000: n = 0 while n < len(t): RA = t[RA_main][n] DEC = t[DEC_main][n] query_ukidss = "select ra, dec, yAperMag3, yAperMag3Err, j_1AperMag3, \ j_1AperMag3Err, hAperMag3, hAperMag3Err, kAperMag3, \ kAperMag3Err FROM ukidssdr10.lassource WHERE \ q3c_radial_query(ra, dec, " + str(RA) + ", " + \ str(DEC) + ", 0.1102)" RA_u, DEC_u, Y, Yerr, J, Jerr, H, Herr, K, Kerr = \ sqlutil.get(query_ukidss, db="wsdb", host=host, user=uname, password=pword) if len(RA_u) > 0: dists, inds = match_lists.match_lists([RA], [DEC], RA_u, DEC_u, width, 1) j = inds[0] if j <> len(RA_u): info = [RA_u[j], DEC_u[j], Y[j], Yerr[j], J[j], Jerr[j], \ H[j], Herr[j], K[j], Kerr[j]] for (k, col) in enumerate(cols): t[col][n] = info[k] else: for (k, col) in enumerate(cols): t[col][n] = np.float64(np.nan) else: for (k, col) in enumerate(cols): t[col][n] = np.float64(np.nan) n += 1 print "Done UKIDSS" if area > 2000 and len(t) > 1000: t.sort(RA_main) l1 = np.ceil(area/1001.0) l = len(t)/l1 i = 0 while i < l1: if (i+1)*l < len(t): t1 = t[i*l:(i+1)*l] else: t1 = t[i*l:] t1 = UKIDSS_match(t1, RA_main, DEC_main, c_graph = c_graph, width = width, \ w_units = w_units, DES_box = DES_box) if i == 0: t_out = t1 else: t_out = vstack([t_out, t1]) i += 1 t = t_out if c_graph: width = width *3600.0 RA_med, DEC_med = check_graph(t, RA_main, DEC_main, "UKIDSS", width) return t
if vmin is None: vmin = 0 if vmax is None: vmax = hh.max() if weight_norm: hh = hh * 1. / (hhsum + 1 * (hhsum == 0)) if skip_empty: hh1 = hh[hhsum > 0] else: hh1 = hh coll.set_array(hh1) coll.set_cmap(cmap) coll.set_clim(vmin, vmax) coll.set_edgecolors(coll.cmap(coll.norm(hh1))) coll.set_rasterized(rasterized) tmpcoll.set_clim(vmin, vmax) ax.add_collection(coll) return coll if __name__ == '__main__': ras, decs = sqlutil.get('select radeg,dedeg from sdss_phot_qso.main', host='cappc118') doit(ras, decs, ramin=0, ramax=360, decmin=-90, decmax=90, nside=64) plt.savefig('xx.png', dpi=200)
def VIKING_match(t, RA_main, DEC_main, width = 0.0004, w_units = "degrees", c_graph = True, DES_box = False): """ Matches to Viking Data from WSDB Table used is viking_201207.main DES_box returns all objects within a box of about the size of a DES tile centred on the median RA and DEC of the table. Iterates through a table where the RA column is RA_main and the DEC_column is DEC_main Puts NaNs in for no matches c_graph plots diagnostic plot of the match - crashes if there are no matches """ import numpy as np import match_lists import sqlutil import ConfigParser config_file = "srpylib.cfg" uname = config.get("wsdb", uname) pword = config.get("wsdb", pword) host = config.get("wsdb", host) if w_units == "arcsecs": width = width / 3600.0 cols = ["RA_VIKING", "DEC_VIKING", "VIKING_Y", "VIKING_J"] try: for col in cols: t.add_empty_column(col, dtype = np.float64) except AttributeError as e: for col in cols: t[col] = np.zeros(len(t)) n = 0 if len(t) > 1000 and len(t) < 1000000: ra_min, ra_max, dec_min, dec_max = min_max_match(t[RA_main], t[DEC_main], \ width, DES_box = DES_box) print "Retreiving data" query_V = "select ra, dec, mag_y, mag_j FROM viking_201207.main WHERE ra > "\ + ra_min + " and ra < " + ra_max + " and dec > " + \ dec_min + " and dec < " + dec_max RA_V, DEC_V, Y_V, J_V = sqlutil.get(query_V, db = "wsdb", host = host,\ user = uname, password = pword) if len(RA_wise) == 0: for (i, col) in enumerate(cols): t[col] = [np.float64(np.nan)]*len(t) else: info = [RA_V, DEC_V, Y_V, J_V] print "Retreived data, matching lists" dists, inds = match_lists.match_lists(t[RA_main], t[DEC_main], \ RA_V, DEC_V, width, 1) print "Matched lists, updating table" match = np.where( (inds <> len(RA_V)) )[0] no_match = np.where( (inds == len(RA_V)) )[0] for (i, col) in enumerate(cols): t[col][match] = info[i][inds[match]] t[col][no_match] = np.float64(np.nan) print "Table updated" elif len(t) <= 1000: while n < len(t): RA = t[RA_main][n] DEC = t[DEC_main][n] query = "select ra, dec, mag_y, mag_j FROM viking_201207.main WHERE " + \ "q3c_radial_query(ra, dec, " + str(RA) + \ ", " + str(DEC) + ", 0.1002)" RA_V, DEC_V, Y_V, J_V = sqlutil.get(query, db='wsdb',host=host, \ user = uname, password=pword) if len(RA_V) > 0: dists, inds = match_lists.match_lists([RA], [DEC], RA_V, DEC_V, width, 1) j = inds[0] info = [RA_V, DEC_V, Y_V, J_V] for (i, col) in enumerate(cols): t[col][n] = info[i] else: for (i, col) in enumerate(cols): t[col][n] = np.float64(np.nan) n += 1 elif len(t) > 1000000: print "Splitting by tile" tiles_all = t["TILENAME"] tiles = set(tiles_all) tiles = list(tiles) print len(tiles), "distinct tiles" t.sort("TILENAME") tiles.sort() n = 0 m = 0 rows = [] tile_rows = [] while n < len(t): tile = tiles[m] if t["TILENAME"][n] == tile: rows.append(n) n += 1 else: tile_rows.append(rows) rows = [] m += 1 for rows in tile_rows: t1 = t.rows(rows) print "Matching", t1["TILENAME"][0], "with length", len(t1) t1 = VIKING_match(t1, RA_main, DEC_main, width, c_graph = False) print "Returning table of length:", len(t1) if tile_rows.index(rows) == 0: t_out = t1 else: t_out.append(t1) t = t_out if c_graph: width = width *3600.0 RA_med, DEC_med = check_graph(t, RA_main, DEC_main, "VIKING", width, gtype = "square") return t
query = sqlquery if debug: print(query) for iline, line in enumerate(query.splitlines()): print(iline, ':', line) print('host:', host) print('query', len(query)) print(sqlparse.format(query, strip_comments=True)) print('getting data...') t1 = time.time() result = sqlutil.get(query, db=db, host=host, user=user, password=password, asDict=True) querytime = time.time() - t1 if USING_atpy: result = atpy.Table('postgres', query=query, user=user, database='wsdb', password=password, host='cappc127') if debug: help(result)
def get_coords(infile=None, coord_file=None, tilename=None): """ # Take DES tile, find all WISE objects on that tile. # infile = "/data/desardata/Y1A1/" + tile + "/" + tile + "_z.fits.fz" # DATAPATH = '/data/desardata/Y1A1/' """ import math logger = logging.getLogger() outpath = os.path.dirname(coord_file) if not os.path.exists(outpath): try: os.makedirs(outpath) except Exception as error: logger.exception(error) print("Unexpected error:", sys.exc_info()[0]) traceback.print_exc(file=sys.stdout) raise logger.debug('Reading tile image to get corners: %s', infile) with fits.open(infile) as fhlist: hdr = fhlist[1].header w = wcs.WCS(hdr, naxis=2) # If this is ever used for non square inmages this needs checking pix_corners = [[0, 0], [0, hdr["NAXIS1"]], [ hdr["NAXIS2"], 0], [hdr["NAXIS1"], hdr["NAXIS2"]]] w_corners = w.wcs_pix2world(pix_corners, 1) [corner_ra, corner_dec] = zip(*w_corners) """ corner_ra = sorted(corner_ra) corner_dec = sorted(corner_dec) #Take the slightly smaller square inside the not square image ra_min = corner_ra[1] ra_max = corner_ra[2] dec_min = corner_dec[1] dec_max = corner_dec[2] """ delta_ra = max(corner_ra) - min(corner_ra) delta_dec = max(corner_dec) - min(corner_dec) dec_mean = (min(corner_dec) + min(corner_dec)) / 2.0 delta_ra = delta_ra * math.cos(math.radians(dec_mean)) logger.debug('dec_mean: %s', dec_mean) logger.debug('delta_ra, delta_dec: %s; %s', delta_ra, delta_dec) # raw_input("Enter any key to continue: ") # Some of these objects won't be on the DES tile so will return infs ra_min = str(min(corner_ra)) ra_max = str(max(corner_ra)) dec_min = str(min(corner_dec)) dec_max = str(max(corner_dec)) query_wise = "select ra, dec FROM allwise.main WHERE ra > " + \ ra_min + \ " and ra < " + ra_max + " and dec > " + \ dec_min + " and dec < " + dec_max # special case where crossing 24hrs ra > ra_max or ra < ra_min if delta_ra > 180.0: query_wise = "select ra, dec FROM allwise.main WHERE (ra > " + \ ra_max + \ " or ra < " + ra_min + ") and dec > " + \ dec_min + " and dec < " + dec_max db, host, user, password, db_table = rd_config_wsdb(table='wise') RA_wise, DEC_wise = sqlutil.get( query_wise, db=db, host=host, user=user, password=password ) logger.info('%s: Number of wise sources: %s', tilename, len(RA_wise)) info = "" for (ra, dec) in zip(RA_wise, DEC_wise): info += (str(ra) + " " + str(dec) + "\n") with open(coord_file, "w") as f: logger.debug('Writing coords: %s', coord_file) try: f.write(info) logger.debug('Successfully written coordinate file: %s', coord_file) except IOError as e: print("I/O error({0}): {1}".format(e.errno, e.strerror)) except: print("Unexpected error:", sys.exc_info()[0]) raise # except IOError as (errno, strerror): # print("I/O error({0}): {1}".format(errno, strerror)) # print('Failed to write coordinate file: ' + str(e)) # logger.error('Failed to write coordinate file: ' + str(e)) # sys.exit() # pass # except OSError as (errno, strerror): # print("Hello World:OS error({0}): {1}".format(errno, strerror)) # pass # except: # print("Unexpected error:", sys.exc_info()[0]) # # traceback.print_exc(file=sys.stdout) # raise return coord_file