def flux_match(ra, dec, t_all, RA_main, DEC_main, l, width, RA_all, DEC_all, bands): """ Does something with fluxes and I'm not sure what or why I wrote this """ from astropy.table import Table import match_lists t_all.sort([RA_all, DEC_all]) dist, ind = match_lists.match_lists(ra, dec, t_all[RA_all], t_all[DEC_all], width) i = ind[0] inds = range(i-l, i+l) t_short = t_all[inds] mc = 0 mags = [] fs = [] ffs = [] fmags = [] for band in bands: mag = t_all["MAG_APER_5_" + band][i] flux = t_all["FLUX_APER_5_" + band][i] mc += 1 mags.append(mag) fs.append(flux) if mc == 5: fmags = mags ffs = fs return ffs, fmags
def add_DEScat(tile, outdir, filename_suffix='_WISEfp',
radius_match=5.0,
overwrite=False):
from astropy.table import Table, hstack
import numpy as np
import match_lists
config = configparser.RawConfigParser()
config.read(config_file)
release = config.get("des", "release")
infile = datapath + tile + "/" + tile + ".fits"
logger.debug('Read: %s', infile)
t_cat = Table.read(infile)
t_fp = Table.read(outdir + tile + "_WISEfp.fits")
dists, inds = match_lists.match_lists(
t_cat["ALPHAWIN_J2000_G"].data, t_cat["DELTAWIN_J2000_G"].data,
t_fp["RA_CALC_G"].data, t_fp["DEC_CALC_G"].data,
radius_match / 3600.0, 1)
ids = np.where((inds != len(t_fp)))[0]
t_cat = t_cat[ids]
t_fp = t_fp[inds[ids]]
t_out = hstack([t_fp, t_cat])
t_out.write(outdir + tile + "_WISEfp_DEScat.fits",
overwrite=overwrite)
def DES_tile_run(ras, decs, release = "SVA1"):
"""
Takes a list of ras and decs and finds the possible tiles the objects might be
on
Returns a table of RA_IN and DEC_IN with potential TILENAME and RUN
Designed to then be passed to DES_data to get object details
"""
from astropy.table import Table
import numpy as np
import match_lists
t = Table([ras, decs], names = ("RA_IN", "DEC_IN"))
tiles = Table.read("/data/desardata/" + release + "/InfoTables/" + release + "_Tilename_Run_COADD.fits")
tile_ras = []
tile_decs = []
for tile in tiles["TILENAME"]:
ra = float(tile[3:5])/24.0*360.0 + float(tile[5:7])/(24.0*60.0)*360.0
if tile[7] == "-":
dec = float(tile[7:10]) - float(tile[10:])/60.0
else:
dec = float(tile[7:10]) + float(tile[10:])/60.0
tile_ras.append(ra)
tile_decs.append(dec)
dists, inds = match_lists.match_lists(ras, decs, tile_ras, tile_decs, 0.5, numNei = 5)
l = len(list(set(inds[inds < len(tile_ras)].flatten())))
ids = np.where( (inds <> len(tile_ras)) )[0]
l = len(ids)
t = Table([["DES0000-0000"]*l, ["00000000000000_DES0000-0000"]*l, [0.000000]*l, [0.000000]*l], names = ("TILENAME", "RUN", "RA_IN", "DEC_IN"))
n = 0
i = 0
ids = np.where( (inds <> len(tile_ras)) )[0]
while n < len(ras):
ra = ras[n]
dec = decs[n]
for (dist, ind) in zip(dists[n], inds[n]):
if ind <> len(tile_ras):
t["RA_IN"][i] = ra
t["DEC_IN"][i] = dec
t["TILENAME"][i] = tiles["TILENAME"][ind]
t["RUN"][i] = tiles["RUN"][ind]
i += 1
n += 1
return t
def get_apm_cat(RA, DEC, db = "vst", aper_num = "3"):
"""
Finds the APM catalogue for a given RA and DEC
"""
import cutout_vista
import xmlrpclib
import os
import subprocess
import numpy as np
from astropy.table import Table, vstack, hstack
try:
files, apm_files = cutout_vista.querydb(RA, DEC, db=db, onlytiles=True, ctype='fit')
if files == []:
"Print no files returned"
except xmlrpclib.Fault as e:
if e.faultCode == 1:
#faultCode 1 is IOError
print "Data not found"
else:
raise e
n = 0
for filename in apm_files:
dir, file = os.path.split(filename[:-6])
file = "/" + file + "cat.fits"
subprocess.call(["/home/sr525/bash_scripts/scp_amp.bash", dir, file])
with fits.open("/tmp/" + file) as hlist:
th = hlist[1].header
td = hlist[1].data
td["RA"] = np.rad2deg(td["RA"])
td["DEC"] = np.rad2deg(td["DEC"])
dists, inds = match_lists.match_lists([RA], [DEC], td["RA"], td["DEC"], 10.1)
print dists, inds
if "177.A-3011" in hlist[0].header["HIERARCH ESO OBS PROG ID"]:
band = files[n][-8:-7]
print band
apcor = th["APCOR" + aper_num]
#pho_cal.
else:
print "Can't use"
return
n += 1
os.remove("/tmp" + file)
for file in files:
os.remove(file)
def DES_data(t, release = "SVA1"):
"""
Takes a table produced by DES_tile_run of ra, dec and possible TILENAME and RUN
Uses this to then search each tile for the potential DES match to the object.
Puts matching objects into a new table
"""
import os
import subprocess
import match_lists
from astropy.table import Table, vstack
if release == "SVA1":
rn = 0
elif release == "Y1P1":
rn = 1
elif release == "Y1A1":
rn = 2
n = 0
m = 0
for (ra, dec, tile, run) in zip(t["RA_IN"], t["DEC_IN"], t["TILENAME"], t["RUN"]):
print m, "out of", len(t)
if tile <> "DES0000-0000":
print tile
if os.path.exists("/data/desardata/" + release + "/" + tile + "/" + tile + ".fits"):
t_info = Table.read("/data/desardata/" + release + "/" + tile + "/" + tile + ".fits")
else:
subprocess.call(["bash", "/home/sr525/bash_scripts/tile_file.bash", release, tile])
t_info = Table.read("/data/desardata/" + release + "/" + tile + "/" + tile + ".fits")
dists, inds = match_lists.match_lists([ra], [dec], t_info["RA"], t_info["DEC"], 2.0/3600)
print dists, inds
if inds[0] <> len(t_info):
if n == 0:
t1 = Table(t_info[inds[0]])
n += 1
else:
if inds[0] <> len(t_info):
t_info = Table(t_info[inds[0]])
t1 = vstack([t1, t_info], join_type = "exact")
m += 1
try:
return t1
except UnboundLocalError as e:
print e
print "Returning Nothing"
return Table()
def test__match_lists__with_nonmatching_lists():
assert match_lists([{
'name': 'A'
}, {
'name': 'B',
'num': 8
}, {
'name': 'C'
}], [{
'name': 'H'
}, {
'name': 'A'
}], '/name') == ([{
'name': 'A'
}, None, None])
def add_kernel_DEScat(tile, outdir, kernel_path, suffix_for_kernelcats):
from astropy.table import Table, hstack
import numpy as np
import match_lists
import astropy.io.fits as fits
import subprocess
import os
config = ConfigParser.RawConfigParser()
config.read(config_file)
tile_header = fits.open('/data/desardata/Y1A1/'+str(tile)+'/'+str(tile)+'_g.fits.fz')[1].header
RA_centre = tile_header['CRVAL1']
DEC_centre = tile_header['CRVAL2']
colours = ["g","r","i","z","Y"]
for colour in colours:
if not os.path.exists('/data/desardata/Y1A1/'+str(tile)+'/'+str(tile)+'_'+str(colour)+'.fits'):
subprocess.call(['bash','/data/cl522/WISE2DES/kernelised_extraction/funpack.sh','/data/desardata/Y1A1/'+str(tile)+'/'+str(tile)+'_'+str(colour)+'.fits.fz'])
subprocess.call(['bash','/data/cl522/WISE2DES/kernelised_extraction/sextractor_script.sh',str(tile),str(RA_centre),str(DEC_centre),str(kernel_path),str(suffix_for_kernelcats),str(outdir)])
for (n, band) in enumerate(["G", "R", "I", "Z"]): #, "Y"]): removed because final fits file must have < 1000 columns :/
if n!=4:
t = Table.read(outdir+"/"+tile+"_"+band.lower()+"_cat"+suffix_for_kernelcats+".fits")
if n==4:
t = Table.read(outdir+"/"+tile+"_"+band+"_cat"+suffix_for_kernelcats+".fits")
for col in t.columns:
t.rename_column(col, col + "_" + band)
if n == 0:
t_cat = t
else:
t_cat = hstack([t_cat, t])
t_cat.write(outdir+"/"+tile+suffix_for_kernelcats+".fits", overwrite = True)
release = config.get("des", "release")
t_cat = Table.read(outdir+"/"+tile+suffix_for_kernelcats+".fits")
t_fp = Table.read(outdir + tile + "_WISEfp.fits")
dists, inds = match_lists.match_lists(t_cat["ALPHAWIN_J2000_G"].data, t_cat["DELTAWIN_J2000_G"].data, t_fp["RA_CALC_G"].data, t_fp["DEC_CALC_G"].data, 5.0/3600.0, 1)
ids = np.where( (inds <> len(t_fp)) )[0]
t_cat = t_cat[ids]
t_fp = t_fp[inds[ids]]
t_out = hstack([t_fp, t_cat])
t_out.write(outdir + "/" + tile + "_WISEfp_DEScat"+suffix_for_kernelcats+".fits", overwrite = True)
def merge(t, t1, suf1, col_ra, col_dec, col1_ra, col1_dec, radius = 0.0006, join = "inner"): old_cols = list(t1.columns) new_cols = [] for col in old_cols: new_cols.append(col + "_" + suf1) t1.rename_column(col, col + "_" + suf1) dists, inds = match_lists.match_lists(t[col_ra], t[col_dec], t1[col1_ra+ "_" + suf1], t1[col1_dec + "_" + suf1], radius) ms = np.where( inds <> len(t1) )[0] if join == "inner": t = t[ms] t1 = t1[inds[ms]] for col in new_cols: t[col] = t1[col] elif join == "outer": ws = np.where( inds == len(t1) )[0] tm = t[ms] t1m = t1[inds[ms]] for col in new_cols: tm[col] = t1m[col] tw = t[ws] for col in t1m.columns: tw[col] = [-99.0]*len(tw) for row in tw: tm.add_row(row) n = 0 ws1 = [] while n < len(t1): if n not in inds[ms]: ws1.append(n) n += 1 t1w = t1[ws1] for col in t.columns: ncol = Column([-99.0]*len(t1w), name = col) t1w.add_column(ncol, index = 0) #t1w[col] = [-99.0]*len(t1w) for row in t1w: tm.add_row(row) t = tm return t
def nearest_neighbour(tile, outdir, checkplots=False, saveplots=False):
"""
"""
from astropy.table import Table, hstack
import srpylib
import match_lists
import matplotlib.pyplot as plt
import numpy as np
db, host, user, password, db_table = rd_config_wsdb(table='wise')
t = Table.read(outdir + tile + "_WISEfp_DEScat.fits")
ra_min, ra_max, dec_min, dec_max = srpylib.min_max_match(
t["RA_CALC_G"], t["DEC_CALC_G"], 1.0, DES_box=True
)
query_wise = "select ra, dec FROM allwise.main WHERE ra > " + \
ra_min + " and ra < " + ra_max + " and dec > " + dec_min + \
" and dec < " + dec_max
RA, DEC = sqlutil.get(query_wise, db=db, host=host,
user=user, password=password)
dists, inds = match_lists.match_lists(
t["RA_CALC_G"].data, t["DEC_CALC_G"].data, RA, DEC, 1.0, 2
)
match_distance = dists[:,1]*3600.0
if checkplots:
plt.axvline(6.1)
plt.axvline(6.4)
plt.axvline(6.5)
plt.axvline(12.0)
plt.hist(match_distance, bins=100)
plt.xlabel('Match Distance(")')
plt.ylabel("Number")
plt.title("Nearest Neighbours:" + tile)
plt.show()
print("Median Distance:", np.median(match_distance))
def test__match_lists__with_matching_lists():
assert match_lists([{
'name': 'A'
}, {
'name': 'B'
}, {
'name': 'C'
}], [{
'name': 'B'
}, {
'name': 'A'
}, {
'name': 'C'
}], '/name') == ([{
'name': 'A'
}, {
'name': 'B'
}, {
'name': 'C'
}])
def VHS_match(t, RA_main, DEC_main, c_graph = True, width = 0.0004, w_units = "degrees", survey = "des", bands = ["Y", "J", "H", "K"], DES_box = False):
"""
Matches to VHS Data from WSDB
Tables used are vhs_1504. + survey
DES_box returns all objects within a box of about the size of a DES tile centred
on the median RA and DEC of the table.
Iterates through a table where the RA column is RA_main and the DEC_column is
DEC_main
Puts NaNs in for no matches
c_graph plots diagnostic graphs of the match - crashes if there are no matches
Can specify not the full number of bands
#29/03/16 - Changed from 1504 to 1603
"""
import numpy as np
import match_lists
import sqlutil
from astropy.table import Table, vstack
import ConfigParser
config_file = "/home/sr525/Python_Code/srpylib/srpylib.cfg"
config = ConfigParser.RawConfigParser()
config.read(config_file)
uname = config.get("wsdb", "uname")
pword = config.get("wsdb", "pword")
host = config.get("wsdb", "host")
if w_units == "arcsecs":
width = width / 3600.0
if survey == "des":
cols = ["RA_VHS", "DEC_VHS", "J", "J_ERR", "H", "H_ERR", "K", "K_ERR"]
elif survey == "atlas":
cols = ["RA_VHS", "DEC_VHS", "Y", "Y_ERR", "J", "J_ERR", "H", "H_ERR", \
"K", "K_ERR"]
try:
for col in cols:
t.add_empty_column(col, dtype = np.float64)
except AttributeError as e:
for col in cols:
t[col] = np.zeros(len(t))
ra_min, ra_max, dec_min, dec_max = min_max_match(t[RA_main], t[DEC_main], \
width, DES_box = DES_box)
area = (float(ra_max)-float(ra_min))*(float(dec_max)-float(dec_min))
print area
if len(t) > 1000 and area < 2000:
ra_min, ra_max, dec_min, dec_max = min_max_match(t[RA_main], t[DEC_main], \
width, DES_box = DES_box)
print "Retreiving data"
if survey == "atlas":
query_vhs = "select ra, dec, yapercormag3, yapercormag3_err, " + \
"japercormag3, japercormag3_err, hapercormag3, " + \
"hapercormag3_err, kapercormag3, kapercormag3_err " + \
"FROM vhs_1603." + survey + " WHERE ra > " + ra_min + \
" and ra < " + ra_max + " and dec > " + dec_min + \
" and dec < " + dec_max
RA_vhs, DEC_vhs, Y, Yerr, J, Jerr, H, Herr, K, Kerr = \
sqlutil.get(query_vhs, db="wsdb", host=host, user=uname, password=pword)
elif survey == "des":
query_vhs = "select ra, dec, japercormag3, japercormag3_err, " + \
"hapercormag3, hapercormag3_err, kapercormag3, " + \
"kapercormag3_err FROM vhs_1603." + survey + \
" WHERE ra > " + ra_min + " and ra < " + ra_max + \
" and dec > " + dec_min + " and dec < " + dec_max
RA_vhs, DEC_vhs, J, Jerr, H, Herr, K, Kerr = \
sqlutil.get(query_vhs, db="wsdb", host=host, user=uname, password=pword)
print "Retreived data"
if len(RA_vhs) == 0:
for (i, col) in enumerate(cols):
t[col] = [np.float64(np.nan)]*len(t)
else:
dists, inds = match_lists.match_lists(t[RA_main], t[DEC_main], RA_vhs, DEC_vhs, width, 1)
if survey == "des":
info = [RA_vhs, DEC_vhs, J, Jerr, H, Herr, K, Kerr]
elif survey == "atlas":
info = [RA_vhs, DEC_vhs, Y, Yerr, J, Jerr, H, Herr, K, Kerr]
match = np.where( (inds <> len(RA_vhs)) )[0]
no_match = np.where( (inds == len(RA_vhs)) )[0]
for (i, col) in enumerate(cols):
t[col][match] = info[i][inds[match]]
t[col][no_match] = np.float64(np.nan)
if len(t) <= 1000:
n = 0
while n < len(t):
RA = t[RA_main][n]
DEC = t[DEC_main][n]
if survey == "atlas":
query_vhs = "select ra, dec, yapercormag3, yapercormag3_err, " + \
"japercormag3, japercormag3_err, hapercormag3, " + \
"hapercormag3_err, kapercormag3, kapercormag3_err " + \
"FROM vhs_1504." + survey + \
" WHERE q3c_radial_query(ra, dec, " + str(RA) + \
", " + str(DEC) + ", 0.1102)"
RA_vhs, DEC_vhs, Y, Yerr, J, Jerr, H, Herr, K, Kerr = \
sqlutil.get(query_vhs, db="wsdb", host=host, user=uname, password=pword)
elif survey == "des":
query_vhs = "select ra, dec, japercormag3, japercormag3_err, " + \
"hapercormag3, hapercormag3_err, kapercormag3, " + \
"kapercormag3_err FROM vhs_1504." + survey + \
" WHERE q3c_radial_query(ra, dec, " + str(RA) + \
", " + str(DEC) + ", 0.1102)"
RA_vhs, DEC_vhs, J, Jerr, H, Herr, K, Kerr = \
sqlutil.get(query_vhs, db="wsdb", host=host, user=uname, password=pword)
if len(RA_vhs) > 0:
dists, inds = match_lists.match_lists([RA], [DEC], RA_vhs, DEC_vhs, width, 1)
j = inds[0]
if j <> len(RA_vhs):
if survey == "des":
info = [RA_vhs[j], DEC_vhs[j], J[j], Jerr[j], H[j], Herr[j], \
K[j], Kerr[j]]
if survey == "atlas":
info = [RA_vhs[j], DEC_vhs[j], Y[j], Yerr[j], J[j], Jerr[j], \
H[j], Herr[j], K[j], Kerr[j]]
for (k, col) in enumerate(cols):
t[col][n] = info[k]
else:
for (k, col) in enumerate(cols):
t[col][n] = np.float64(np.nan)
else:
for (k, col) in enumerate(cols):
t[col][n] = np.float64(np.nan)
n += 1
print "Done VHS"
if area > 2000 and len(t) > 1000:
t.sort(RA_main)
l1 = np.ceil(area/1001.0)
l = len(t)/l1
i = 0
while i < l1:
if (i+1)*l < len(t):
t1 = t[i*l:(i+1)*l]
else:
t1 = t[i*l:]
t1 = VHS_match(t1, RA_main, DEC_main, c_graph = c_graph, width = width, \
w_units = w_units, survey = survey, bands = bands, \
DES_box = DES_box)
if i == 0:
t_out = t1
else:
t_out = vstack([t_out, t1])
i += 1
t = t_out
if c_graph:
width = width *3600.0
RA_med, DEC_med = check_graph(t, RA_main, DEC_main, "VHS", width)
return t
def GALEX_match(t, RA_main, DEC_main, width = 0.0004, w_units = "degrees", c_graph = True, table = "ais"):
"""
Matches to GALEX table
DES_box returns all objects within a box of about the size of a DES tile centred
on the median RA and DEC of the table.
Puts NaNs in for no matches
c_graph plots diagnostic graphs of the match
"""
import numpy as np
import match_lists
import sqlutil
import ConfigParser
config_file = "/home/sr525/Python_Code/srpylib/srpylib.cfg"
config = ConfigParser.RawConfigParser()
config.read(config_file)
uname = config.get("wsdb", "uname")
pword = config.get("wsdb", "pword")
host = config.get("wsdb", "host")
if w_units == "arcsecs":
width = width / 3600.0
cols = ["RA_GALEX", "DEC_GALEX", "NUV_MAG", "NUV_MAGERR", "FUV_MAG", \
"FUV_MAGERR", "E_BV", "NUV_CLASS_STAR", "FUV_CLASS_STAR"]
try:
for col in cols:
t.add_empty_column(col, dtype = np.float64)
except AttributeError as e:
for col in cols:
t[col] = np.zeros(len(t))
if len(t) > 1000:
ra_min, ra_max, dec_min, dec_max = min_max_match(t[RA_main], t[DEC_main], \
width, DES_box = DES_box)
print "Retreiving data"
query = "select ra, dec, nuv_mag, nuv_magerr, fuv_mag, fuv_magerr, e_bv, \
nuv_class_star, fuv_class_star FROM galexgr6." + table + "primary \
WHERE ra > " + ra_min + " and ra < " + ra_max + " and dec > " + \
dec_min + " and dec < " + dec_max
RA_galex, DEC_galex, nuv_mag, nuv_magerr, fuv_mag, fuv_magerr, e_bv, \
nuv_class_star, fuv_class_star = \
sqlutil.get(query, db = "wsdb", host=host, user=uname, password=pword)
print "Retreived data"
if len(RA_galex) == 0:
for (i, col) in enumerate(cols):
t[col] = [np.float64(np.nan)]*len(t)
else:
dists, inds = match_lists.match_lists(t[RA_main], t[DEC_main], RA_vhs, DEC_vhs, width, 1)
info = [ra_galex, dec_galex, nuv_mag, nuv_magerr, fuv_mag, fuv_magerr, \
e_bv, nuv_class_star, fuv_class_star]
match = np.where( (inds <> len(RA_galex)) )[0]
no_match = np.where( (inds == len(RA_galex)) )[0]
for (i, col) in enumerate(cols):
t[col][match] = info[i][inds[match]]
t[col][no_match] = np.float64(np.nan)
elif len(t) <= 1000:
n = 0
while n < len(t):
RA = t[RA_main][n]
DEC = t[DEC_main][n]
query_galex = "select ra, dec, nuv_mag, nuv_magerr, fuv_mag, \
fuv_magerr, e_bv, nuv_class_star, fuv_class_star FROM \
galexgr6." + table + "primary WHERE \
q3c_radial_query(ra, dec, " + str(RA) + ", " + str(DEC) + ", 0.1102)"
RA_galex, DEC_galex, nuv_mag, nuv_magerr, fuv_mag, fuv_magerr, e_bv, \
nuv_class_star, fuv_class_star = sqlutil.get(query_galex, db = "wsdb", \
host=host, user=uname, password=pword)
if len(RA_galex) > 0:
dists, inds = match_lists.match_lists([RA], [DEC], RA_galex, DEC_galex, width, 1)
j = inds[0]
if j <> len(RA_galex):
info = [RA_galex[j], DEC_galex[j], nuv_mag[j], nuv_magerr[j], \
fuv_mag[j], fuv_magerr[j], e_bv[j], nuv_class_star[j], \
fuv_class_star[j]]
for (k, col) in enumerate(cols):
t[col][n] = info[rki]
else:
for (k, col) in enumerate(cols):
t[col][n] = np.float64(np.nan)
else:
for (k, col) in enumerate(cols):
t[col][n] = np.float64(np.nan)
n += 1
print "Done GALEX"
elif len(t) > 1000000000:
print "Splitting by tile"
tiles_all = t["TILENAME"]
tiles = set(tiles_all)
tiles = list(tiles)
print len(tiles), "distinct tiles"
t.sort("TILENAME")
tiles.sort()
n = 0
m = 0
rows = []
tile_rows = []
while n < len(t):
tile = tiles[m]
if t["TILENAME"][n] == tile:
rows.append(n)
n += 1
else:
tile_rows.append(rows)
rows = []
m += 1
for rows in tile_rows:
t1 = t.rows(rows)
print "Matching", t1["TILENAME"][0], "with length", len(t1)
t1 = GALEX_match(t1, RA_main, DEC_main, width, c_graph = False)
if tile_rows.index(rows) == 0:
t_out = t1
else:
t_out.append(t1)
t = t_out
if c_graph:
width = width *3600.0
RA_med, DEC_med = check_graph(t, RA_main, DEC_main, "GALEX", width)
return t
def VST_match(t, RA_main, DEC_main, c_graph = True, width = 0.0004, w_units = "degrees", DES_box = False):
"""
Matches to VST Data from WSDB
Table used is vst_201310.atlas_rgi
DES_box returns all objects within a box of about the size of a DES tile centred
on the median RA and DEC of the table.
Iterates through a table where the RA column is RA_main and the DEC_column is
DEC_main
Puts NaNs in for no matches
c_graph plots diagnostic graphs of the match - crashes if there are no matches
"""
import numpy as np
import match_lists
import sqlutil
import ConfigParser
config_file = "/home/sr525/Python_Code/srpylib/srpylib.cfg"
config = ConfigParser.RawConfigParser()
config.read(config_file)
uname = config.get("wsdb", "uname")
pword = config.get("wsdb", "pword")
host = config.get("wsdb", "host")
if w_units == "arcsecs":
width = width / 3600.0
cols = ["RA_VST", "DEC_VST", "G_VST", "G_VST_ERR", "R_VST", "R_VST_ERR", \
"I_VST", "I_VST_ERR"]
try:
for col in cols:
t.add_empty_column(col, dtype = np.float64)
except AttributeError as e:
for col in cols:
t[col] = np.zeros(len(t))
if len(t) > 1000 and len(t) < 1000000000:
ra_min, ra_max, dec_min, dec_max = min_max_match(t[RA_main], t[DEC_main], \
width, DES_box = DES_box)
print "Retreiving data"
query = "select ra, dec, mag_g, magerr_g, mag_r, magerr_r, mag_i, " + \
"magerr_i FROM vst_201310.atlas_rgi WHERE ra > " + \
ra_min + " and ra < " + ra_max + " and dec > " + \
dec_min + " and dec < " + dec_max
RA_vst, DEC_vst, g, gerr, r, rerr, i, ierr = \
sqlutil.get(query, db="wsdb", host=host, user=uname, password=pword)
print "Retreived data"
dists, inds = match_lists.match_lists(t[RA_main], t[DEC_main], RA_vst, DEC_vst, width, 1)
print "Matched lists, updating table"
info = [RA_vst, DEC_vst, g, gerr, r, rerr, i, ierr]
match = np.where( (inds <> len(RA_vst)) )[0]
no_match = np.where( (inds == len(RA_vst)) )[0]
for (i, col) in enumerate(cols):
t[col][match] = info[i][inds[match]]
t[col][no_match] = np.float64(np.nan)
print "Table updated"
elif len(t) <= 1000:
n = 0
while n < len(t):
RA = t[RA_main][n]
DEC = t[DEC_main][n]
query_vhs = "select ra, dec, mag_g, magerr_g, mag_r, magerr_r, " + \
"mag_i, magerr_i FROM vst_201310.atlas_rgi " + \
"WHERE q3c_radial_query(ra, dec, " + str(RA) + ", " + \
str(DEC) + ", 0.1102)"
RA_vst, DEC_vst, g, gerr, r, rerr, i, ierr = \
sqlutil.get(query_vhs, db="wsdb", host=host, user=uname, password=pword)
if len(RA_vst) > 0:
dists, inds = match_lists.match_lists([RA], [DEC], RA_vst, DEC_vst, width, 1)
j = inds[0]
if j <> len(RA_vst):
info = [RA_vst[j], DEC_vst[j], g[j], gerr[j], r[j], rerr[j], \
i[j], ierr[j]]
for (k, col) in enumerate(cols):
t[col][n] = info[k]
else:
for (k, col) in enumerate(cols):
t[col][n] = np.float64(np.nan)
else:
for (k, col) in enumerate(cols):
t[col][n] = np.float64(np.nan)
n += 1
elif len(t) > 1000000000:
print "Splitting by tile"
tiles_all = t["TILENAME"]
tiles = set(tiles_all)
tiles = list(tiles)
print len(tiles), "distinct tiles"
t.sort("TILENAME")
tiles.sort()
n = 0
m = 0
rows = []
tile_rows = []
while n < len(t):
tile = tiles[m]
if t["TILENAME"][n] == tile:
rows.append(n)
n += 1
else:
tile_rows.append(rows)
rows = []
m += 1
for rows in tile_rows:
t1 = t.rows(rows)
print "Matching", t1["TILENAME"][0], "with length", len(t1)
t1 = VST_match(t1, RA_main, DEC_main, width, c_graph = False)
if tile_rows.index(rows) == 0:
t_out = t1
else:
t_out.append(t1)
t = t_out
if c_graph:
width = width *3600.0
RA_med, DEC_med = check_graph(t, RA_main, DEC_main, "VST", width)
return t
def WISE_match(t, RA_main, DEC_main, width = 0.0004, w_units = "degrees", c_graph = True, DES_box = False):
"""
Matches to WISE Data from WSDB
Table used is allwise.main
DES_box returns all objects within a box of about the size of a DES tile centred
on the median RA and DEC of the table.
Iterates through a table where the RA column is RA_main and the DEC_column is
DEC_main
Puts NaNs in for no matches
c_graph plots diagnostic graphs of the match - crashes if there are no matches
"""
import numpy as np
import match_lists
import sqlutil
import ConfigParser
config_file = "/home/sr525/Python_Code/srpylib/srpylib.cfg"
config = ConfigParser.RawConfigParser()
config.read(config_file)
uname = config.get("wsdb", "uname")
pword = config.get("wsdb", "pword")
host = config.get("wsdb", "host")
if w_units == "arcsecs":
width = width / 3600.0
cols = ["RA_WISE", "DEC_WISE", "W1", "W1_ERR", "W2", "W2_ERR", "W3", "W3_ERR", "W4", "W4_ERR"]
try:
for col in cols:
t.add_empty_column(col, dtype = np.float64)
except AttributeError as e:
for col in cols:
t[col] = np.zeros(len(t))
n = 0
if len(t) > 5000 and len(t) < 100000000:
ra_min, ra_max, dec_min, dec_max = min_max_match(t[RA_main], t[DEC_main], \
width, DES_box = DES_box)
print "Retreiving data"
query_wise = "select ra, dec, w1mpro, w1sigmpro, w2mpro, w2sigmpro, " + \
"w3mpro, w3sigmpro, w4mpro, w4sigmpro FROM " + \
"allwise.main WHERE ra > " + ra_min + " and ra < " + \
ra_max + " and dec > " + dec_min + " and dec < " + dec_max
RA_wise, DEC_wise, W1, W1err, W2, W2err, W3, W3err, W4, W4err = \
sqlutil.get(query_wise, db="wsdb", host=host, user=uname, password=pword)
info = [RA_wise, DEC_wise, W1, W1err, W2, W2err, W3, W3err, W4, W4err]
if len(RA_wise) == 0:
for (i, col) in enumerate(cols):
t[col] = [np.float64(np.nan)]*len(t)
else:
print "Retreived data, matching lists"
dists, inds = match_lists.match_lists(t[RA_main], t[DEC_main], RA_wise, DEC_wise, width, 1)
print "Matched lists, updating table"
match = np.where( (inds <> len(RA_wise)) )[0]
no_match = np.where( (inds == len(RA_wise)) )[0]
for (i, col) in enumerate(cols):
t[col][match] = info[i][inds[match]]
t[col][no_match] = np.float64(np.nan)
print "Table updated"
elif len(t) <= 5000:
while n < len(t):
RA = t[RA_main][n]
DEC = t[DEC_main][n]
query = "select ra, dec, w1mpro, w1sigmpro, w2mpro, w2sigmpro, " + \
"w3mpro, w3sigmpro, w4mpro, w4sigmpro FROM allwise.main " + \
"WHERE q3c_radial_query(ra, dec, " + str(RA) + ", " + \
str(DEC) + ", 0.1002)"
RA_W, DEC_W, W1, W1err, W2, W2err, W3, W3err, W4, W4err = \
sqlutil.get(query, db='wsdb',host=host, user=uname, password=pword)
if len(RA_W) > 0:
dists, inds = match_lists.match_lists([RA], [DEC], RA_W, DEC_W, width, 1)
j = inds[0]
if j <> len(RA_W):
info = [RA_W[j], DEC_W[j], W1[j], W1err[j], W2[j], W2err[j], \
W3[j], W3err[j], W4[j], W4err[j]]
for (i, col) in enumerate(cols):
t[col][n] = info[i]
else:
for (i, col) in enumerate(cols):
t[col][n] = np.float64(np.nan)
else:
for (i, col) in enumerate(cols):
t[col][n] = np.float64(np.nan)
n += 1
elif len(t) > 1000000000:
print "Splitting by tile"
tiles_all = t["TILENAME"]
tiles = set(tiles_all)
tiles = list(tiles)
print len(tiles), "distinct tiles"
t.sort("TILENAME")
tiles.sort()
n = 0
m = 0
rows = []
tile_rows = []
while n < len(t):
tile = tiles[m]
if t["TILENAME"][n] == tile:
rows.append(n)
n += 1
else:
tile_rows.append(rows)
rows = []
m += 1
for rows in tile_rows:
t1 = t.rows(rows)
print "Matching", t1["TILENAME"][0], "with length", len(t1)
t1 = WISE_match(t1, RA_main, DEC_main, width, c_graph = False)
if tile_rows.index(rows) == 0:
t_out = t1
else:
t_out.append(t1)
t = t_out
if c_graph:
width = width *3600.0
RA_med, DEC_med = check_graph(t, RA_main, DEC_main, "WISE", width, gtype = "square")
return t
def SDSS_match(t, RA_main, DEC_main, width = 0.0004, w_units = "degrees", c_graph = True, dr = "dr9", DES_box = False):
"""
Matches to SDSS Data from WSDB
Table used is dr + .phototag
DES_box returns all objects within a box of about the size of a DES tile centred
on the median RA and DEC of the table.
Iterates through a table where the RA column is RA_main and the DEC_column is
DEC_main
Puts NaNs in for no matches
c_graph plots diagnostic graphs of the match - crashes if there are no matches
"""
import numpy as np
import match_lists
import sqlutil
from astropy.table import Table, vstack
import ConfigParser
config_file = "/home/sr525/Python_Code/srpylib/srpylib.cfg"
config = ConfigParser.RawConfigParser()
config.read(config_file)
uname = config.get("wsdb", "uname")
pword = config.get("wsdb", "pword")
host = config.get("wsdb", "host")
if w_units == "arcsecs":
width = width / 3600.0
cols = ["RA_SDSS", "DEC_SDSS", "MAG_PSF_U_SDSS", "MAGERR_PSF_U_SDSS", \
"MAG_PSF_G_SDSS", "MAGERR_PSF_G_SDSS", "MAG_PSF_R_SDSS", \
"MAGERR_PSF_R_SDSS", "MAG_PSF_I_SDSS", "MAGERR_PSF_I_SDSS", \
"MAG_PSF_Z_SDSS", "MAGERR_PSF_Z_SDSS", "MAG_MODEL_U_SDSS", \
"MAGERR_MODEL_U_SDSS", "MAG_MODEL_G_SDSS", "MAGERR_MODEL_G_SDSS", \
"MAG_MODEL_R_SDSS", "MAGERR_MODEL_R_SDSS", "MAG_MODEL_I_SDSS", \
"MAGERR_MODEL_I_SDSS", "MAG_MODEL_Z_SDSS", "MAGERR_MODEL_Z_SDSS", "TYPE"]
try:
for col in cols:
t.add_empty_column(col, dtype = np.float64)
except AttributeError as e:
for col in cols:
t[col] = np.zeros(len(t))
n = 0
if len(t) > 100 and len(t) < 1000000:
print "Finding max and mins"
ra_min, ra_max, dec_min, dec_max = min_max_match(t[RA_main], t[DEC_main], \
width, DES_box = DES_box)
print "Retreiving data"
query_sdss = "select ra, dec, psfMag_u, psfMagErr_u, psfMag_g, " + \
"psfMagErr_g, psfMag_r, psfMagErr_r, psfMag_i, psfMagErr_i, " + \
"psfMag_z, psfMagErr_z, modelMag_u, modelMagErr_u, " + \
"modelMag_g, modelMagErr_g, modelMag_r, modelMagErr_r, " + \
"modelMag_i, modelMagErr_i, modelMag_z, modelMagErr_z, " + \
"type FROM sdss" + dr + ".phototag WHERE ra > " + ra_min + \
" and ra < " + ra_max + " and dec > " + dec_min + \
" and dec < " + dec_max
RA_sdss, DEC_sdss, psfu, psfuerr, psfg, psfgerr, psfr, psfrerr, psfi, \
psfierr, psfz, psfzerr, modelu, modeluerr, modelg, modelgerr, modelr, \
modelrerr, modeli, modelierr, modelz, modelzerr, types = \
sqlutil.get(query_sdss, db = "wsdb", host = host, user = uname, password = pword)
info = [RA_sdss, DEC_sdss, psfu, psfuerr, psfg, psfgerr, psfr, psfrerr, \
psfi, psfierr, psfz, psfzerr, modelu, modeluerr, modelg, modelgerr, \
modelr, modelrerr, modeli, modelierr, modelz, modelzerr, types]
if len(RA_sdss) == 0:
for (i, col) in enumerate(cols):
t[col] = [np.float64(np.nan)]*len(t)
else:
print "Retreived data, matching lists"
dists, inds = match_lists.match_lists(t[RA_main], t[DEC_main], RA_sdss, DEC_sdss, width, 1)
print "Matched lists, updating table"
match = np.where( (inds <> len(RA_sdss)) )[0]
no_match = np.where( (inds == len(RA_sdss)) )[0]
for (i, col) in enumerate(cols):
t[col][match] = info[i][inds[match]]
t[col][no_match] = np.float64(np.nan)
print "Table updated"
elif len(t) <= 100:
while n < len(t):
RA = t[RA_main][n]
DEC = t[DEC_main][n]
print RA, DEC
query_sdss = "select ra, dec, psfMag_u, psfMagErr_u, psfMag_g, " + \
"psfMagErr_g, psfMag_r, psfMagErr_r, psfMag_i, " + \
"psfMagErr_i, psfMag_z, psfMagErr_z, modelMag_u, " + \
"modelMagErr_u, modelMag_g, modelMagErr_g, modelMag_r, " + \
"modelMagErr_r, modelMag_i, modelMagErr_i, modelMag_z, " + \
"modelMagErr_z, type FROM sdss" + dr + \
".phototag WHERE q3c_radial_query(ra, dec, " + str(RA) + \
", " + str(DEC) + ", 0.1002)"
RA_sdss, DEC_sdss, psfu, psfuerr, psfg, psfgerr, psfr, psfrerr, psfi, \
psfierr, psfz, psfzerr, modelu, modeluerr, modelg, modelgerr, modelr, \
modelrerr, modeli, modelierr, modelz, modelzerr, stype = \
sqlutil.get(query_sdss, db = "wsdb", host = host, user = uname, password = pword)
if len(RA_sdss) > 0:
dist, inds = match_lists.match_lists([RA], [DEC], RA_sdss, DEC_sdss, width, 1)
j = inds[0]
info = [RA_sdss[j], DEC_sdss[j], psfu[j], psfuerr[j], psfg[j], \
psfgerr[j], psfr[j], psfrerr[j], psfi[j], psfierr[j], \
psfz[j], psfzerr[j], modelu[j], modeluerr[j], modelg[j], \
modelgerr[j], modelr[j], modelrerr[j], modeli[j], \
modelierr[j], modelz[j], modelzerr[j], stype[j]]
for (i, col) in enumerate(cols):
t[col][n] = info[i]
else:
for (i, col) in enumerate(cols):
t[col][n] = np.float64(np.nan)
n += 1
elif len(t) > 1000000:
print "Splitting by tile"
tiles_all = t["TILENAME"]
tiles = set(tiles_all)
tiles = list(tiles)
print len(tiles), "distinct tiles"
t.sort("TILENAME")
tiles.sort()
n = 0
m = 0
rows = []
tile_rows = []
while n < len(t):
tile = tiles[m]
if t["TILENAME"][n] == tile:
rows.append(n)
n += 1
else:
print len(rows)
tile_rows.append(rows)
rows = []
m += 1
for rows in tile_rows:
try:
t1 = t.rows(rows)
except AttributeError:
t1 = t[rows]
print "Matching", t1["TILENAME"][0], "with length", len(t1)
t1 = SDSS_match(t1, RA_main, DEC_main, width, c_graph = False)
if tile_rows.index(rows) == 0:
t_out = t1
else:
try:
t_out.append(t1)
except AttributeError:
t_out = vstack([t_out, t1])
t = t_out
if c_graph:
width = width *3600.0
RA_med, DEC_med = check_graph(t, RA_main, DEC_main, "SDSS", width, gtype = "square")
return t
def DES_match(t, RA_main, DEC_main, c_graph = True, width = 0.0004, w_units = "degrees", release = "Y1A1"):
"""
Matches to DES data ingested into WSDB
Tables used are des_y1a1.coadd_objects and des_y2q1.objects
Iterates through a table where the RA column is RA_main and the DEC_column is
DEC_main
Puts NaNs in for no matches
c_graph plots diagnostic graphs of the match - crashes if there are no matches
"""
import numpy as np
import match_lists
import sqlutil
from astropy.table import Table, vstack
import ConfigParser
config_file = "/home/sr525/Python_Code/srpylib/srpylib.cfg"
config = ConfigParser.RawConfigParser()
config.read(config_file)
uname = config.get("wsdb", "uname")
pword = config.get("wsdb", "pword")
host = config.get("wsdb", "host")
if w_units == "arcsecs":
width = width / 3600.0
if release == "Y1A1":
cols = ["COADD_OBJECTS_ID", "RA_DES_Y1A1", "DEC_DES_Y1A1", "TILENAME_Y1A1", \
"RUN_Y1A1", "MPSF_G_Y1A1", "MPSF_ERR_G_Y1A1", "MPSF_R_Y1A1", \
"MPSF_ERR_R_Y1A1", "MPSF_I_Y1A1", "MPSF_ERR_I_Y1A1", "MPSF_Z_Y1A1", \
"MPSF_ERR_Z_Y1A1", "MPSF_Y_Y1A1", "MPSF_ERR_Y_Y1A1", "MAPER3_G_Y1A1", \
"MAPER3_ERR_G_Y1A1", "MAPER3_R_Y1A1", "MAPER3_ERR_R_Y1A1", \
"MAPER3_I_Y1A1", "MAPER3_ERR_I_Y1A1", "MAPER3_Z_Y1A1", \
"MAPER3_ERR_Z_Y1A1", "MAPER3_Y_Y1A1", "MAPER3_ERR_Y_Y1A1"]
str_cols = ["TILENAME_Y1A1", "RUN_Y1A1"]
elif release == "Y2Q1":
cols = ["QUICK_OBJECT_ID", "RA_DES_Y2Q1", "DEC_DES_Y2Q1", "MPSF_G_Y2Q1", \
"MPSF_ERR_G_Y2Q1", "MPSF_R_Y2Q1", "MPSF_ERR_R_Y2Q1", "MPSF_I_Y2Q1", \
"MPSF_ERR_I_Y2Q1", "MPSF_Z_Y2Q1", "MPSF_ERR_Z_Y2Q1", "MPSF_Y_Y2Q1", \
"MPSF_ERR_Y_Y2Q1", "MAUTO_G_Y2Q1", "MAUTO_ERR_G_Y2Q1", \
"MAUTO_R_Y2Q1", "MAUTO_ERR_R_Y2Q1", "MAUTO_I_Y2Q1", \
"MAUTO_ERR_I_Y2Q1", "MAUTO_Z_Y2Q1", "MAUTO_ERR_Z_Y2Q1", \
"MAUTO_Y_Y2Q1", "MAUTO_ERR_Y_Y2Q1"]
str_cols = []
try:
for col in cols:
if col not in str_cols:
t.add_empty_column(col, dtype = np.float64)
else:
t.add_empty_col(col, dtype = np.string)
except AttributeError as e:
for col in cols:
if col not in str_cols:
t[col] = np.zeros(len(t))
else:
t[col] = ["AAAAAAAAAAAAAAAAAAAAAAAAAA"]
ra_min, ra_max, dec_min, dec_max = min_max_match(t[RA_main], t[DEC_main], \
width, DES_box = False)
area = (float(ra_max)-float(ra_min))*(float(dec_max)-float(dec_min))
print area
if len(t) > 1000 and area < 2000:
ra_min, ra_max, dec_min, dec_max = min_max_match(t[RA_main], t[DEC_main], \
width, DES_box = False)
print "Retreiving data"
if release == "Y1A1":
query_des = "select COADD_OBJECTS_ID, RA, DEC, TILENAME, RUN, MAG_PSF_G, \
MAGERR_PSF_G, MAG_PSF_R, MAGERR_PSF_R, MAG_PSF_I, MAGERR_PSF_I, \
MAG_PSF_Z, MAGERR_PSF_Z, MAG_PSF_Y, MAGERR_PSF_Y, MAG_APER_3_G, \
MAGERR_APER_3_G, MAG_APER_3_R, MAGERR_APER_3_R, MAG_APER_3_I, \
MAGERR_APER_3_I, MAG_APER_3_Z, MAGERR_APER_3_Z, MAG_APER_3_Y, \
MAGERR_APER_3_Y FROM des_y1a1.coadd_objects WHERE \
ra > " + ra_min + " and ra < " + ra_max + " and dec > " + \
dec_min + " and dec < " + dec_max
co_id, RA_des, DEC_des, tile, run, G, Gerr, R, Rerr, I, Ierr, Z, Zerr, Y, \
Yerr, Ga, Gaerr, Ra, Raerr, Ia, Iaerr, Za, Zaerr, Ya, Yaerr = \
sqlutil.get(query_des, db="wsdb", host=host, user=uname, password=pword)
elif release == "Y2Q1":
query_des = "select QUICK_OBJECT_ID, RA, DEC, MAG_PSF_G, MAGERR_PSF_G, \
MAG_PSF_R, MAGERR_PSF_R, MAG_PSF_I, MAGERR_PSF_I, MAG_PSF_Z, \
MAGERR_PSF_Z, MAG_PSF_Y, MAGERR_PSF_Y, MAG_AUTO_G, \
MAGERR_AUTO_G, MAG_AUTO_R, MAGERR_AUTO_R, MAG_AUTO_I, \
MAGERR_AUTO_I, MAG_AUTO_Z, MAGERR_AUTO_Z, MAG_AUTO_Y, \
MAGERR_AUTO_Y FROM des_y2q1.objects WHERE ra > " + ra_min \
+ " and ra < " + ra_max + " and dec > " + dec_min + \
" and dec < " + dec_max
q_id, RA_des, DEC_des, G, Gerr, R, Rerr, I, Ierr, Z, Zerr, Y, Yerr, Ga, \
Gaerr, Ra, Raerr, Ia, Iaerr, Za, Zaerr, Ya, Yaerr = \
sqlutil.get(query_des, db="wsdb", host=host, user=uname, password=pword)
print "Retreived data"
if len(RA_des) == 0:
for (i, col) in enumerate(cols):
t[col] = [np.float64(np.nan)]*len(t)
else:
dists, inds = match_lists.match_lists(t[RA_main], t[DEC_main], RA_des, DEC_des, width, 1)
if release == "Y1A1":
info = [co_id, RA_des, DEC_des, tile, run, G, Gerr, R, Rerr, I, \
Ierr, Z, Zerr, Y, Yerr, Ga, Gaerr, Ra, Raerr, Ia, Iaerr, Za, Zaerr, Ya, Yaerr]
elif release == "Y2Q1":
info = [q_id, RA_des, DEC_des, G, Gerr, R, Rerr, I, Ierr, Z, Zerr, \
Y, Yerr, Ga, Gaerr, Ra, Raerr, Ia, Iaerr, Za, Zaerr, Ya, Yaerr]
match = np.where( (inds <> len(RA_des)) )[0]
no_match = np.where( (inds == len(RA_des)) )[0]
for (i, col) in enumerate(cols):
t[col][match] = info[i][inds[match]]
t[col][no_match] = np.float64(np.nan)
if len(t) <= 1000:
n = 0
while n < len(t):
RA = t[RA_main][n]
DEC = t[DEC_main][n]
if release == "Y1A1":
query_des = "select COADD_OBJECTS_ID, RA, DEC, TILENAME, RUN, MAG_PSF_G, \
MAGERR_PSF_G, MAG_PSF_R, MAGERR_PSF_R, MAG_PSF_I, MAGERR_PSF_I, \
MAG_PSF_Z, MAGERR_PSF_Z, MAG_PSF_Y, MAGERR_PSF_Y, MAG_APER_3_G, \
MAGERR_APER_3_G, MAG_APER_3_R, MAGERR_APER_3_R, MAG_APER_3_I, \
MAGERR_APER_3_I, MAG_APER_3_Z, MAGERR_APER_3_Z, MAG_APER_3_Y, \
MAGERR_APER_3_Y FROM des_y1a1.coadd_objects WHERE \
q3c_radial_query(ra, dec, " + str(RA) + ", " + str(DEC) + ", 0.1102)"
co_id, RA_des, DEC_des, tile, run, G, Gerr, R, Rerr, I, Ierr, Z, \
Zerr, Y, Yerr, Ga, Gaerr, Ra, Raerr, Ia, Iaerr, Za, Zaerr, Ya, Yaerr = \
sqlutil.get(query_des, db="wsdb", host=host, user=uname, password=pword)
elif release == "Y2Q1":
query_des = "select QUICK_OBJECT_ID, RA, DEC, MAG_PSF_G, MAGERR_PSF_G, MAG_PSF_R, \
MAGERR_PSF_R, MAG_PSF_I, MAGERR_PSF_I, MAG_PSF_Z, \
MAGERR_PSF_Z, MAG_PSF_Y, MAGERR_PSF_Y, MAG_AUTO_G, \
MAGERR_AUTO_G, MAG_AUTO_R, MAGERR_AUTO_R, MAG_AUTO_I, \
MAGERR_AUTO_I, MAG_AUTO_Z, MAGERR_AUTO_Z, MAG_AUTO_Y, \
MAGERR_AUTO_Y FROM des_y2q1.objects WHERE \
q3c_radial_query(ra, dec, " + str(RA) + ", " + str(DEC) + ", 0.1102)"
q_id, RA_des, DEC_des, G, Gerr, R, Rerr, I, Ierr, Z, Zerr, Y, Yerr, \
Ga, Gaerr, Ra, Raerr, Ia, Iaerr, Za, Zaerr, Ya, Yaerr = \
sqlutil.get(query_des, db="wsdb", host=host, user=uname, password=pword)
if len(RA_des) > 0:
dists, inds = match_lists.match_lists([RA], [DEC], RA_des, DEC_des, width, 1)
j = inds[0]
if j <> len(RA_des):
if release == "Y1A1":
info = [co_id[j], RA_des[j], DEC_des[j], tile[j], run[j], G[j], \
Gerr[j], R[j], Rerr[j], I[j], Ierr[j], Z[j], \
Zerr[j], Y[j], Yerr[j], Ga[j], Gaerr[j], Ra[j], \
Raerr[j], Ia[j], Iaerr[j], Za[j], Zaerr[j], Ya[j], Yaerr[j]]
if release == "Y2Q1":
info = [q_id[j], RA_des[j], DEC_des[j], G[j], Gerr[j], R[j], Rerr[j], \
I[j], Ierr[j], Z[j], Zerr[j], Y[j], Yerr[j], Ga[j], \
Gaerr[j], Ra[j], Raerr[j], Ia[j], Iaerr[j], Za[j], \
Zaerr[j], Ya[j], Yaerr[j]]
for (k, col) in enumerate(cols):
t[col][n] = info[k]
else:
for (k, col) in enumerate(cols):
t[col][n] = np.float64(np.nan)
else:
for (k, col) in enumerate(cols):
t[col][n] = np.float64(np.nan)
n += 1
print "Done DES"
if area > 2000 and len(t) > 1000:
t.sort(RA_main)
l1 = np.ceil(area/1001.0)
l = len(t)/l1
i = 0
while i < l1:
if (i+1)*l < len(t):
t1 = t[i*l:(i+1)*l]
else:
t1 = t[i*l:]
t1 = DES_match(t1, RA_main, DEC_main, c_graph = False, width = width, \
w_units = w_units, release = release)
if i == 0:
t_out = t1
else:
t_out = vstack([t_out, t1])
i += 1
t = t_out
if c_graph:
width = width *3600.0
if release == "Y1A1":
survey = "DES_Y1A1"
elif release == "Y2Q1":
survey = "DES_Y2Q1"
RA_med, DEC_med = check_graph(t, RA_main, DEC_main, survey, width)
return t
def VHS_match_all_cols(t, RA_main, DEC_main, survey = "des"):
"""
Matches to VHS Data from WSDB
Table used is vhs_201306.atlas
Iterates through a table where the RA column is RA_main and the DEC_column is
DEC_main
Puts NaNs in for no matches
c_graph plots diagnostic graphs of the match - crashes if there are no matches
Best to do for small tables only - othersise takes a long time and uses lots of
memory
"""
import numpy
import match_lists
import sqlutil
import atpy
import numpy as np
from astropy.table import Table, hstack
import ConfigParser
config_file = "/home/sr525/Python_Code/srpylib/srpylib.cfg"
config = ConfigParser.RawConfigParser()
config.read(config_file)
uname = config.get("wsdb", "uname")
pword = config.get("wsdb", "pword")
host = config.get("wsdb", "host")
n = 0
any_matches = False
while n < len(t):
RA = t[RA_main][n]
DEC = t[DEC_main][n]
print n+1, "out of", len(t)
query = "select * FROM vhs_1504." + survey + " WHERE q3c_radial_query(ra, \
dec, " + str(RA) + ", " + str(DEC) + ", 0.05)"
try:
t1 = atpy.Table("postgres", query=query, database="wsdb", host=host, \
user=uname, password=pword)
if not any_matches:
t2 = t1
any_matches = True
else:
t2.append(t1)
except Exception as e:
print e
n += 1
if not any_matches:
print "No matches"
return t
t_vhs = Table(t2[:])
dists, inds = match_lists.match_lists(t[RA_main], t[DEC_main], t_vhs["ra"], t_vhs["dec"], 0.0006)
ms = np.where( (inds < len(t_vhs)) )[0]
t_vhs = t_vhs[inds[ms]]
try:
t = t.rows([ms])
for col in t2.columns:
t.add_column(col, t2[col])
except AttributeError as e:
t = t[ms]
t = hstack([t, t_vhs])
return t
def UKIDSS_match(t, RA_main, DEC_main, c_graph = True, width = 0.0004, w_units = "degrees", DES_box = False):
"""
Matches to VHS Data from WSDB
Tables used are vhs_1504. + survey
DES_box returns all objects within a box of about the size of a DES tile centred
on the median RA and DEC of the table.
Iterates through a table where the RA column is RA_main and the DEC_column is
DEC_main
Puts NaNs in for no matches
c_graph plots diagnostic graphs of the match - crashes if there are no matches
Can specify not the full number of bands
"""
import numpy as np
import match_lists
import sqlutil
from astropy.table import Table, vstack
import ConfigParser
config_file = "srpylib.cfg"
uname = config.get("wsdb", uname)
pword = config.get("wsdb", pword)
host = config.get("wsdb", host)
if w_units == "arcsecs":
width = width / 3600.0
cols = ["RA_UKIDSS", "DEC_UKIDSS", "Y_UKIDSS", "Y_ERR_UKIDSS", "J_UKIDSS", \
"J_ERR_UKIDSS", "H_UKIDSS", "H_ERR_UKIDSS", "K_UKIDSS", "K_ERR_UKIDSS"]
try:
for col in cols:
t.add_empty_column(col, dtype = np.float64)
except AttributeError as e:
for col in cols:
t[col] = np.zeros(len(t))
ra_min, ra_max, dec_min, dec_max = min_max_match(t[RA_main], t[DEC_main], \
width, DES_box = DES_box)
area = (float(ra_max)-float(ra_min))*(float(dec_max)-float(dec_min))
print area
if len(t) > 1000 and area < 2000:
ra_min, ra_max, dec_min, dec_max = min_max_match(t[RA_main], t[DEC_main], \
width, DES_box = DES_box)
print "Retreiving data"
query_ukidss = "select ra, dec, yAperMag3, yAperMag3Err, j_1AperMag3, \
j_1AperMag3Err, hAperMag3, hAperMag3Err, kAperMag3, \
kAperMag3Err FROM ukidssdr10.lassource \
WHERE ra > " + ra_min + " and ra < " + ra_max + " and \
dec > " + dec_min + " and dec < " + dec_max
RA_u, DEC_u, Y, Yerr, J, Jerr, H, Herr, K, Kerr = \
sqlutil.get(query_ukidss, db="wsdb", host=host, user=uname, password=pword)
print "Retreived data"
if len(RA_u) == 0:
for (i, col) in enumerate(cols):
t[col] = [np.float64(np.nan)]*len(t)
else:
dists, inds = match_lists.match_lists(t[RA_main], t[DEC_main], RA_u, DEC_u, width, 1)
info = [RA_u, DEC_u, Y, Yerr, J, Jerr, H, Herr, K, Kerr]
match = np.where( (inds <> len(RA_u)) )[0]
no_match = np.where( (inds == len(RA_u)) )[0]
for (i, col) in enumerate(cols):
t[col][match] = info[i][inds[match]]
t[col][no_match] = np.float64(np.nan)
if len(t) <= 1000:
n = 0
while n < len(t):
RA = t[RA_main][n]
DEC = t[DEC_main][n]
query_ukidss = "select ra, dec, yAperMag3, yAperMag3Err, j_1AperMag3, \
j_1AperMag3Err, hAperMag3, hAperMag3Err, kAperMag3, \
kAperMag3Err FROM ukidssdr10.lassource WHERE \
q3c_radial_query(ra, dec, " + str(RA) + ", " + \
str(DEC) + ", 0.1102)"
RA_u, DEC_u, Y, Yerr, J, Jerr, H, Herr, K, Kerr = \
sqlutil.get(query_ukidss, db="wsdb", host=host, user=uname, password=pword)
if len(RA_u) > 0:
dists, inds = match_lists.match_lists([RA], [DEC], RA_u, DEC_u, width, 1)
j = inds[0]
if j <> len(RA_u):
info = [RA_u[j], DEC_u[j], Y[j], Yerr[j], J[j], Jerr[j], \
H[j], Herr[j], K[j], Kerr[j]]
for (k, col) in enumerate(cols):
t[col][n] = info[k]
else:
for (k, col) in enumerate(cols):
t[col][n] = np.float64(np.nan)
else:
for (k, col) in enumerate(cols):
t[col][n] = np.float64(np.nan)
n += 1
print "Done UKIDSS"
if area > 2000 and len(t) > 1000:
t.sort(RA_main)
l1 = np.ceil(area/1001.0)
l = len(t)/l1
i = 0
while i < l1:
if (i+1)*l < len(t):
t1 = t[i*l:(i+1)*l]
else:
t1 = t[i*l:]
t1 = UKIDSS_match(t1, RA_main, DEC_main, c_graph = c_graph, width = width, \
w_units = w_units, DES_box = DES_box)
if i == 0:
t_out = t1
else:
t_out = vstack([t_out, t1])
i += 1
t = t_out
if c_graph:
width = width *3600.0
RA_med, DEC_med = check_graph(t, RA_main, DEC_main, "UKIDSS", width)
return t
def VIKING_match(t, RA_main, DEC_main, width = 0.0004, w_units = "degrees", c_graph = True, DES_box = False):
"""
Matches to Viking Data from WSDB
Table used is viking_201207.main
DES_box returns all objects within a box of about the size of a DES tile centred
on the median RA and DEC of the table.
Iterates through a table where the RA column is RA_main and the DEC_column is
DEC_main
Puts NaNs in for no matches
c_graph plots diagnostic plot of the match - crashes if there are no matches
"""
import numpy as np
import match_lists
import sqlutil
import ConfigParser
config_file = "srpylib.cfg"
uname = config.get("wsdb", uname)
pword = config.get("wsdb", pword)
host = config.get("wsdb", host)
if w_units == "arcsecs":
width = width / 3600.0
cols = ["RA_VIKING", "DEC_VIKING", "VIKING_Y", "VIKING_J"]
try:
for col in cols:
t.add_empty_column(col, dtype = np.float64)
except AttributeError as e:
for col in cols:
t[col] = np.zeros(len(t))
n = 0
if len(t) > 1000 and len(t) < 1000000:
ra_min, ra_max, dec_min, dec_max = min_max_match(t[RA_main], t[DEC_main], \
width, DES_box = DES_box)
print "Retreiving data"
query_V = "select ra, dec, mag_y, mag_j FROM viking_201207.main WHERE ra > "\
+ ra_min + " and ra < " + ra_max + " and dec > " + \
dec_min + " and dec < " + dec_max
RA_V, DEC_V, Y_V, J_V = sqlutil.get(query_V, db = "wsdb", host = host,\
user = uname, password = pword)
if len(RA_wise) == 0:
for (i, col) in enumerate(cols):
t[col] = [np.float64(np.nan)]*len(t)
else:
info = [RA_V, DEC_V, Y_V, J_V]
print "Retreived data, matching lists"
dists, inds = match_lists.match_lists(t[RA_main], t[DEC_main], \
RA_V, DEC_V, width, 1)
print "Matched lists, updating table"
match = np.where( (inds <> len(RA_V)) )[0]
no_match = np.where( (inds == len(RA_V)) )[0]
for (i, col) in enumerate(cols):
t[col][match] = info[i][inds[match]]
t[col][no_match] = np.float64(np.nan)
print "Table updated"
elif len(t) <= 1000:
while n < len(t):
RA = t[RA_main][n]
DEC = t[DEC_main][n]
query = "select ra, dec, mag_y, mag_j FROM viking_201207.main WHERE " + \
"q3c_radial_query(ra, dec, " + str(RA) + \
", " + str(DEC) + ", 0.1002)"
RA_V, DEC_V, Y_V, J_V = sqlutil.get(query, db='wsdb',host=host, \
user = uname, password=pword)
if len(RA_V) > 0:
dists, inds = match_lists.match_lists([RA], [DEC], RA_V, DEC_V, width, 1)
j = inds[0]
info = [RA_V, DEC_V, Y_V, J_V]
for (i, col) in enumerate(cols):
t[col][n] = info[i]
else:
for (i, col) in enumerate(cols):
t[col][n] = np.float64(np.nan)
n += 1
elif len(t) > 1000000:
print "Splitting by tile"
tiles_all = t["TILENAME"]
tiles = set(tiles_all)
tiles = list(tiles)
print len(tiles), "distinct tiles"
t.sort("TILENAME")
tiles.sort()
n = 0
m = 0
rows = []
tile_rows = []
while n < len(t):
tile = tiles[m]
if t["TILENAME"][n] == tile:
rows.append(n)
n += 1
else:
tile_rows.append(rows)
rows = []
m += 1
for rows in tile_rows:
t1 = t.rows(rows)
print "Matching", t1["TILENAME"][0], "with length", len(t1)
t1 = VIKING_match(t1, RA_main, DEC_main, width, c_graph = False)
print "Returning table of length:", len(t1)
if tile_rows.index(rows) == 0:
t_out = t1
else:
t_out.append(t1)
t = t_out
if c_graph:
width = width *3600.0
RA_med, DEC_med = check_graph(t, RA_main, DEC_main, "VIKING", width, gtype = "square")
return t
