def lens_on_color_mag(TGASid, filename):
"""
Plots the lens with TGASid on a color magnitude diagram
with the rest of the TGAS sources - uses 2mass [j-k] colour
and absolute G magnitude, saves the plot with file name.
Args:
TGASid (long) : indentified for lens id
in TGAS source_id column
filename (string) : output file name
Returns:
file (.png) : output file of the plot with
filename.
"""
#Get color mag info for lens
querystringLens = 'select j_m,k_m,parallax,phot_g_mean_mag from gaia_dr1_aux.gaia_source_2mass_xm where source_id=' + str(
TGASid)
JmagLens, kMagLens, parallaxLens, gMagLens = sqlutil.get(
querystringLens,
db='wsdb',
host='cappc127.ast.cam.ac.uk',
user='******',
password='******')
#Calculate colour and absolute mag for lens
absMagLens = gMagLens + 5 * (np.log10(parallaxLens / 1000.0) + 1)
colorLens = JmagLens - kMagLens
#Get colour mag for 10000 source form tgas
querystring2mass = 'select j_m,k_m,parallax,phot_g_mean_mag from gaia_dr1_aux.gaia_source_2mass_xm where parallax > 0.0001 and j_m > 0.0001 and k_m > 0.0001 limit 100000'
Jmag, kMag, parallax, gMag = sqlutil.get(querystring2mass,
db='wsdb',
host='cappc127.ast.cam.ac.uk',
user='******',
password='******')
#Calculate colour and absolute mags
color = Jmag - kMag
absmag = gMag + 5 * (np.log10(parallax / 1000.0) + 1)
#plot 2dhist with scatter of lens overlaid
plt.hist2d(color,
absmag,
bins=300,
range=[[-0.1, 1.25], [-2, 13.5]],
cmap='plasma')
plt.scatter([colorLens], [absMagLens], color='white', label='TGAS Lens')
plt.gca().invert_yaxis()
plt.xlabel(r'J-K [2MASS]')
plt.ylabel('Absolute G [mag]')
plt.legend()
plt.savefig(filename, dpi=200)
plt.clf()
def get_gaia_source_pos(centerRa, centerDec, size):
"""
Get all the gaia source positions are epoch 2015.0
in a search radius of size.
Args:
centerRA (double) : Right acession of the center of
the search radius [Degrees]
centerDec (souble) : Declination of the center of the
the search radius [Degrees]
size (double) : size of the search radius [arminutes]
Returns:
pos (np.array(double)) : Position of gaia sources [
[Ra,Ra],[dec,dec]..]
"""
query = 'select ra,dec from gaia_dr1.gaia_source where ra BETWEEN ' + str(
centerRa) + ' - 0.016 and ' + str(
centerRa) + '+ 0.016 AND dec BETWEEN ' + str(
centerDec) + '- 0.016 and ' + str(centerDec) + '+ 0.016'
ra, dec = sqlutil.get(query,
db='wsdb',
host='cappc127.ast.cam.ac.uk',
user='******',
password='******')
return [ra, dec]
def getdata(cached=True):
if not cached:
import sqlutil
Q = 'select fieldid, sky_u,sky_g,sky_r,sky_i,sky_z,ramin as ra,decmin as dec,mjd_r as mjd from sdssdr9.field '
D = sqlutil.get(Q, host='cappc127', asDict=True)
fieldid, sky_u, sky_g, sky_r, sky_i, sky_z, ra, dec, mjd = D.values()
tab = atpy.Table(data=D)
tab.write('sdss_sky.fits', format='fits', overwrite=True)
else:
tab = atpy.Table().read('sdss_sky.fits')
fieldid, sky_u, sky_g, sky_r, sky_i, sky_z, ra, dec, mjd = [
tab[_] for _ in
'fieldid,sky_u,sky_g,sky_r,sky_i,sky_z,ra,dec,mjd'.split(',')
]
return fieldid, sky_u, sky_g, sky_r, sky_i, sky_z, ra, dec, mjd
def queryDatabase(query, columns, post_query, cache_file='db.pickle'):
#cache_file = "reducedPM2.pickle"
if not path.exists(cache_file):
res = sqlutil.get(query.format(','.join(columns), post_query),
db='wsdb', host='cappc127.ast.cam.ac.uk',
user=user, password=password,
asDict=True)
with open(cache_file, "wb") as f:
pickle.dump(res, f)
else:
with open(cache_file, "rb") as f:
res = pickle.load(f)
return res
def nearest_neighbour(tile, outdir, checkplots=False, saveplots=False):
"""
"""
from astropy.table import Table, hstack
import srpylib
import match_lists
import matplotlib.pyplot as plt
import numpy as np
db, host, user, password, db_table = rd_config_wsdb(table='wise')
t = Table.read(outdir + tile + "_WISEfp_DEScat.fits")
ra_min, ra_max, dec_min, dec_max = srpylib.min_max_match(
t["RA_CALC_G"], t["DEC_CALC_G"], 1.0, DES_box=True
)
query_wise = "select ra, dec FROM allwise.main WHERE ra > " + \
ra_min + " and ra < " + ra_max + " and dec > " + dec_min + \
" and dec < " + dec_max
RA, DEC = sqlutil.get(query_wise, db=db, host=host,
user=user, password=password)
dists, inds = match_lists.match_lists(
t["RA_CALC_G"].data, t["DEC_CALC_G"].data, RA, DEC, 1.0, 2
)
match_distance = dists[:,1]*3600.0
if checkplots:
plt.axvline(6.1)
plt.axvline(6.4)
plt.axvline(6.5)
plt.axvline(12.0)
plt.hist(match_distance, bins=100)
plt.xlabel('Match Distance(")')
plt.ylabel("Number")
plt.title("Nearest Neighbours:" + tile)
plt.show()
print("Median Distance:", np.median(match_distance))
def get_closes_gaia_source_match(centerRa, centerDec, size):
"""
Get all the gaia source information for all gaia sources
within a small search radius. And have low proper motions
Args:
centerRA (double) : Right acession of the center of
the search radius [Degrees]
centerDec (souble) : Declination of the center of the
the search radius [Degrees]
size (double) : size of the search radius [arminutes]
Returns:
pos (np.array(double)) : Position of gaia sources [
[[id, id,..],[Ra,Ra],[dec,dec],[gmag,gmag...]..]
"""
query = 'select ra,dec, source_id, phot_g_mag_mean from gaia_dr1.gaia_source where ra BETWEEN ' + str(
centerRa) + ' - 0.026 and ' + str(
centerRa) + '+ 0.026 AND dec BETWEEN ' + str(
centerDec) + '- 0.026 and ' + str(
centerDec) + '+ 0.026 AND hypot(pmra,pmdec) < 50.0'
ra, dec, id, gmag = sqlutil.get(query,
db='wsdb',
host='cappc127.ast.cam.ac.uk',
user='******',
password='******')
return [ra, dec, id, gmag]
if vmin is None:
vmin = 0
if vmax is None:
vmax = hh.max()
if weight_norm:
hh = hh * 1. / (hhsum + 1 * (hhsum == 0))
if skip_empty:
hh1 = hh[hhsum > 0]
else:
hh1 = hh
coll.set_array(hh1)
coll.set_cmap(cmap)
coll.set_clim(vmin, vmax)
coll.set_edgecolors(coll.cmap(coll.norm(hh1)))
coll.set_rasterized(rasterized)
tmpcoll.set_clim(vmin, vmax)
ax.add_collection(coll)
return coll
if __name__=='__main__':
ras,decs=sqlutil.get('select radeg,dedeg from sdss_phot_qso.main',
host='cappc118')
doit(ras,decs, ramin=0,ramax=360,decmin=-90, decmax=90, nside=64)
plt.savefig('xx.png',dpi=200)
def VHS_match(t, RA_main, DEC_main, c_graph = True, width = 0.0004, w_units = "degrees", survey = "des", bands = ["Y", "J", "H", "K"], DES_box = False):
"""
Matches to VHS Data from WSDB
Tables used are vhs_1504. + survey
DES_box returns all objects within a box of about the size of a DES tile centred
on the median RA and DEC of the table.
Iterates through a table where the RA column is RA_main and the DEC_column is
DEC_main
Puts NaNs in for no matches
c_graph plots diagnostic graphs of the match - crashes if there are no matches
Can specify not the full number of bands
#29/03/16 - Changed from 1504 to 1603
"""
import numpy as np
import match_lists
import sqlutil
from astropy.table import Table, vstack
import ConfigParser
config_file = "/home/sr525/Python_Code/srpylib/srpylib.cfg"
config = ConfigParser.RawConfigParser()
config.read(config_file)
uname = config.get("wsdb", "uname")
pword = config.get("wsdb", "pword")
host = config.get("wsdb", "host")
if w_units == "arcsecs":
width = width / 3600.0
if survey == "des":
cols = ["RA_VHS", "DEC_VHS", "J", "J_ERR", "H", "H_ERR", "K", "K_ERR"]
elif survey == "atlas":
cols = ["RA_VHS", "DEC_VHS", "Y", "Y_ERR", "J", "J_ERR", "H", "H_ERR", \
"K", "K_ERR"]
try:
for col in cols:
t.add_empty_column(col, dtype = np.float64)
except AttributeError as e:
for col in cols:
t[col] = np.zeros(len(t))
ra_min, ra_max, dec_min, dec_max = min_max_match(t[RA_main], t[DEC_main], \
width, DES_box = DES_box)
area = (float(ra_max)-float(ra_min))*(float(dec_max)-float(dec_min))
print area
if len(t) > 1000 and area < 2000:
ra_min, ra_max, dec_min, dec_max = min_max_match(t[RA_main], t[DEC_main], \
width, DES_box = DES_box)
print "Retreiving data"
if survey == "atlas":
query_vhs = "select ra, dec, yapercormag3, yapercormag3_err, " + \
"japercormag3, japercormag3_err, hapercormag3, " + \
"hapercormag3_err, kapercormag3, kapercormag3_err " + \
"FROM vhs_1603." + survey + " WHERE ra > " + ra_min + \
" and ra < " + ra_max + " and dec > " + dec_min + \
" and dec < " + dec_max
RA_vhs, DEC_vhs, Y, Yerr, J, Jerr, H, Herr, K, Kerr = \
sqlutil.get(query_vhs, db="wsdb", host=host, user=uname, password=pword)
elif survey == "des":
query_vhs = "select ra, dec, japercormag3, japercormag3_err, " + \
"hapercormag3, hapercormag3_err, kapercormag3, " + \
"kapercormag3_err FROM vhs_1603." + survey + \
" WHERE ra > " + ra_min + " and ra < " + ra_max + \
" and dec > " + dec_min + " and dec < " + dec_max
RA_vhs, DEC_vhs, J, Jerr, H, Herr, K, Kerr = \
sqlutil.get(query_vhs, db="wsdb", host=host, user=uname, password=pword)
print "Retreived data"
if len(RA_vhs) == 0:
for (i, col) in enumerate(cols):
t[col] = [np.float64(np.nan)]*len(t)
else:
dists, inds = match_lists.match_lists(t[RA_main], t[DEC_main], RA_vhs, DEC_vhs, width, 1)
if survey == "des":
info = [RA_vhs, DEC_vhs, J, Jerr, H, Herr, K, Kerr]
elif survey == "atlas":
info = [RA_vhs, DEC_vhs, Y, Yerr, J, Jerr, H, Herr, K, Kerr]
match = np.where( (inds <> len(RA_vhs)) )[0]
no_match = np.where( (inds == len(RA_vhs)) )[0]
for (i, col) in enumerate(cols):
t[col][match] = info[i][inds[match]]
t[col][no_match] = np.float64(np.nan)
if len(t) <= 1000:
n = 0
while n < len(t):
RA = t[RA_main][n]
DEC = t[DEC_main][n]
if survey == "atlas":
query_vhs = "select ra, dec, yapercormag3, yapercormag3_err, " + \
"japercormag3, japercormag3_err, hapercormag3, " + \
"hapercormag3_err, kapercormag3, kapercormag3_err " + \
"FROM vhs_1504." + survey + \
" WHERE q3c_radial_query(ra, dec, " + str(RA) + \
", " + str(DEC) + ", 0.1102)"
RA_vhs, DEC_vhs, Y, Yerr, J, Jerr, H, Herr, K, Kerr = \
sqlutil.get(query_vhs, db="wsdb", host=host, user=uname, password=pword)
elif survey == "des":
query_vhs = "select ra, dec, japercormag3, japercormag3_err, " + \
"hapercormag3, hapercormag3_err, kapercormag3, " + \
"kapercormag3_err FROM vhs_1504." + survey + \
" WHERE q3c_radial_query(ra, dec, " + str(RA) + \
", " + str(DEC) + ", 0.1102)"
RA_vhs, DEC_vhs, J, Jerr, H, Herr, K, Kerr = \
sqlutil.get(query_vhs, db="wsdb", host=host, user=uname, password=pword)
if len(RA_vhs) > 0:
dists, inds = match_lists.match_lists([RA], [DEC], RA_vhs, DEC_vhs, width, 1)
j = inds[0]
if j <> len(RA_vhs):
if survey == "des":
info = [RA_vhs[j], DEC_vhs[j], J[j], Jerr[j], H[j], Herr[j], \
K[j], Kerr[j]]
if survey == "atlas":
info = [RA_vhs[j], DEC_vhs[j], Y[j], Yerr[j], J[j], Jerr[j], \
H[j], Herr[j], K[j], Kerr[j]]
for (k, col) in enumerate(cols):
t[col][n] = info[k]
else:
for (k, col) in enumerate(cols):
t[col][n] = np.float64(np.nan)
else:
for (k, col) in enumerate(cols):
t[col][n] = np.float64(np.nan)
n += 1
print "Done VHS"
if area > 2000 and len(t) > 1000:
t.sort(RA_main)
l1 = np.ceil(area/1001.0)
l = len(t)/l1
i = 0
while i < l1:
if (i+1)*l < len(t):
t1 = t[i*l:(i+1)*l]
else:
t1 = t[i*l:]
t1 = VHS_match(t1, RA_main, DEC_main, c_graph = c_graph, width = width, \
w_units = w_units, survey = survey, bands = bands, \
DES_box = DES_box)
if i == 0:
t_out = t1
else:
t_out = vstack([t_out, t1])
i += 1
t = t_out
if c_graph:
width = width *3600.0
RA_med, DEC_med = check_graph(t, RA_main, DEC_main, "VHS", width)
return t
def GALEX_match(t, RA_main, DEC_main, width = 0.0004, w_units = "degrees", c_graph = True, table = "ais"):
"""
Matches to GALEX table
DES_box returns all objects within a box of about the size of a DES tile centred
on the median RA and DEC of the table.
Puts NaNs in for no matches
c_graph plots diagnostic graphs of the match
"""
import numpy as np
import match_lists
import sqlutil
import ConfigParser
config_file = "/home/sr525/Python_Code/srpylib/srpylib.cfg"
config = ConfigParser.RawConfigParser()
config.read(config_file)
uname = config.get("wsdb", "uname")
pword = config.get("wsdb", "pword")
host = config.get("wsdb", "host")
if w_units == "arcsecs":
width = width / 3600.0
cols = ["RA_GALEX", "DEC_GALEX", "NUV_MAG", "NUV_MAGERR", "FUV_MAG", \
"FUV_MAGERR", "E_BV", "NUV_CLASS_STAR", "FUV_CLASS_STAR"]
try:
for col in cols:
t.add_empty_column(col, dtype = np.float64)
except AttributeError as e:
for col in cols:
t[col] = np.zeros(len(t))
if len(t) > 1000:
ra_min, ra_max, dec_min, dec_max = min_max_match(t[RA_main], t[DEC_main], \
width, DES_box = DES_box)
print "Retreiving data"
query = "select ra, dec, nuv_mag, nuv_magerr, fuv_mag, fuv_magerr, e_bv, \
nuv_class_star, fuv_class_star FROM galexgr6." + table + "primary \
WHERE ra > " + ra_min + " and ra < " + ra_max + " and dec > " + \
dec_min + " and dec < " + dec_max
RA_galex, DEC_galex, nuv_mag, nuv_magerr, fuv_mag, fuv_magerr, e_bv, \
nuv_class_star, fuv_class_star = \
sqlutil.get(query, db = "wsdb", host=host, user=uname, password=pword)
print "Retreived data"
if len(RA_galex) == 0:
for (i, col) in enumerate(cols):
t[col] = [np.float64(np.nan)]*len(t)
else:
dists, inds = match_lists.match_lists(t[RA_main], t[DEC_main], RA_vhs, DEC_vhs, width, 1)
info = [ra_galex, dec_galex, nuv_mag, nuv_magerr, fuv_mag, fuv_magerr, \
e_bv, nuv_class_star, fuv_class_star]
match = np.where( (inds <> len(RA_galex)) )[0]
no_match = np.where( (inds == len(RA_galex)) )[0]
for (i, col) in enumerate(cols):
t[col][match] = info[i][inds[match]]
t[col][no_match] = np.float64(np.nan)
elif len(t) <= 1000:
n = 0
while n < len(t):
RA = t[RA_main][n]
DEC = t[DEC_main][n]
query_galex = "select ra, dec, nuv_mag, nuv_magerr, fuv_mag, \
fuv_magerr, e_bv, nuv_class_star, fuv_class_star FROM \
galexgr6." + table + "primary WHERE \
q3c_radial_query(ra, dec, " + str(RA) + ", " + str(DEC) + ", 0.1102)"
RA_galex, DEC_galex, nuv_mag, nuv_magerr, fuv_mag, fuv_magerr, e_bv, \
nuv_class_star, fuv_class_star = sqlutil.get(query_galex, db = "wsdb", \
host=host, user=uname, password=pword)
if len(RA_galex) > 0:
dists, inds = match_lists.match_lists([RA], [DEC], RA_galex, DEC_galex, width, 1)
j = inds[0]
if j <> len(RA_galex):
info = [RA_galex[j], DEC_galex[j], nuv_mag[j], nuv_magerr[j], \
fuv_mag[j], fuv_magerr[j], e_bv[j], nuv_class_star[j], \
fuv_class_star[j]]
for (k, col) in enumerate(cols):
t[col][n] = info[rki]
else:
for (k, col) in enumerate(cols):
t[col][n] = np.float64(np.nan)
else:
for (k, col) in enumerate(cols):
t[col][n] = np.float64(np.nan)
n += 1
print "Done GALEX"
elif len(t) > 1000000000:
print "Splitting by tile"
tiles_all = t["TILENAME"]
tiles = set(tiles_all)
tiles = list(tiles)
print len(tiles), "distinct tiles"
t.sort("TILENAME")
tiles.sort()
n = 0
m = 0
rows = []
tile_rows = []
while n < len(t):
tile = tiles[m]
if t["TILENAME"][n] == tile:
rows.append(n)
n += 1
else:
tile_rows.append(rows)
rows = []
m += 1
for rows in tile_rows:
t1 = t.rows(rows)
print "Matching", t1["TILENAME"][0], "with length", len(t1)
t1 = GALEX_match(t1, RA_main, DEC_main, width, c_graph = False)
if tile_rows.index(rows) == 0:
t_out = t1
else:
t_out.append(t1)
t = t_out
if c_graph:
width = width *3600.0
RA_med, DEC_med = check_graph(t, RA_main, DEC_main, "GALEX", width)
return t
def VST_match(t, RA_main, DEC_main, c_graph = True, width = 0.0004, w_units = "degrees", DES_box = False):
"""
Matches to VST Data from WSDB
Table used is vst_201310.atlas_rgi
DES_box returns all objects within a box of about the size of a DES tile centred
on the median RA and DEC of the table.
Iterates through a table where the RA column is RA_main and the DEC_column is
DEC_main
Puts NaNs in for no matches
c_graph plots diagnostic graphs of the match - crashes if there are no matches
"""
import numpy as np
import match_lists
import sqlutil
import ConfigParser
config_file = "/home/sr525/Python_Code/srpylib/srpylib.cfg"
config = ConfigParser.RawConfigParser()
config.read(config_file)
uname = config.get("wsdb", "uname")
pword = config.get("wsdb", "pword")
host = config.get("wsdb", "host")
if w_units == "arcsecs":
width = width / 3600.0
cols = ["RA_VST", "DEC_VST", "G_VST", "G_VST_ERR", "R_VST", "R_VST_ERR", \
"I_VST", "I_VST_ERR"]
try:
for col in cols:
t.add_empty_column(col, dtype = np.float64)
except AttributeError as e:
for col in cols:
t[col] = np.zeros(len(t))
if len(t) > 1000 and len(t) < 1000000000:
ra_min, ra_max, dec_min, dec_max = min_max_match(t[RA_main], t[DEC_main], \
width, DES_box = DES_box)
print "Retreiving data"
query = "select ra, dec, mag_g, magerr_g, mag_r, magerr_r, mag_i, " + \
"magerr_i FROM vst_201310.atlas_rgi WHERE ra > " + \
ra_min + " and ra < " + ra_max + " and dec > " + \
dec_min + " and dec < " + dec_max
RA_vst, DEC_vst, g, gerr, r, rerr, i, ierr = \
sqlutil.get(query, db="wsdb", host=host, user=uname, password=pword)
print "Retreived data"
dists, inds = match_lists.match_lists(t[RA_main], t[DEC_main], RA_vst, DEC_vst, width, 1)
print "Matched lists, updating table"
info = [RA_vst, DEC_vst, g, gerr, r, rerr, i, ierr]
match = np.where( (inds <> len(RA_vst)) )[0]
no_match = np.where( (inds == len(RA_vst)) )[0]
for (i, col) in enumerate(cols):
t[col][match] = info[i][inds[match]]
t[col][no_match] = np.float64(np.nan)
print "Table updated"
elif len(t) <= 1000:
n = 0
while n < len(t):
RA = t[RA_main][n]
DEC = t[DEC_main][n]
query_vhs = "select ra, dec, mag_g, magerr_g, mag_r, magerr_r, " + \
"mag_i, magerr_i FROM vst_201310.atlas_rgi " + \
"WHERE q3c_radial_query(ra, dec, " + str(RA) + ", " + \
str(DEC) + ", 0.1102)"
RA_vst, DEC_vst, g, gerr, r, rerr, i, ierr = \
sqlutil.get(query_vhs, db="wsdb", host=host, user=uname, password=pword)
if len(RA_vst) > 0:
dists, inds = match_lists.match_lists([RA], [DEC], RA_vst, DEC_vst, width, 1)
j = inds[0]
if j <> len(RA_vst):
info = [RA_vst[j], DEC_vst[j], g[j], gerr[j], r[j], rerr[j], \
i[j], ierr[j]]
for (k, col) in enumerate(cols):
t[col][n] = info[k]
else:
for (k, col) in enumerate(cols):
t[col][n] = np.float64(np.nan)
else:
for (k, col) in enumerate(cols):
t[col][n] = np.float64(np.nan)
n += 1
elif len(t) > 1000000000:
print "Splitting by tile"
tiles_all = t["TILENAME"]
tiles = set(tiles_all)
tiles = list(tiles)
print len(tiles), "distinct tiles"
t.sort("TILENAME")
tiles.sort()
n = 0
m = 0
rows = []
tile_rows = []
while n < len(t):
tile = tiles[m]
if t["TILENAME"][n] == tile:
rows.append(n)
n += 1
else:
tile_rows.append(rows)
rows = []
m += 1
for rows in tile_rows:
t1 = t.rows(rows)
print "Matching", t1["TILENAME"][0], "with length", len(t1)
t1 = VST_match(t1, RA_main, DEC_main, width, c_graph = False)
if tile_rows.index(rows) == 0:
t_out = t1
else:
t_out.append(t1)
t = t_out
if c_graph:
width = width *3600.0
RA_med, DEC_med = check_graph(t, RA_main, DEC_main, "VST", width)
return t
def WISE_match(t, RA_main, DEC_main, width = 0.0004, w_units = "degrees", c_graph = True, DES_box = False):
"""
Matches to WISE Data from WSDB
Table used is allwise.main
DES_box returns all objects within a box of about the size of a DES tile centred
on the median RA and DEC of the table.
Iterates through a table where the RA column is RA_main and the DEC_column is
DEC_main
Puts NaNs in for no matches
c_graph plots diagnostic graphs of the match - crashes if there are no matches
"""
import numpy as np
import match_lists
import sqlutil
import ConfigParser
config_file = "/home/sr525/Python_Code/srpylib/srpylib.cfg"
config = ConfigParser.RawConfigParser()
config.read(config_file)
uname = config.get("wsdb", "uname")
pword = config.get("wsdb", "pword")
host = config.get("wsdb", "host")
if w_units == "arcsecs":
width = width / 3600.0
cols = ["RA_WISE", "DEC_WISE", "W1", "W1_ERR", "W2", "W2_ERR", "W3", "W3_ERR", "W4", "W4_ERR"]
try:
for col in cols:
t.add_empty_column(col, dtype = np.float64)
except AttributeError as e:
for col in cols:
t[col] = np.zeros(len(t))
n = 0
if len(t) > 5000 and len(t) < 100000000:
ra_min, ra_max, dec_min, dec_max = min_max_match(t[RA_main], t[DEC_main], \
width, DES_box = DES_box)
print "Retreiving data"
query_wise = "select ra, dec, w1mpro, w1sigmpro, w2mpro, w2sigmpro, " + \
"w3mpro, w3sigmpro, w4mpro, w4sigmpro FROM " + \
"allwise.main WHERE ra > " + ra_min + " and ra < " + \
ra_max + " and dec > " + dec_min + " and dec < " + dec_max
RA_wise, DEC_wise, W1, W1err, W2, W2err, W3, W3err, W4, W4err = \
sqlutil.get(query_wise, db="wsdb", host=host, user=uname, password=pword)
info = [RA_wise, DEC_wise, W1, W1err, W2, W2err, W3, W3err, W4, W4err]
if len(RA_wise) == 0:
for (i, col) in enumerate(cols):
t[col] = [np.float64(np.nan)]*len(t)
else:
print "Retreived data, matching lists"
dists, inds = match_lists.match_lists(t[RA_main], t[DEC_main], RA_wise, DEC_wise, width, 1)
print "Matched lists, updating table"
match = np.where( (inds <> len(RA_wise)) )[0]
no_match = np.where( (inds == len(RA_wise)) )[0]
for (i, col) in enumerate(cols):
t[col][match] = info[i][inds[match]]
t[col][no_match] = np.float64(np.nan)
print "Table updated"
elif len(t) <= 5000:
while n < len(t):
RA = t[RA_main][n]
DEC = t[DEC_main][n]
query = "select ra, dec, w1mpro, w1sigmpro, w2mpro, w2sigmpro, " + \
"w3mpro, w3sigmpro, w4mpro, w4sigmpro FROM allwise.main " + \
"WHERE q3c_radial_query(ra, dec, " + str(RA) + ", " + \
str(DEC) + ", 0.1002)"
RA_W, DEC_W, W1, W1err, W2, W2err, W3, W3err, W4, W4err = \
sqlutil.get(query, db='wsdb',host=host, user=uname, password=pword)
if len(RA_W) > 0:
dists, inds = match_lists.match_lists([RA], [DEC], RA_W, DEC_W, width, 1)
j = inds[0]
if j <> len(RA_W):
info = [RA_W[j], DEC_W[j], W1[j], W1err[j], W2[j], W2err[j], \
W3[j], W3err[j], W4[j], W4err[j]]
for (i, col) in enumerate(cols):
t[col][n] = info[i]
else:
for (i, col) in enumerate(cols):
t[col][n] = np.float64(np.nan)
else:
for (i, col) in enumerate(cols):
t[col][n] = np.float64(np.nan)
n += 1
elif len(t) > 1000000000:
print "Splitting by tile"
tiles_all = t["TILENAME"]
tiles = set(tiles_all)
tiles = list(tiles)
print len(tiles), "distinct tiles"
t.sort("TILENAME")
tiles.sort()
n = 0
m = 0
rows = []
tile_rows = []
while n < len(t):
tile = tiles[m]
if t["TILENAME"][n] == tile:
rows.append(n)
n += 1
else:
tile_rows.append(rows)
rows = []
m += 1
for rows in tile_rows:
t1 = t.rows(rows)
print "Matching", t1["TILENAME"][0], "with length", len(t1)
t1 = WISE_match(t1, RA_main, DEC_main, width, c_graph = False)
if tile_rows.index(rows) == 0:
t_out = t1
else:
t_out.append(t1)
t = t_out
if c_graph:
width = width *3600.0
RA_med, DEC_med = check_graph(t, RA_main, DEC_main, "WISE", width, gtype = "square")
return t
def SDSS_match(t, RA_main, DEC_main, width = 0.0004, w_units = "degrees", c_graph = True, dr = "dr9", DES_box = False):
"""
Matches to SDSS Data from WSDB
Table used is dr + .phototag
DES_box returns all objects within a box of about the size of a DES tile centred
on the median RA and DEC of the table.
Iterates through a table where the RA column is RA_main and the DEC_column is
DEC_main
Puts NaNs in for no matches
c_graph plots diagnostic graphs of the match - crashes if there are no matches
"""
import numpy as np
import match_lists
import sqlutil
from astropy.table import Table, vstack
import ConfigParser
config_file = "/home/sr525/Python_Code/srpylib/srpylib.cfg"
config = ConfigParser.RawConfigParser()
config.read(config_file)
uname = config.get("wsdb", "uname")
pword = config.get("wsdb", "pword")
host = config.get("wsdb", "host")
if w_units == "arcsecs":
width = width / 3600.0
cols = ["RA_SDSS", "DEC_SDSS", "MAG_PSF_U_SDSS", "MAGERR_PSF_U_SDSS", \
"MAG_PSF_G_SDSS", "MAGERR_PSF_G_SDSS", "MAG_PSF_R_SDSS", \
"MAGERR_PSF_R_SDSS", "MAG_PSF_I_SDSS", "MAGERR_PSF_I_SDSS", \
"MAG_PSF_Z_SDSS", "MAGERR_PSF_Z_SDSS", "MAG_MODEL_U_SDSS", \
"MAGERR_MODEL_U_SDSS", "MAG_MODEL_G_SDSS", "MAGERR_MODEL_G_SDSS", \
"MAG_MODEL_R_SDSS", "MAGERR_MODEL_R_SDSS", "MAG_MODEL_I_SDSS", \
"MAGERR_MODEL_I_SDSS", "MAG_MODEL_Z_SDSS", "MAGERR_MODEL_Z_SDSS", "TYPE"]
try:
for col in cols:
t.add_empty_column(col, dtype = np.float64)
except AttributeError as e:
for col in cols:
t[col] = np.zeros(len(t))
n = 0
if len(t) > 100 and len(t) < 1000000:
print "Finding max and mins"
ra_min, ra_max, dec_min, dec_max = min_max_match(t[RA_main], t[DEC_main], \
width, DES_box = DES_box)
print "Retreiving data"
query_sdss = "select ra, dec, psfMag_u, psfMagErr_u, psfMag_g, " + \
"psfMagErr_g, psfMag_r, psfMagErr_r, psfMag_i, psfMagErr_i, " + \
"psfMag_z, psfMagErr_z, modelMag_u, modelMagErr_u, " + \
"modelMag_g, modelMagErr_g, modelMag_r, modelMagErr_r, " + \
"modelMag_i, modelMagErr_i, modelMag_z, modelMagErr_z, " + \
"type FROM sdss" + dr + ".phototag WHERE ra > " + ra_min + \
" and ra < " + ra_max + " and dec > " + dec_min + \
" and dec < " + dec_max
RA_sdss, DEC_sdss, psfu, psfuerr, psfg, psfgerr, psfr, psfrerr, psfi, \
psfierr, psfz, psfzerr, modelu, modeluerr, modelg, modelgerr, modelr, \
modelrerr, modeli, modelierr, modelz, modelzerr, types = \
sqlutil.get(query_sdss, db = "wsdb", host = host, user = uname, password = pword)
info = [RA_sdss, DEC_sdss, psfu, psfuerr, psfg, psfgerr, psfr, psfrerr, \
psfi, psfierr, psfz, psfzerr, modelu, modeluerr, modelg, modelgerr, \
modelr, modelrerr, modeli, modelierr, modelz, modelzerr, types]
if len(RA_sdss) == 0:
for (i, col) in enumerate(cols):
t[col] = [np.float64(np.nan)]*len(t)
else:
print "Retreived data, matching lists"
dists, inds = match_lists.match_lists(t[RA_main], t[DEC_main], RA_sdss, DEC_sdss, width, 1)
print "Matched lists, updating table"
match = np.where( (inds <> len(RA_sdss)) )[0]
no_match = np.where( (inds == len(RA_sdss)) )[0]
for (i, col) in enumerate(cols):
t[col][match] = info[i][inds[match]]
t[col][no_match] = np.float64(np.nan)
print "Table updated"
elif len(t) <= 100:
while n < len(t):
RA = t[RA_main][n]
DEC = t[DEC_main][n]
print RA, DEC
query_sdss = "select ra, dec, psfMag_u, psfMagErr_u, psfMag_g, " + \
"psfMagErr_g, psfMag_r, psfMagErr_r, psfMag_i, " + \
"psfMagErr_i, psfMag_z, psfMagErr_z, modelMag_u, " + \
"modelMagErr_u, modelMag_g, modelMagErr_g, modelMag_r, " + \
"modelMagErr_r, modelMag_i, modelMagErr_i, modelMag_z, " + \
"modelMagErr_z, type FROM sdss" + dr + \
".phototag WHERE q3c_radial_query(ra, dec, " + str(RA) + \
", " + str(DEC) + ", 0.1002)"
RA_sdss, DEC_sdss, psfu, psfuerr, psfg, psfgerr, psfr, psfrerr, psfi, \
psfierr, psfz, psfzerr, modelu, modeluerr, modelg, modelgerr, modelr, \
modelrerr, modeli, modelierr, modelz, modelzerr, stype = \
sqlutil.get(query_sdss, db = "wsdb", host = host, user = uname, password = pword)
if len(RA_sdss) > 0:
dist, inds = match_lists.match_lists([RA], [DEC], RA_sdss, DEC_sdss, width, 1)
j = inds[0]
info = [RA_sdss[j], DEC_sdss[j], psfu[j], psfuerr[j], psfg[j], \
psfgerr[j], psfr[j], psfrerr[j], psfi[j], psfierr[j], \
psfz[j], psfzerr[j], modelu[j], modeluerr[j], modelg[j], \
modelgerr[j], modelr[j], modelrerr[j], modeli[j], \
modelierr[j], modelz[j], modelzerr[j], stype[j]]
for (i, col) in enumerate(cols):
t[col][n] = info[i]
else:
for (i, col) in enumerate(cols):
t[col][n] = np.float64(np.nan)
n += 1
elif len(t) > 1000000:
print "Splitting by tile"
tiles_all = t["TILENAME"]
tiles = set(tiles_all)
tiles = list(tiles)
print len(tiles), "distinct tiles"
t.sort("TILENAME")
tiles.sort()
n = 0
m = 0
rows = []
tile_rows = []
while n < len(t):
tile = tiles[m]
if t["TILENAME"][n] == tile:
rows.append(n)
n += 1
else:
print len(rows)
tile_rows.append(rows)
rows = []
m += 1
for rows in tile_rows:
try:
t1 = t.rows(rows)
except AttributeError:
t1 = t[rows]
print "Matching", t1["TILENAME"][0], "with length", len(t1)
t1 = SDSS_match(t1, RA_main, DEC_main, width, c_graph = False)
if tile_rows.index(rows) == 0:
t_out = t1
else:
try:
t_out.append(t1)
except AttributeError:
t_out = vstack([t_out, t1])
t = t_out
if c_graph:
width = width *3600.0
RA_med, DEC_med = check_graph(t, RA_main, DEC_main, "SDSS", width, gtype = "square")
return t
def DES_match(t, RA_main, DEC_main, c_graph = True, width = 0.0004, w_units = "degrees", release = "Y1A1"):
"""
Matches to DES data ingested into WSDB
Tables used are des_y1a1.coadd_objects and des_y2q1.objects
Iterates through a table where the RA column is RA_main and the DEC_column is
DEC_main
Puts NaNs in for no matches
c_graph plots diagnostic graphs of the match - crashes if there are no matches
"""
import numpy as np
import match_lists
import sqlutil
from astropy.table import Table, vstack
import ConfigParser
config_file = "/home/sr525/Python_Code/srpylib/srpylib.cfg"
config = ConfigParser.RawConfigParser()
config.read(config_file)
uname = config.get("wsdb", "uname")
pword = config.get("wsdb", "pword")
host = config.get("wsdb", "host")
if w_units == "arcsecs":
width = width / 3600.0
if release == "Y1A1":
cols = ["COADD_OBJECTS_ID", "RA_DES_Y1A1", "DEC_DES_Y1A1", "TILENAME_Y1A1", \
"RUN_Y1A1", "MPSF_G_Y1A1", "MPSF_ERR_G_Y1A1", "MPSF_R_Y1A1", \
"MPSF_ERR_R_Y1A1", "MPSF_I_Y1A1", "MPSF_ERR_I_Y1A1", "MPSF_Z_Y1A1", \
"MPSF_ERR_Z_Y1A1", "MPSF_Y_Y1A1", "MPSF_ERR_Y_Y1A1", "MAPER3_G_Y1A1", \
"MAPER3_ERR_G_Y1A1", "MAPER3_R_Y1A1", "MAPER3_ERR_R_Y1A1", \
"MAPER3_I_Y1A1", "MAPER3_ERR_I_Y1A1", "MAPER3_Z_Y1A1", \
"MAPER3_ERR_Z_Y1A1", "MAPER3_Y_Y1A1", "MAPER3_ERR_Y_Y1A1"]
str_cols = ["TILENAME_Y1A1", "RUN_Y1A1"]
elif release == "Y2Q1":
cols = ["QUICK_OBJECT_ID", "RA_DES_Y2Q1", "DEC_DES_Y2Q1", "MPSF_G_Y2Q1", \
"MPSF_ERR_G_Y2Q1", "MPSF_R_Y2Q1", "MPSF_ERR_R_Y2Q1", "MPSF_I_Y2Q1", \
"MPSF_ERR_I_Y2Q1", "MPSF_Z_Y2Q1", "MPSF_ERR_Z_Y2Q1", "MPSF_Y_Y2Q1", \
"MPSF_ERR_Y_Y2Q1", "MAUTO_G_Y2Q1", "MAUTO_ERR_G_Y2Q1", \
"MAUTO_R_Y2Q1", "MAUTO_ERR_R_Y2Q1", "MAUTO_I_Y2Q1", \
"MAUTO_ERR_I_Y2Q1", "MAUTO_Z_Y2Q1", "MAUTO_ERR_Z_Y2Q1", \
"MAUTO_Y_Y2Q1", "MAUTO_ERR_Y_Y2Q1"]
str_cols = []
try:
for col in cols:
if col not in str_cols:
t.add_empty_column(col, dtype = np.float64)
else:
t.add_empty_col(col, dtype = np.string)
except AttributeError as e:
for col in cols:
if col not in str_cols:
t[col] = np.zeros(len(t))
else:
t[col] = ["AAAAAAAAAAAAAAAAAAAAAAAAAA"]
ra_min, ra_max, dec_min, dec_max = min_max_match(t[RA_main], t[DEC_main], \
width, DES_box = False)
area = (float(ra_max)-float(ra_min))*(float(dec_max)-float(dec_min))
print area
if len(t) > 1000 and area < 2000:
ra_min, ra_max, dec_min, dec_max = min_max_match(t[RA_main], t[DEC_main], \
width, DES_box = False)
print "Retreiving data"
if release == "Y1A1":
query_des = "select COADD_OBJECTS_ID, RA, DEC, TILENAME, RUN, MAG_PSF_G, \
MAGERR_PSF_G, MAG_PSF_R, MAGERR_PSF_R, MAG_PSF_I, MAGERR_PSF_I, \
MAG_PSF_Z, MAGERR_PSF_Z, MAG_PSF_Y, MAGERR_PSF_Y, MAG_APER_3_G, \
MAGERR_APER_3_G, MAG_APER_3_R, MAGERR_APER_3_R, MAG_APER_3_I, \
MAGERR_APER_3_I, MAG_APER_3_Z, MAGERR_APER_3_Z, MAG_APER_3_Y, \
MAGERR_APER_3_Y FROM des_y1a1.coadd_objects WHERE \
ra > " + ra_min + " and ra < " + ra_max + " and dec > " + \
dec_min + " and dec < " + dec_max
co_id, RA_des, DEC_des, tile, run, G, Gerr, R, Rerr, I, Ierr, Z, Zerr, Y, \
Yerr, Ga, Gaerr, Ra, Raerr, Ia, Iaerr, Za, Zaerr, Ya, Yaerr = \
sqlutil.get(query_des, db="wsdb", host=host, user=uname, password=pword)
elif release == "Y2Q1":
query_des = "select QUICK_OBJECT_ID, RA, DEC, MAG_PSF_G, MAGERR_PSF_G, \
MAG_PSF_R, MAGERR_PSF_R, MAG_PSF_I, MAGERR_PSF_I, MAG_PSF_Z, \
MAGERR_PSF_Z, MAG_PSF_Y, MAGERR_PSF_Y, MAG_AUTO_G, \
MAGERR_AUTO_G, MAG_AUTO_R, MAGERR_AUTO_R, MAG_AUTO_I, \
MAGERR_AUTO_I, MAG_AUTO_Z, MAGERR_AUTO_Z, MAG_AUTO_Y, \
MAGERR_AUTO_Y FROM des_y2q1.objects WHERE ra > " + ra_min \
+ " and ra < " + ra_max + " and dec > " + dec_min + \
" and dec < " + dec_max
q_id, RA_des, DEC_des, G, Gerr, R, Rerr, I, Ierr, Z, Zerr, Y, Yerr, Ga, \
Gaerr, Ra, Raerr, Ia, Iaerr, Za, Zaerr, Ya, Yaerr = \
sqlutil.get(query_des, db="wsdb", host=host, user=uname, password=pword)
print "Retreived data"
if len(RA_des) == 0:
for (i, col) in enumerate(cols):
t[col] = [np.float64(np.nan)]*len(t)
else:
dists, inds = match_lists.match_lists(t[RA_main], t[DEC_main], RA_des, DEC_des, width, 1)
if release == "Y1A1":
info = [co_id, RA_des, DEC_des, tile, run, G, Gerr, R, Rerr, I, \
Ierr, Z, Zerr, Y, Yerr, Ga, Gaerr, Ra, Raerr, Ia, Iaerr, Za, Zaerr, Ya, Yaerr]
elif release == "Y2Q1":
info = [q_id, RA_des, DEC_des, G, Gerr, R, Rerr, I, Ierr, Z, Zerr, \
Y, Yerr, Ga, Gaerr, Ra, Raerr, Ia, Iaerr, Za, Zaerr, Ya, Yaerr]
match = np.where( (inds <> len(RA_des)) )[0]
no_match = np.where( (inds == len(RA_des)) )[0]
for (i, col) in enumerate(cols):
t[col][match] = info[i][inds[match]]
t[col][no_match] = np.float64(np.nan)
if len(t) <= 1000:
n = 0
while n < len(t):
RA = t[RA_main][n]
DEC = t[DEC_main][n]
if release == "Y1A1":
query_des = "select COADD_OBJECTS_ID, RA, DEC, TILENAME, RUN, MAG_PSF_G, \
MAGERR_PSF_G, MAG_PSF_R, MAGERR_PSF_R, MAG_PSF_I, MAGERR_PSF_I, \
MAG_PSF_Z, MAGERR_PSF_Z, MAG_PSF_Y, MAGERR_PSF_Y, MAG_APER_3_G, \
MAGERR_APER_3_G, MAG_APER_3_R, MAGERR_APER_3_R, MAG_APER_3_I, \
MAGERR_APER_3_I, MAG_APER_3_Z, MAGERR_APER_3_Z, MAG_APER_3_Y, \
MAGERR_APER_3_Y FROM des_y1a1.coadd_objects WHERE \
q3c_radial_query(ra, dec, " + str(RA) + ", " + str(DEC) + ", 0.1102)"
co_id, RA_des, DEC_des, tile, run, G, Gerr, R, Rerr, I, Ierr, Z, \
Zerr, Y, Yerr, Ga, Gaerr, Ra, Raerr, Ia, Iaerr, Za, Zaerr, Ya, Yaerr = \
sqlutil.get(query_des, db="wsdb", host=host, user=uname, password=pword)
elif release == "Y2Q1":
query_des = "select QUICK_OBJECT_ID, RA, DEC, MAG_PSF_G, MAGERR_PSF_G, MAG_PSF_R, \
MAGERR_PSF_R, MAG_PSF_I, MAGERR_PSF_I, MAG_PSF_Z, \
MAGERR_PSF_Z, MAG_PSF_Y, MAGERR_PSF_Y, MAG_AUTO_G, \
MAGERR_AUTO_G, MAG_AUTO_R, MAGERR_AUTO_R, MAG_AUTO_I, \
MAGERR_AUTO_I, MAG_AUTO_Z, MAGERR_AUTO_Z, MAG_AUTO_Y, \
MAGERR_AUTO_Y FROM des_y2q1.objects WHERE \
q3c_radial_query(ra, dec, " + str(RA) + ", " + str(DEC) + ", 0.1102)"
q_id, RA_des, DEC_des, G, Gerr, R, Rerr, I, Ierr, Z, Zerr, Y, Yerr, \
Ga, Gaerr, Ra, Raerr, Ia, Iaerr, Za, Zaerr, Ya, Yaerr = \
sqlutil.get(query_des, db="wsdb", host=host, user=uname, password=pword)
if len(RA_des) > 0:
dists, inds = match_lists.match_lists([RA], [DEC], RA_des, DEC_des, width, 1)
j = inds[0]
if j <> len(RA_des):
if release == "Y1A1":
info = [co_id[j], RA_des[j], DEC_des[j], tile[j], run[j], G[j], \
Gerr[j], R[j], Rerr[j], I[j], Ierr[j], Z[j], \
Zerr[j], Y[j], Yerr[j], Ga[j], Gaerr[j], Ra[j], \
Raerr[j], Ia[j], Iaerr[j], Za[j], Zaerr[j], Ya[j], Yaerr[j]]
if release == "Y2Q1":
info = [q_id[j], RA_des[j], DEC_des[j], G[j], Gerr[j], R[j], Rerr[j], \
I[j], Ierr[j], Z[j], Zerr[j], Y[j], Yerr[j], Ga[j], \
Gaerr[j], Ra[j], Raerr[j], Ia[j], Iaerr[j], Za[j], \
Zaerr[j], Ya[j], Yaerr[j]]
for (k, col) in enumerate(cols):
t[col][n] = info[k]
else:
for (k, col) in enumerate(cols):
t[col][n] = np.float64(np.nan)
else:
for (k, col) in enumerate(cols):
t[col][n] = np.float64(np.nan)
n += 1
print "Done DES"
if area > 2000 and len(t) > 1000:
t.sort(RA_main)
l1 = np.ceil(area/1001.0)
l = len(t)/l1
i = 0
while i < l1:
if (i+1)*l < len(t):
t1 = t[i*l:(i+1)*l]
else:
t1 = t[i*l:]
t1 = DES_match(t1, RA_main, DEC_main, c_graph = False, width = width, \
w_units = w_units, release = release)
if i == 0:
t_out = t1
else:
t_out = vstack([t_out, t1])
i += 1
t = t_out
if c_graph:
width = width *3600.0
if release == "Y1A1":
survey = "DES_Y1A1"
elif release == "Y2Q1":
survey = "DES_Y2Q1"
RA_med, DEC_med = check_graph(t, RA_main, DEC_main, survey, width)
return t
def UKIDSS_match(t, RA_main, DEC_main, c_graph = True, width = 0.0004, w_units = "degrees", DES_box = False):
"""
Matches to VHS Data from WSDB
Tables used are vhs_1504. + survey
DES_box returns all objects within a box of about the size of a DES tile centred
on the median RA and DEC of the table.
Iterates through a table where the RA column is RA_main and the DEC_column is
DEC_main
Puts NaNs in for no matches
c_graph plots diagnostic graphs of the match - crashes if there are no matches
Can specify not the full number of bands
"""
import numpy as np
import match_lists
import sqlutil
from astropy.table import Table, vstack
import ConfigParser
config_file = "srpylib.cfg"
uname = config.get("wsdb", uname)
pword = config.get("wsdb", pword)
host = config.get("wsdb", host)
if w_units == "arcsecs":
width = width / 3600.0
cols = ["RA_UKIDSS", "DEC_UKIDSS", "Y_UKIDSS", "Y_ERR_UKIDSS", "J_UKIDSS", \
"J_ERR_UKIDSS", "H_UKIDSS", "H_ERR_UKIDSS", "K_UKIDSS", "K_ERR_UKIDSS"]
try:
for col in cols:
t.add_empty_column(col, dtype = np.float64)
except AttributeError as e:
for col in cols:
t[col] = np.zeros(len(t))
ra_min, ra_max, dec_min, dec_max = min_max_match(t[RA_main], t[DEC_main], \
width, DES_box = DES_box)
area = (float(ra_max)-float(ra_min))*(float(dec_max)-float(dec_min))
print area
if len(t) > 1000 and area < 2000:
ra_min, ra_max, dec_min, dec_max = min_max_match(t[RA_main], t[DEC_main], \
width, DES_box = DES_box)
print "Retreiving data"
query_ukidss = "select ra, dec, yAperMag3, yAperMag3Err, j_1AperMag3, \
j_1AperMag3Err, hAperMag3, hAperMag3Err, kAperMag3, \
kAperMag3Err FROM ukidssdr10.lassource \
WHERE ra > " + ra_min + " and ra < " + ra_max + " and \
dec > " + dec_min + " and dec < " + dec_max
RA_u, DEC_u, Y, Yerr, J, Jerr, H, Herr, K, Kerr = \
sqlutil.get(query_ukidss, db="wsdb", host=host, user=uname, password=pword)
print "Retreived data"
if len(RA_u) == 0:
for (i, col) in enumerate(cols):
t[col] = [np.float64(np.nan)]*len(t)
else:
dists, inds = match_lists.match_lists(t[RA_main], t[DEC_main], RA_u, DEC_u, width, 1)
info = [RA_u, DEC_u, Y, Yerr, J, Jerr, H, Herr, K, Kerr]
match = np.where( (inds <> len(RA_u)) )[0]
no_match = np.where( (inds == len(RA_u)) )[0]
for (i, col) in enumerate(cols):
t[col][match] = info[i][inds[match]]
t[col][no_match] = np.float64(np.nan)
if len(t) <= 1000:
n = 0
while n < len(t):
RA = t[RA_main][n]
DEC = t[DEC_main][n]
query_ukidss = "select ra, dec, yAperMag3, yAperMag3Err, j_1AperMag3, \
j_1AperMag3Err, hAperMag3, hAperMag3Err, kAperMag3, \
kAperMag3Err FROM ukidssdr10.lassource WHERE \
q3c_radial_query(ra, dec, " + str(RA) + ", " + \
str(DEC) + ", 0.1102)"
RA_u, DEC_u, Y, Yerr, J, Jerr, H, Herr, K, Kerr = \
sqlutil.get(query_ukidss, db="wsdb", host=host, user=uname, password=pword)
if len(RA_u) > 0:
dists, inds = match_lists.match_lists([RA], [DEC], RA_u, DEC_u, width, 1)
j = inds[0]
if j <> len(RA_u):
info = [RA_u[j], DEC_u[j], Y[j], Yerr[j], J[j], Jerr[j], \
H[j], Herr[j], K[j], Kerr[j]]
for (k, col) in enumerate(cols):
t[col][n] = info[k]
else:
for (k, col) in enumerate(cols):
t[col][n] = np.float64(np.nan)
else:
for (k, col) in enumerate(cols):
t[col][n] = np.float64(np.nan)
n += 1
print "Done UKIDSS"
if area > 2000 and len(t) > 1000:
t.sort(RA_main)
l1 = np.ceil(area/1001.0)
l = len(t)/l1
i = 0
while i < l1:
if (i+1)*l < len(t):
t1 = t[i*l:(i+1)*l]
else:
t1 = t[i*l:]
t1 = UKIDSS_match(t1, RA_main, DEC_main, c_graph = c_graph, width = width, \
w_units = w_units, DES_box = DES_box)
if i == 0:
t_out = t1
else:
t_out = vstack([t_out, t1])
i += 1
t = t_out
if c_graph:
width = width *3600.0
RA_med, DEC_med = check_graph(t, RA_main, DEC_main, "UKIDSS", width)
return t
if vmin is None:
vmin = 0
if vmax is None:
vmax = hh.max()
if weight_norm:
hh = hh * 1. / (hhsum + 1 * (hhsum == 0))
if skip_empty:
hh1 = hh[hhsum > 0]
else:
hh1 = hh
coll.set_array(hh1)
coll.set_cmap(cmap)
coll.set_clim(vmin, vmax)
coll.set_edgecolors(coll.cmap(coll.norm(hh1)))
coll.set_rasterized(rasterized)
tmpcoll.set_clim(vmin, vmax)
ax.add_collection(coll)
return coll
if __name__ == '__main__':
ras, decs = sqlutil.get('select radeg,dedeg from sdss_phot_qso.main',
host='cappc118')
doit(ras, decs, ramin=0, ramax=360, decmin=-90, decmax=90, nside=64)
plt.savefig('xx.png', dpi=200)
def VIKING_match(t, RA_main, DEC_main, width = 0.0004, w_units = "degrees", c_graph = True, DES_box = False):
"""
Matches to Viking Data from WSDB
Table used is viking_201207.main
DES_box returns all objects within a box of about the size of a DES tile centred
on the median RA and DEC of the table.
Iterates through a table where the RA column is RA_main and the DEC_column is
DEC_main
Puts NaNs in for no matches
c_graph plots diagnostic plot of the match - crashes if there are no matches
"""
import numpy as np
import match_lists
import sqlutil
import ConfigParser
config_file = "srpylib.cfg"
uname = config.get("wsdb", uname)
pword = config.get("wsdb", pword)
host = config.get("wsdb", host)
if w_units == "arcsecs":
width = width / 3600.0
cols = ["RA_VIKING", "DEC_VIKING", "VIKING_Y", "VIKING_J"]
try:
for col in cols:
t.add_empty_column(col, dtype = np.float64)
except AttributeError as e:
for col in cols:
t[col] = np.zeros(len(t))
n = 0
if len(t) > 1000 and len(t) < 1000000:
ra_min, ra_max, dec_min, dec_max = min_max_match(t[RA_main], t[DEC_main], \
width, DES_box = DES_box)
print "Retreiving data"
query_V = "select ra, dec, mag_y, mag_j FROM viking_201207.main WHERE ra > "\
+ ra_min + " and ra < " + ra_max + " and dec > " + \
dec_min + " and dec < " + dec_max
RA_V, DEC_V, Y_V, J_V = sqlutil.get(query_V, db = "wsdb", host = host,\
user = uname, password = pword)
if len(RA_wise) == 0:
for (i, col) in enumerate(cols):
t[col] = [np.float64(np.nan)]*len(t)
else:
info = [RA_V, DEC_V, Y_V, J_V]
print "Retreived data, matching lists"
dists, inds = match_lists.match_lists(t[RA_main], t[DEC_main], \
RA_V, DEC_V, width, 1)
print "Matched lists, updating table"
match = np.where( (inds <> len(RA_V)) )[0]
no_match = np.where( (inds == len(RA_V)) )[0]
for (i, col) in enumerate(cols):
t[col][match] = info[i][inds[match]]
t[col][no_match] = np.float64(np.nan)
print "Table updated"
elif len(t) <= 1000:
while n < len(t):
RA = t[RA_main][n]
DEC = t[DEC_main][n]
query = "select ra, dec, mag_y, mag_j FROM viking_201207.main WHERE " + \
"q3c_radial_query(ra, dec, " + str(RA) + \
", " + str(DEC) + ", 0.1002)"
RA_V, DEC_V, Y_V, J_V = sqlutil.get(query, db='wsdb',host=host, \
user = uname, password=pword)
if len(RA_V) > 0:
dists, inds = match_lists.match_lists([RA], [DEC], RA_V, DEC_V, width, 1)
j = inds[0]
info = [RA_V, DEC_V, Y_V, J_V]
for (i, col) in enumerate(cols):
t[col][n] = info[i]
else:
for (i, col) in enumerate(cols):
t[col][n] = np.float64(np.nan)
n += 1
elif len(t) > 1000000:
print "Splitting by tile"
tiles_all = t["TILENAME"]
tiles = set(tiles_all)
tiles = list(tiles)
print len(tiles), "distinct tiles"
t.sort("TILENAME")
tiles.sort()
n = 0
m = 0
rows = []
tile_rows = []
while n < len(t):
tile = tiles[m]
if t["TILENAME"][n] == tile:
rows.append(n)
n += 1
else:
tile_rows.append(rows)
rows = []
m += 1
for rows in tile_rows:
t1 = t.rows(rows)
print "Matching", t1["TILENAME"][0], "with length", len(t1)
t1 = VIKING_match(t1, RA_main, DEC_main, width, c_graph = False)
print "Returning table of length:", len(t1)
if tile_rows.index(rows) == 0:
t_out = t1
else:
t_out.append(t1)
t = t_out
if c_graph:
width = width *3600.0
RA_med, DEC_med = check_graph(t, RA_main, DEC_main, "VIKING", width, gtype = "square")
return t
query = sqlquery
if debug:
print(query)
for iline, line in enumerate(query.splitlines()):
print(iline, ':', line)
print('host:', host)
print('query', len(query))
print(sqlparse.format(query, strip_comments=True))
print('getting data...')
t1 = time.time()
result = sqlutil.get(query,
db=db,
host=host,
user=user,
password=password,
asDict=True)
querytime = time.time() - t1
if USING_atpy:
result = atpy.Table('postgres',
query=query,
user=user,
database='wsdb',
password=password,
host='cappc127')
if debug:
help(result)
def get_coords(infile=None, coord_file=None, tilename=None):
"""
# Take DES tile, find all WISE objects on that tile.
# infile = "/data/desardata/Y1A1/" + tile + "/" + tile + "_z.fits.fz"
# DATAPATH = '/data/desardata/Y1A1/'
"""
import math
logger = logging.getLogger()
outpath = os.path.dirname(coord_file)
if not os.path.exists(outpath):
try:
os.makedirs(outpath)
except Exception as error:
logger.exception(error)
print("Unexpected error:", sys.exc_info()[0])
traceback.print_exc(file=sys.stdout)
raise
logger.debug('Reading tile image to get corners: %s', infile)
with fits.open(infile) as fhlist:
hdr = fhlist[1].header
w = wcs.WCS(hdr, naxis=2)
# If this is ever used for non square inmages this needs checking
pix_corners = [[0, 0], [0, hdr["NAXIS1"]], [
hdr["NAXIS2"], 0], [hdr["NAXIS1"], hdr["NAXIS2"]]]
w_corners = w.wcs_pix2world(pix_corners, 1)
[corner_ra, corner_dec] = zip(*w_corners)
"""
corner_ra = sorted(corner_ra)
corner_dec = sorted(corner_dec)
#Take the slightly smaller square inside the not square image
ra_min = corner_ra[1]
ra_max = corner_ra[2]
dec_min = corner_dec[1]
dec_max = corner_dec[2]
"""
delta_ra = max(corner_ra) - min(corner_ra)
delta_dec = max(corner_dec) - min(corner_dec)
dec_mean = (min(corner_dec) + min(corner_dec)) / 2.0
delta_ra = delta_ra * math.cos(math.radians(dec_mean))
logger.debug('dec_mean: %s', dec_mean)
logger.debug('delta_ra, delta_dec: %s; %s', delta_ra, delta_dec)
# raw_input("Enter any key to continue: ")
# Some of these objects won't be on the DES tile so will return infs
ra_min = str(min(corner_ra))
ra_max = str(max(corner_ra))
dec_min = str(min(corner_dec))
dec_max = str(max(corner_dec))
query_wise = "select ra, dec FROM allwise.main WHERE ra > " + \
ra_min + \
" and ra < " + ra_max + " and dec > " + \
dec_min + " and dec < " + dec_max
# special case where crossing 24hrs ra > ra_max or ra < ra_min
if delta_ra > 180.0:
query_wise = "select ra, dec FROM allwise.main WHERE (ra > " + \
ra_max + \
" or ra < " + ra_min + ") and dec > " + \
dec_min + " and dec < " + dec_max
db, host, user, password, db_table = rd_config_wsdb(table='wise')
RA_wise, DEC_wise = sqlutil.get(
query_wise, db=db, host=host,
user=user, password=password
)
logger.info('%s: Number of wise sources: %s', tilename, len(RA_wise))
info = ""
for (ra, dec) in zip(RA_wise, DEC_wise):
info += (str(ra) + " " + str(dec) + "\n")
with open(coord_file, "w") as f:
logger.debug('Writing coords: %s', coord_file)
try:
f.write(info)
logger.debug('Successfully written coordinate file: %s', coord_file)
except IOError as e:
print("I/O error({0}): {1}".format(e.errno, e.strerror))
except:
print("Unexpected error:", sys.exc_info()[0])
raise
# except IOError as (errno, strerror):
# print("I/O error({0}): {1}".format(errno, strerror))
# print('Failed to write coordinate file: ' + str(e))
# logger.error('Failed to write coordinate file: ' + str(e))
# sys.exit()
# pass
# except OSError as (errno, strerror):
# print("Hello World:OS error({0}): {1}".format(errno, strerror))
# pass
# except:
# print("Unexpected error:", sys.exc_info()[0])
# # traceback.print_exc(file=sys.stdout)
# raise
return coord_file
