def plot(x, y, outpath, xlabel, ylabel, plot_style, plot_type):
if plot_type == 'mag-mag':
xlim = (10, 18)
ylim = (10, 18)
elif plot_type == 'color-mag':
xlim = (10, 18)
ylim = (-1, 1)
elif plot_type == 'color-color':
xlim = (-1, 1)
ylim = (-1, 1)
else:
raise (ValueError(
"plot_type should be one of ['mag-mag', 'color-mag', 'color-color'] "
))
isinrange = lambda a, b: (a >= b[0]) & (a <= b[1])
g = isinrange(x, xlim) & isinrange(y, ylim)
if plot_style == 'scatter':
plt.scatter(x[g], y[g])
elif plot_style == 'hexbin':
plt.hexbin(x[g], y[g])
elif plot_style == 'hist2d':
plt.hist2d(x[g], y[g], bins=100)
else:
raise (ValueError(
"plot_style should be one of ['scatter', 'hexbin', 'hist2d'] "))
plt.xlabel(xlabel)
plt.ylabel(ylabel)
ax = plt.gca()
ax.set_xlim(xlim)
ax.set_ylim(ylim)
plt.savefig(outpath, dpi=120)
plt.close()
print("created file: {}".format(outpath))
def plot(x, y, outpath, xlabel, ylabel, plot_style, plot_type):
if plot_type == 'mag-mag':
xlim = (10, 18)
ylim = (10, 18)
elif plot_type == 'color-mag':
xlim = (10, 18)
ylim = (-1, 1)
elif plot_type == 'color-color':
xlim = (-1, 1)
ylim = (-1, 1)
else:
raise(ValueError("plot_type should be one of ['mag-mag', 'color-mag', 'color-color'] "))
isinrange = lambda a,b: (a>=b[0]) & (a<=b[1])
g = isinrange(x, xlim) & isinrange(y, ylim)
if plot_style == 'scatter':
plt.scatter(x[g], y[g])
elif plot_style == 'hexbin':
plt.hexbin(x[g], y[g])
elif plot_style == 'hist2d':
plt.hist2d(x[g], y[g], bins=100)
else:
raise(ValueError("plot_style should be one of ['scatter', 'hexbin', 'hist2d'] "))
plt.xlabel(xlabel)
plt.ylabel(ylabel)
ax = plt.gca()
ax.set_xlim(xlim)
ax.set_ylim(ylim)
plt.savefig(outpath, dpi=120)
plt.close()
print("created file: {}".format(outpath))
def plot_wise(cat_path):
for catfile in find_files(cat_path, "*merged+wise.csv"):
print("\nreading catalog: {}".format(catfile))
df = pd.read_csv(catfile)
# convert to magnitudes
nbadflux = (df.flux <= 0).sum()
try:
assert nbadflux == 0
except:
print("warning: {} negative flux source(s)".format(nbadflux))
ch = catfile.split('/')[-1].split('_')[1]
mags = spz_jy_to_mags(df.flux * 1e-3, float(ch))
if ch == '1':
plt.scatter(df.W1mag, mags)
plt.xlabel('W1 [mag]')
plt.ylabel('I1 [mag]')
elif ch == '2':
plt.scatter(df.W2mag, mags)
plt.xlabel('W2 [mag]')
plt.ylabel('I2 [mag]')
ax = plt.gca()
xlim, ylim = ax.get_xlim(), ax.get_ylim()
plt.plot([-5, ylim[1] * 2], [-5, ylim[1] * 2], 'r-')
ax.set_xlim(xlim)
ax.set_ylim(ylim)
reg = catfile.split('/')[-1].split('_')[0]
name = '{}_{}_IRAC_vs_WISE.png'.format(reg, ch)
outpath = '/'.join(catfile.split('/')[:-1] + [name])
plt.savefig(outpath, dpi=120)
plt.close()
def plot_wise(cat_path):
for catfile in find_files(cat_path, "*merged+wise.csv"):
print("\nreading catalog: {}".format(catfile))
df = pd.read_csv(catfile)
# convert to magnitudes
nbadflux = (df.flux <= 0).sum()
try:
assert nbadflux == 0
except:
print("warning: {} negative flux source(s)".format(nbadflux))
ch = catfile.split('/')[-1].split('_')[1]
mags = spz_jy_to_mags(df.flux*1e-3, float(ch))
if ch == '1':
plt.scatter(df.W1mag, mags)
plt.xlabel('W1 [mag]')
plt.ylabel('I1 [mag]')
elif ch == '2':
plt.scatter(df.W2mag, mags)
plt.xlabel('W2 [mag]')
plt.ylabel('I2 [mag]')
ax = plt.gca()
xlim, ylim = ax.get_xlim(), ax.get_ylim()
plt.plot([-5, ylim[1]*2], [-5, ylim[1]*2], 'r-')
ax.set_xlim(xlim) ; ax.set_ylim(ylim)
reg = catfile.split('/')[-1].split('_')[0]
name = '{}_{}_IRAC_vs_WISE.png'.format(reg, ch)
outpath = '/'.join(catfile.split('/')[:-1]+[name])
plt.savefig(outpath, dpi=120)
plt.close()
def maybe_get_ax(*args, **kwargs):
"""
It used to be that the first argument of prettyplotlib had to be the 'ax'
object, but that's not the case anymore.
@param args:
@type args:
@param kwargs:
@type kwargs:
@return:
@rtype:
"""
if 'ax' in kwargs:
ax = kwargs.pop('ax')
elif len(args) == 0:
fig = plt.gcf()
ax = plt.gca()
elif isinstance(args[0], mpl.axes.Axes):
ax = args[0]
args = args[1:]
else:
ax = plt.gca()
return ax, args, dict(kwargs)
def run_xsc_phot(bcdphot_out_path, mosaic_path):
replaced = {}
for cat in find_files(bcdphot_out_path, "*_combined_hdr_catalog.txt"):
print("\n======================================================")
print("\nadjusting photometry in: {}".format(cat.split('/')[-1]))
print("------------------------------------------------------")
outpath = cat.replace('combined_hdr_catalog.txt','2mass_xsc.tbl')
# retrieve 2mass data if file doesn't already exist (from previous run)
if not os.path.isfile(outpath):
# get url and retrieve data
url = query_2mass_xsc_polygon(*get_region_corners(cat))
print("\ndownloading 2MASS photometry from: {}".format(url))
text = urllib2.urlopen(url).read()
# write to disk
with open(outpath, 'w') as f:
f.write(text)
print("\ncreated file: {}".format(outpath))
# read back in as recarray
print("\nreading: {}".format(outpath))
names = open(outpath).read().split('\n')[76].split('|')[1:-1]
da = np.recfromtxt(outpath, skip_header=80, names=names)
# write input file for xsc_phot.pro
infile_outpath = '/'.join(cat.split('/')[:-1])+'/xsc.txt'
with open(infile_outpath,'w') as w:
for i in range(da.shape[0]):
w.write("{} {} {} {}\n".format(da.designation[i], da.ra[i], da.dec[i], da.r_ext[i]))
print("\ncreated input file for xsc_phot.pro: {}".format(infile_outpath))
# locate the FITS mosaic file for xsc_phot.pro to do photometry on
reg, ch = cat.split('/')[-1].split('_')[:2]
mosaicfile = filter(lambda x: 'dirbe{}/ch{}/long/full/Combine'\
.format(reg,ch) in x, find_files(mosaic_path, '*mosaic.fits'))[0]
print("\nfound mosaic file: {}".format(mosaicfile))
# spawn IDL subprocess running xsc_phot.pro and catch stdout in file
outpath = infile_outpath.replace('xsc.txt', 'xsc_phot_out.txt')
if not os.path.isfile(outpath):
outfile = open(outpath,'w')
print("\nspawning xsc_phot.pro IDL subprocess")
cmd = "xsc_phot,'"+mosaicfile+"','"+infile_outpath+"','long'"
rc = subprocess.call(['/usr/local/itt/idl71/bin/idl','-quiet','-e',cmd],
stderr = subprocess.PIPE, stdout = outfile)
outfile.close()
# read in output to recarray
print("\nreading: {}".format(outpath))
phot = np.recfromtxt(outpath, names=['id','flux','unc','sky','skyunc'])
# make sure rows are aligned
assert (da.designation == phot.id).all()
# ignore xsc sources we got a NaN or negative flux for
bad = np.isnan(phot.flux) | (phot.flux < 0)
print("\naper.pro returned NaN or negative flux for {} sources".format(bad.sum()))
if bad.sum() > 0:
for i in phot[bad].id:
print(i)
outpath = cat.replace('combined_hdr_catalog.txt','xsc_nan_phot.csv')
with open(outpath,'w') as f:
w = csv.writer(f)
w.writerow(da.dtype.names)
w.writerows(da[bad].tolist())
print('\ncreated file: {}'.format(outpath))
phot = phot[~bad]
da = da[~bad]
# read in pipeline catalog
print("\nreading: {}".format(cat))
names = open(cat).readline().split()[1:]
c = np.recfromtxt(cat, names=names)
# loop through xsc sources and find matches in pipeline catalog
print("\nfinding records associated with XSC sources in pipeline catalog")
c_flux_total = []
n_in_aper = []
c_idx = []
coords = radec_to_coords(c.ra, c.dec)
kdt = KDT(coords)
for i in range(phot.size):
radius = da.r_ext[i]/3600.
# idx1, idx2, ds = spherematch(da.ra[i], da.dec[i],
# c.ra, c.dec, tolerance=radius)
idx, ds = spherematch2(da.ra[i], da.dec[i], c.ra, c.dec,
kdt, tolerance=radius, k=500)
# c_flux_total.append(c.flux[idx2].sum())
# n_in_aper.append(c.flux[idx2].size)
# c_idx.append(idx2.tolist())
c_flux_total.append(c.flux[idx].sum())
n_in_aper.append(ds.size)
c_idx.append(idx.tolist())
print("\nhistogram of source counts in r_ext aperture")
for i in [(i,n_in_aper.count(i)) for i in set(n_in_aper)]:
print i
# create new version of catalog file with xsc-associated entries replaced
c_idx = np.array(flatten(c_idx))
print("\nremoving {}, adding {}".format(c_idx.size, phot.size))
replaced[cat] = {'old':c_idx.size, 'new':phot.size}
replaced[cat]['hist'] = [(i,n_in_aper.count(i)) for i in set(n_in_aper)]
c = np.delete(c, c_idx)
newrows = np.rec.array([(-i, da.ra[i], da.dec[i],
phot.flux[i], phot.unc[i], 1) for i in \
range(phot.size)], dtype=c.dtype)
newcat = np.hstack((c, newrows))
# write new version of catalog to disk
fmt = ['%i']+['%0.8f']*2+['%.4e']*2+['%i']
outpath = cat.replace('catalog.txt', 'catalog_xsc_cor.txt')
np.savetxt(outpath, newcat, fmt = fmt, header = ' '.join(names))
print('\ncreated file: {}'.format(outpath))
# make plot of total old vs. new flux
plt.scatter(c_flux_total, phot.flux)
ylim = plt.gca().get_ylim()
plt.xlim(*ylim)
max_y = ylim[1]
plt.plot(ylim, ylim, 'r-')
plt.xlabel('old flux [mJy]')
plt.ylabel('new flux [mJy]')
name = ' '.join(cat.split('/')[-1].split('_')[:2])
plt.title(name)
outpath = cat.replace('combined_hdr_catalog.txt','xsc_new_vs_old_phot.png')
plt.savefig(outpath, dpi=200)
plt.close()
print('\ncreated file: {}'.format(outpath))
outfile = 'xsc_replaced.json'
json.dump(replaced, open(outfile,'w'))
print("\ncreated file: {}".format(outfile))
print("\nremoved / added")
for k,v in replaced.iteritems():
print k.split('/')[-1], v['old'], v['new']
m = np.mean([i['old']/float(i['new']) for i in replaced.values()])
print("average ratio: {}".format(m))
print("\nK mag and r_ext of sources with NaN photometry:")
for i in find_files(bcdphot_out_path, "*xsc_nan_phot.csv"):
reg = i.split('/')[-1]
rec = np.recfromcsv(i)
bad_id = rec.designation.tolist()
bad_k = rec.k_m_k20fe.tolist()
bad_r_ext = rec.r_ext.tolist()
print reg
print ("\tid\t\t\tKmag\tr_ext")
if type(bad_id) is list:
seq = sorted(zip(bad_id, bad_k, bad_r_ext), key=lambda x: x[0])
for j,k,l in seq:
print("\t{}\t{}\t{}".format(j,k,l))
else:
print("\t{}\t{}\t{}".format(bad_id, bad_k, bad_r_ext))
import numpy
from prettyplotlib import plt
if __name__=='__main__':
xs = numpy.arange(-5.0,5.0,0.001)
def f(x):
return 1.0*numpy.exp(-x**2/2)
def fdash(x):
return -x*numpy.exp(-x**2/2)
tuning = f(xs)
info = fdash(xs)**2/f(xs)
plt.plot(xs,tuning,label='Tuning function')
plt.plot(xs,info,label='Fisher Information')
plt.gca().set_xlabel('x')
plt.legend()
plt.savefig('../figures/figure_3_5.eps')
def run_xsc_phot(bcdphot_out_path, mosaic_path):
replaced = {}
for cat in find_files(bcdphot_out_path, "*_combined_hdr_catalog.txt"):
print("\n======================================================")
print("\nadjusting photometry in: {}".format(cat.split('/')[-1]))
print("------------------------------------------------------")
outpath = cat.replace('combined_hdr_catalog.txt', '2mass_xsc.tbl')
# retrieve 2mass data if file doesn't already exist (from previous run)
if not os.path.isfile(outpath):
# get url and retrieve data
url = query_2mass_xsc_polygon(*get_region_corners(cat))
print("\ndownloading 2MASS photometry from: {}".format(url))
text = urllib2.urlopen(url).read()
# write to disk
with open(outpath, 'w') as f:
f.write(text)
print("\ncreated file: {}".format(outpath))
# read back in as recarray
print("\nreading: {}".format(outpath))
names = open(outpath).read().split('\n')[76].split('|')[1:-1]
da = np.recfromtxt(outpath, skip_header=80, names=names)
# write input file for xsc_phot.pro
infile_outpath = '/'.join(cat.split('/')[:-1]) + '/xsc.txt'
with open(infile_outpath, 'w') as w:
for i in range(da.shape[0]):
w.write("{} {} {} {}\n".format(da.designation[i], da.ra[i],
da.dec[i], da.r_ext[i]))
print(
"\ncreated input file for xsc_phot.pro: {}".format(infile_outpath))
# locate the FITS mosaic file for xsc_phot.pro to do photometry on
reg, ch = cat.split('/')[-1].split('_')[:2]
mosaicfile = filter(lambda x: 'dirbe{}/ch{}/long/full/Combine'\
.format(reg,ch) in x, find_files(mosaic_path, '*mosaic.fits'))[0]
print("\nfound mosaic file: {}".format(mosaicfile))
# spawn IDL subprocess running xsc_phot.pro and catch stdout in file
outpath = infile_outpath.replace('xsc.txt', 'xsc_phot_out.txt')
if not os.path.isfile(outpath):
outfile = open(outpath, 'w')
print("\nspawning xsc_phot.pro IDL subprocess")
cmd = "xsc_phot,'" + mosaicfile + "','" + infile_outpath + "','long'"
rc = subprocess.call(
['/usr/local/itt/idl71/bin/idl', '-quiet', '-e', cmd],
stderr=subprocess.PIPE,
stdout=outfile)
outfile.close()
# read in output to recarray
print("\nreading: {}".format(outpath))
phot = np.recfromtxt(outpath,
names=['id', 'flux', 'unc', 'sky', 'skyunc'])
# make sure rows are aligned
assert (da.designation == phot.id).all()
# ignore xsc sources we got a NaN or negative flux for
bad = np.isnan(phot.flux) | (phot.flux < 0)
print("\naper.pro returned NaN or negative flux for {} sources".format(
bad.sum()))
if bad.sum() > 0:
for i in phot[bad].id:
print(i)
outpath = cat.replace('combined_hdr_catalog.txt',
'xsc_nan_phot.csv')
with open(outpath, 'w') as f:
w = csv.writer(f)
w.writerow(da.dtype.names)
w.writerows(da[bad].tolist())
print('\ncreated file: {}'.format(outpath))
phot = phot[~bad]
da = da[~bad]
# read in pipeline catalog
print("\nreading: {}".format(cat))
names = open(cat).readline().split()[1:]
c = np.recfromtxt(cat, names=names)
# loop through xsc sources and find matches in pipeline catalog
print(
"\nfinding records associated with XSC sources in pipeline catalog"
)
c_flux_total = []
n_in_aper = []
c_idx = []
coords = radec_to_coords(c.ra, c.dec)
kdt = KDT(coords)
for i in range(phot.size):
radius = da.r_ext[i] / 3600.
# idx1, idx2, ds = spherematch(da.ra[i], da.dec[i],
# c.ra, c.dec, tolerance=radius)
idx, ds = spherematch2(da.ra[i],
da.dec[i],
c.ra,
c.dec,
kdt,
tolerance=radius,
k=500)
# c_flux_total.append(c.flux[idx2].sum())
# n_in_aper.append(c.flux[idx2].size)
# c_idx.append(idx2.tolist())
c_flux_total.append(c.flux[idx].sum())
n_in_aper.append(ds.size)
c_idx.append(idx.tolist())
print("\nhistogram of source counts in r_ext aperture")
for i in [(i, n_in_aper.count(i)) for i in set(n_in_aper)]:
print i
# create new version of catalog file with xsc-associated entries replaced
c_idx = np.array(flatten(c_idx))
print("\nremoving {}, adding {}".format(c_idx.size, phot.size))
replaced[cat] = {'old': c_idx.size, 'new': phot.size}
replaced[cat]['hist'] = [(i, n_in_aper.count(i))
for i in set(n_in_aper)]
c = np.delete(c, c_idx)
newrows = np.rec.array([(-i, da.ra[i], da.dec[i],
phot.flux[i], phot.unc[i], 1) for i in \
range(phot.size)], dtype=c.dtype)
newcat = np.hstack((c, newrows))
# write new version of catalog to disk
fmt = ['%i'] + ['%0.8f'] * 2 + ['%.4e'] * 2 + ['%i']
outpath = cat.replace('catalog.txt', 'catalog_xsc_cor.txt')
np.savetxt(outpath, newcat, fmt=fmt, header=' '.join(names))
print('\ncreated file: {}'.format(outpath))
# make plot of total old vs. new flux
plt.scatter(c_flux_total, phot.flux)
ylim = plt.gca().get_ylim()
plt.xlim(*ylim)
max_y = ylim[1]
plt.plot(ylim, ylim, 'r-')
plt.xlabel('old flux [mJy]')
plt.ylabel('new flux [mJy]')
name = ' '.join(cat.split('/')[-1].split('_')[:2])
plt.title(name)
outpath = cat.replace('combined_hdr_catalog.txt',
'xsc_new_vs_old_phot.png')
plt.savefig(outpath, dpi=200)
plt.close()
print('\ncreated file: {}'.format(outpath))
outfile = 'xsc_replaced.json'
json.dump(replaced, open(outfile, 'w'))
print("\ncreated file: {}".format(outfile))
print("\nremoved / added")
for k, v in replaced.iteritems():
print k.split('/')[-1], v['old'], v['new']
m = np.mean([i['old'] / float(i['new']) for i in replaced.values()])
print("average ratio: {}".format(m))
print("\nK mag and r_ext of sources with NaN photometry:")
for i in find_files(bcdphot_out_path, "*xsc_nan_phot.csv"):
reg = i.split('/')[-1]
rec = np.recfromcsv(i)
bad_id = rec.designation.tolist()
bad_k = rec.k_m_k20fe.tolist()
bad_r_ext = rec.r_ext.tolist()
print reg
print("\tid\t\t\tKmag\tr_ext")
if type(bad_id) is list:
seq = sorted(zip(bad_id, bad_k, bad_r_ext), key=lambda x: x[0])
for j, k, l in seq:
print("\t{}\t{}\t{}".format(j, k, l))
else:
print("\t{}\t{}\t{}".format(bad_id, bad_k, bad_r_ext))