def plot_wise(cat_path):
for catfile in find_files(cat_path, "*merged+wise.csv"):
print("\nreading catalog: {}".format(catfile))
df = pd.read_csv(catfile)
# convert to magnitudes
nbadflux = (df.flux <= 0).sum()
try:
assert nbadflux == 0
except:
print("warning: {} negative flux source(s)".format(nbadflux))
ch = catfile.split('/')[-1].split('_')[1]
mags = spz_jy_to_mags(df.flux * 1e-3, float(ch))
if ch == '1':
plt.scatter(df.W1mag, mags)
plt.xlabel('W1 [mag]')
plt.ylabel('I1 [mag]')
elif ch == '2':
plt.scatter(df.W2mag, mags)
plt.xlabel('W2 [mag]')
plt.ylabel('I2 [mag]')
ax = plt.gca()
xlim, ylim = ax.get_xlim(), ax.get_ylim()
plt.plot([-5, ylim[1] * 2], [-5, ylim[1] * 2], 'r-')
ax.set_xlim(xlim)
ax.set_ylim(ylim)
reg = catfile.split('/')[-1].split('_')[0]
name = '{}_{}_IRAC_vs_WISE.png'.format(reg, ch)
outpath = '/'.join(catfile.split('/')[:-1] + [name])
plt.savefig(outpath, dpi=120)
plt.close()
def plot(x, y, outpath, xlabel, ylabel, plot_style, plot_type):
if plot_type == 'mag-mag':
xlim = (10, 18)
ylim = (10, 18)
elif plot_type == 'color-mag':
xlim = (10, 18)
ylim = (-1, 1)
elif plot_type == 'color-color':
xlim = (-1, 1)
ylim = (-1, 1)
else:
raise (ValueError(
"plot_type should be one of ['mag-mag', 'color-mag', 'color-color'] "
))
isinrange = lambda a, b: (a >= b[0]) & (a <= b[1])
g = isinrange(x, xlim) & isinrange(y, ylim)
if plot_style == 'scatter':
plt.scatter(x[g], y[g])
elif plot_style == 'hexbin':
plt.hexbin(x[g], y[g])
elif plot_style == 'hist2d':
plt.hist2d(x[g], y[g], bins=100)
else:
raise (ValueError(
"plot_style should be one of ['scatter', 'hexbin', 'hist2d'] "))
plt.xlabel(xlabel)
plt.ylabel(ylabel)
ax = plt.gca()
ax.set_xlim(xlim)
ax.set_ylim(ylim)
plt.savefig(outpath, dpi=120)
plt.close()
print("created file: {}".format(outpath))
def plot(x, y, outpath, xlabel, ylabel, plot_style, plot_type):
if plot_type == 'mag-mag':
xlim = (10, 18)
ylim = (10, 18)
elif plot_type == 'color-mag':
xlim = (10, 18)
ylim = (-1, 1)
elif plot_type == 'color-color':
xlim = (-1, 1)
ylim = (-1, 1)
else:
raise(ValueError("plot_type should be one of ['mag-mag', 'color-mag', 'color-color'] "))
isinrange = lambda a,b: (a>=b[0]) & (a<=b[1])
g = isinrange(x, xlim) & isinrange(y, ylim)
if plot_style == 'scatter':
plt.scatter(x[g], y[g])
elif plot_style == 'hexbin':
plt.hexbin(x[g], y[g])
elif plot_style == 'hist2d':
plt.hist2d(x[g], y[g], bins=100)
else:
raise(ValueError("plot_style should be one of ['scatter', 'hexbin', 'hist2d'] "))
plt.xlabel(xlabel)
plt.ylabel(ylabel)
ax = plt.gca()
ax.set_xlim(xlim)
ax.set_ylim(ylim)
plt.savefig(outpath, dpi=120)
plt.close()
print("created file: {}".format(outpath))
def plot_wise(cat_path):
for catfile in find_files(cat_path, "*merged+wise.csv"):
print("\nreading catalog: {}".format(catfile))
df = pd.read_csv(catfile)
# convert to magnitudes
nbadflux = (df.flux <= 0).sum()
try:
assert nbadflux == 0
except:
print("warning: {} negative flux source(s)".format(nbadflux))
ch = catfile.split('/')[-1].split('_')[1]
mags = spz_jy_to_mags(df.flux*1e-3, float(ch))
if ch == '1':
plt.scatter(df.W1mag, mags)
plt.xlabel('W1 [mag]')
plt.ylabel('I1 [mag]')
elif ch == '2':
plt.scatter(df.W2mag, mags)
plt.xlabel('W2 [mag]')
plt.ylabel('I2 [mag]')
ax = plt.gca()
xlim, ylim = ax.get_xlim(), ax.get_ylim()
plt.plot([-5, ylim[1]*2], [-5, ylim[1]*2], 'r-')
ax.set_xlim(xlim) ; ax.set_ylim(ylim)
reg = catfile.split('/')[-1].split('_')[0]
name = '{}_{}_IRAC_vs_WISE.png'.format(reg, ch)
outpath = '/'.join(catfile.split('/')[:-1]+[name])
plt.savefig(outpath, dpi=120)
plt.close()
def scipy_fcluster_merge_strategy(assignments):
# Z-matrix helpers
def groups(assignments):
cluster_map = {}
for idx, gid in enumerate(assignments):
lst = cluster_map.get(gid, [])
lst.append(idx)
cluster_map[gid] = lst
return tuple(sorted(map(tuple, cluster_map.values()), key=len, reverse=True))
def zmatrix(assignments_samples):
n = len(assignments_samples[0])
# should make sparse matrix
zmat = np.zeros((n, n), dtype=np.float32)
for assignments in assignments_samples:
clusters = groups(assignments)
for cluster in clusters:
for i, j in it.product(cluster, repeat=2):
zmat[i, j] += 1
zmat /= float(len(assignments_samples))
return zmat
def reorder_zmat(zmat, order):
zmat = zmat[order]
zmat = zmat[:,order]
return zmat
def linkage(zmat):
assert zmat.shape[0] == zmat.shape[1]
zmat0 = np.array(zmat[np.triu_indices(zmat.shape[0], k=1)])
zmat0 = 1. - zmat0
return scipy.cluster.hierarchy.linkage(zmat0)
def ordering(l):
return np.array(scipy.cluster.hierarchy.leaves_list(l))
zmat = zmatrix(assignments)
li = linkage(zmat)
# diagnostic: draw z-matrix
indices = ordering(li)
plt.imshow(reorder_zmat(zmat, indices),
cmap=plt.cm.binary, interpolation='nearest')
plt.savefig("zmat-before.pdf")
plt.close()
fassignment = scipy.cluster.hierarchy.fcluster(li, 0.001)
clustering = groups(fassignment)
sorted_ordering = list(it.chain.from_iterable(clustering))
# draw post fcluster() ordered z-matrix
plt.imshow(reorder_zmat(zmat, sorted_ordering),
cmap=plt.cm.binary, interpolation='nearest')
plt.savefig("zmat-after.pdf")
plt.close()
return fassignment
def plot_sdss(cat_path):
for catfile in find_files(cat_path, "*merged+sdss.txt"):
# for now ignore the channel 2 files
if catfile.split('/')[-1].split('_')[1] != '1':
continue
print("\nreading catalog: {}".format(catfile))
df = pd.read_table(catfile, sep=' ')
# get rid of negative flux sources, if any
df = df[df.flux > 0]
# convert to magnitudes
mags = spz_jy_to_mags(df.flux * 1e-3, 1)
# print counts per magnitude bin
for i in range(10, 15):
sc = ((df.cl == 3) & (mags > i) & (mags < i + 1)).sum()
xc = ((df.xsc == 1) & (mags > i) & (mags < i + 1)).sum()
msg = "{}th to {}th mag: {} SDSS galaxy sources, {} 2MASS XSC sources"
print(msg.format(i, i + 1, sc, xc))
# print number of sources agreed upon
agree = ((df.xsc == 1) & (df.cl == 3)).sum()
disagree = ((df.xsc == 1) & (df.cl == 6)).sum()
na = ((df.xsc == 1) & (df.cl == 0)).sum()
msg = "{} 2MASS XSC sources classified as galaxies by SDSS"
print(msg.format(agree))
msg = "{} 2MASS XSC sources classified as stars by SDSS"
print(msg.format(disagree))
msg = "{} 2MASS XSC sources not matched to SDSS"
print(msg.format(na))
# plot normed histograms of 2MASS XSC and SDSS galaxy magnitudes
xsc_gals = (mags > 10) & (mags < 15) & (df.xsc == 1)
sdss_gals = (mags > 10) & (mags < 15) & (df.cl == 3)
# mags[xsc_gals].hist(label='2MASS XSC', normed=True)
# mags[sdss_gals].hist(label='SDSS galaxies', normed=True)
plt.hist([mags[xsc_gals].values, mags[sdss_gals].values],
bins=5,
label=['2MASS', 'SDSS'])
plt.xlabel('IRAC1 [mag]')
plt.ylabel('Number Count')
reg = catfile.split('/')[-1].split('_')[0]
plt.title('{} Extended Sources / Galaxies'.format(reg))
plt.legend(loc=2)
name = '{}_2mass_xsc_vs_sdss_hist.png'.format(reg)
outpath = '/'.join(catfile.split('/')[:-1] + [name])
plt.savefig(outpath, dpi=100)
plt.close()
print("created file: {}".format(outpath))
def plot_sdss(cat_path):
for catfile in find_files(cat_path, "*merged+sdss.txt"):
# for now ignore the channel 2 files
if catfile.split('/')[-1].split('_')[1] != '1':
continue
print("\nreading catalog: {}".format(catfile))
df = pd.read_table(catfile, sep=' ')
# get rid of negative flux sources, if any
df = df[df.flux > 0]
# convert to magnitudes
mags = spz_jy_to_mags(df.flux*1e-3, 1)
# print counts per magnitude bin
for i in range(10,15):
sc = ((df.cl == 3) & (mags > i) & (mags < i+1)).sum()
xc = ((df.xsc == 1) & (mags > i) & (mags < i+1)).sum()
msg = "{}th to {}th mag: {} SDSS galaxy sources, {} 2MASS XSC sources"
print(msg.format(i, i+1, sc, xc))
# print number of sources agreed upon
agree = ((df.xsc == 1) & (df.cl == 3)).sum()
disagree = ((df.xsc == 1) & (df.cl == 6)).sum()
na = ((df.xsc == 1) & (df.cl == 0)).sum()
msg = "{} 2MASS XSC sources classified as galaxies by SDSS"
print(msg.format(agree))
msg = "{} 2MASS XSC sources classified as stars by SDSS"
print(msg.format(disagree))
msg = "{} 2MASS XSC sources not matched to SDSS"
print(msg.format(na))
# plot normed histograms of 2MASS XSC and SDSS galaxy magnitudes
xsc_gals = (mags > 10) & (mags < 15) & (df.xsc == 1)
sdss_gals = (mags > 10) & (mags < 15) & (df.cl == 3)
# mags[xsc_gals].hist(label='2MASS XSC', normed=True)
# mags[sdss_gals].hist(label='SDSS galaxies', normed=True)
plt.hist([mags[xsc_gals].values, mags[sdss_gals].values],
bins=5, label=['2MASS', 'SDSS'])
plt.xlabel('IRAC1 [mag]')
plt.ylabel('Number Count')
reg = catfile.split('/')[-1].split('_')[0]
plt.title('{} Extended Sources / Galaxies'.format(reg))
plt.legend(loc=2)
name = '{}_2mass_xsc_vs_sdss_hist.png'.format(reg)
outpath = '/'.join(catfile.split('/')[:-1]+[name])
plt.savefig(outpath, dpi=100)
plt.close()
print("created file: {}".format(outpath))
def plot_signal(signal, shape=None, name=None):
"""
Plot a numeric signal as an image.
"""
s = np.copy(signal)
if s.max() != 0.:
s *= 255.0/s.max()
s = s.astype(int)
if shape:
s = s.reshape(shape)
plt.figure()
plt.imshow(s, cmap=plt.cm.gray, interpolation='nearest')
if name:
plt.savefig(name)
plt.close()
def layerActivations(model, data, labels):
print('Visualizing activations with tSNE...')
if not os.path.exists(cc.cfg['plots']['layer_activations_dir']):
os.makedirs(cc.cfg['plots']['layer_activations_dir'])
numLabels = cc.cfg['plots']['layer_activations_label_cap']
data = data[:cc.cfg['plots']['layer_activations_points_cap']]
labels = labels[:numLabels,:cc.cfg['plots']['layer_activations_points_cap']]
subplotCols = numLabels
subplotRows = len(model.layers)-1
subplotIdx = 1
plt.figure(figsize=(5*subplotCols,5*subplotRows))
for i in range(1,len(model.layers)):
print('Running tSNE for layer {}/{}'.format(i+1,len(model.layers)))
func = K.function([model.layers[0].input], [model.layers[i].output])
out = func([data])[0]
tsneModel = TSNE(n_components = 2, random_state = 0)
tsneOut = tsneModel.fit_transform(out).T
# labeledTsneOut = np.hstack((tsneOut, labels[0].reshape(-1,1)))
for j in range(numLabels):
plt.subplot(subplotRows, subplotCols, subplotIdx)
plt.title('{} / {}'.format(model.layers[i].name,cc.exp['params']['data']['labels'][j]))
plt.scatter(tsneOut[0],tsneOut[1],c=labels[j],cmap = 'plasma')
subplotIdx += 1
# tsneDF = pd.DataFrame(labeledTsneOut, columns = ('a', 'b', 'c'))
# plot = tsneDF.plot.scatter(x = 'a', y = 'b', c = 'c', cmap = 'plasma')
plt.tight_layout()
plt.savefig('{}/activations.png'.format(cc.cfg['plots']['layer_activations_dir']))
plt.close()
print('...done')
def histograms(modelLogger):
if not cc.cfg['plots']['histograms']:
return
if not os.path.exists(cc.cfg['plots']['histograms_dir']):
os.makedirs(cc.cfg['plots']['histograms_dir'])
cntEpochs = len(modelLogger.epochLogs)
cntLayers = len(modelLogger.epochLogs[-1]['weights'])
logVals = [
{'name':'weights','color':'blue'},
{'name':'updates','color':'red'},
{'name':'ratios','color':'green'}
]
subplotRows = len(logVals)
subplotCols = cntLayers
for x in range(cntEpochs):
subplotIdx = 1
plt.figure(figsize=(5*subplotCols,5*subplotRows))
plt.suptitle('Histograms per layer, epoch {}/{}'.format(x,cntEpochs-1), fontsize=14)
for logVal in logVals:
for i,layer in enumerate(modelLogger.epochLogs[x][logVal['name']]):
histmin = layer.min()
histmax = layer.max()
plt.subplot(subplotRows, subplotCols, subplotIdx)
plt.title('{}, {}'.format(modelLogger.model.layers[modelLogger.loggedLayers[i]].name,logVal['name']))
plt.hist(layer, range = (histmin, histmax), bins = 30, color = logVal['color'])
subplotIdx+=1
plt.tight_layout()
plt.subplots_adjust(top=0.9)
plt.savefig('{}/hist_{e:03d}.png'.format(cc.cfg['plots']['histograms_dir'], e = x))
plt.close()
env.reset(np.array([0.0]))
code.reset()
[mp,varp,spsp,sp,msep,parts,ws] = pf.particle_filter(code, env, timewindow=timewindow,
dt=dt, nparticles=nparticles,
mode='v', testf=(lambda x:x))
if gaussian:
print "MSE of gaussian filter %f"% mseg
print "MSE of particle filter %f"% msep
if plotting:
#matplotlib.rcParams['font.size']=10
plt.close()
plt.figure()
ax1 = plt.gcf().add_subplot(2,1,1)
times = np.arange(0.0,dt*timewindow,dt)
if gaussian:
ax1.plot(times,sg,'r',label='Signal')
if sum(sum(spsg)) !=0:
(ts,neurs) = np.where(spsg == 1)
spiketimes = times[ts]
thetas = [code.neurons[i].theta for i in neurs]
else:
spiketimes = []
thetas = []
ax1.plot(spiketimes,thetas,'yo',label='Spike times')
ax1.plot(times,mg,'b',label='Mean prediction')
[78, 376],
[79, 164],
[80, 194],
[81, 342],
[82, 314],
[83, 68],
[84, 54],
[85, 131],
[86, 52],
[87, 220],
[88, 145],
[89, 63],
[90, 608],
[91, 68],
[92, 155],
[93, 49],
[94, 108],
[95, 59],
[96, 67],
[97, 58],
[98, 42],
[99, 210],
[100, 174]])
plt.figure(figsize=(5,5))
plt.scatter(histdata.T[0],histdata.T[1])
plt.savefig('hist.png')
plt.close()
def run_xsc_phot(bcdphot_out_path, mosaic_path):
replaced = {}
for cat in find_files(bcdphot_out_path, "*_combined_hdr_catalog.txt"):
print("\n======================================================")
print("\nadjusting photometry in: {}".format(cat.split('/')[-1]))
print("------------------------------------------------------")
outpath = cat.replace('combined_hdr_catalog.txt','2mass_xsc.tbl')
# retrieve 2mass data if file doesn't already exist (from previous run)
if not os.path.isfile(outpath):
# get url and retrieve data
url = query_2mass_xsc_polygon(*get_region_corners(cat))
print("\ndownloading 2MASS photometry from: {}".format(url))
text = urllib2.urlopen(url).read()
# write to disk
with open(outpath, 'w') as f:
f.write(text)
print("\ncreated file: {}".format(outpath))
# read back in as recarray
print("\nreading: {}".format(outpath))
names = open(outpath).read().split('\n')[76].split('|')[1:-1]
da = np.recfromtxt(outpath, skip_header=80, names=names)
# write input file for xsc_phot.pro
infile_outpath = '/'.join(cat.split('/')[:-1])+'/xsc.txt'
with open(infile_outpath,'w') as w:
for i in range(da.shape[0]):
w.write("{} {} {} {}\n".format(da.designation[i], da.ra[i], da.dec[i], da.r_ext[i]))
print("\ncreated input file for xsc_phot.pro: {}".format(infile_outpath))
# locate the FITS mosaic file for xsc_phot.pro to do photometry on
reg, ch = cat.split('/')[-1].split('_')[:2]
mosaicfile = filter(lambda x: 'dirbe{}/ch{}/long/full/Combine'\
.format(reg,ch) in x, find_files(mosaic_path, '*mosaic.fits'))[0]
print("\nfound mosaic file: {}".format(mosaicfile))
# spawn IDL subprocess running xsc_phot.pro and catch stdout in file
outpath = infile_outpath.replace('xsc.txt', 'xsc_phot_out.txt')
if not os.path.isfile(outpath):
outfile = open(outpath,'w')
print("\nspawning xsc_phot.pro IDL subprocess")
cmd = "xsc_phot,'"+mosaicfile+"','"+infile_outpath+"','long'"
rc = subprocess.call(['/usr/local/itt/idl71/bin/idl','-quiet','-e',cmd],
stderr = subprocess.PIPE, stdout = outfile)
outfile.close()
# read in output to recarray
print("\nreading: {}".format(outpath))
phot = np.recfromtxt(outpath, names=['id','flux','unc','sky','skyunc'])
# make sure rows are aligned
assert (da.designation == phot.id).all()
# ignore xsc sources we got a NaN or negative flux for
bad = np.isnan(phot.flux) | (phot.flux < 0)
print("\naper.pro returned NaN or negative flux for {} sources".format(bad.sum()))
if bad.sum() > 0:
for i in phot[bad].id:
print(i)
outpath = cat.replace('combined_hdr_catalog.txt','xsc_nan_phot.csv')
with open(outpath,'w') as f:
w = csv.writer(f)
w.writerow(da.dtype.names)
w.writerows(da[bad].tolist())
print('\ncreated file: {}'.format(outpath))
phot = phot[~bad]
da = da[~bad]
# read in pipeline catalog
print("\nreading: {}".format(cat))
names = open(cat).readline().split()[1:]
c = np.recfromtxt(cat, names=names)
# loop through xsc sources and find matches in pipeline catalog
print("\nfinding records associated with XSC sources in pipeline catalog")
c_flux_total = []
n_in_aper = []
c_idx = []
coords = radec_to_coords(c.ra, c.dec)
kdt = KDT(coords)
for i in range(phot.size):
radius = da.r_ext[i]/3600.
# idx1, idx2, ds = spherematch(da.ra[i], da.dec[i],
# c.ra, c.dec, tolerance=radius)
idx, ds = spherematch2(da.ra[i], da.dec[i], c.ra, c.dec,
kdt, tolerance=radius, k=500)
# c_flux_total.append(c.flux[idx2].sum())
# n_in_aper.append(c.flux[idx2].size)
# c_idx.append(idx2.tolist())
c_flux_total.append(c.flux[idx].sum())
n_in_aper.append(ds.size)
c_idx.append(idx.tolist())
print("\nhistogram of source counts in r_ext aperture")
for i in [(i,n_in_aper.count(i)) for i in set(n_in_aper)]:
print i
# create new version of catalog file with xsc-associated entries replaced
c_idx = np.array(flatten(c_idx))
print("\nremoving {}, adding {}".format(c_idx.size, phot.size))
replaced[cat] = {'old':c_idx.size, 'new':phot.size}
replaced[cat]['hist'] = [(i,n_in_aper.count(i)) for i in set(n_in_aper)]
c = np.delete(c, c_idx)
newrows = np.rec.array([(-i, da.ra[i], da.dec[i],
phot.flux[i], phot.unc[i], 1) for i in \
range(phot.size)], dtype=c.dtype)
newcat = np.hstack((c, newrows))
# write new version of catalog to disk
fmt = ['%i']+['%0.8f']*2+['%.4e']*2+['%i']
outpath = cat.replace('catalog.txt', 'catalog_xsc_cor.txt')
np.savetxt(outpath, newcat, fmt = fmt, header = ' '.join(names))
print('\ncreated file: {}'.format(outpath))
# make plot of total old vs. new flux
plt.scatter(c_flux_total, phot.flux)
ylim = plt.gca().get_ylim()
plt.xlim(*ylim)
max_y = ylim[1]
plt.plot(ylim, ylim, 'r-')
plt.xlabel('old flux [mJy]')
plt.ylabel('new flux [mJy]')
name = ' '.join(cat.split('/')[-1].split('_')[:2])
plt.title(name)
outpath = cat.replace('combined_hdr_catalog.txt','xsc_new_vs_old_phot.png')
plt.savefig(outpath, dpi=200)
plt.close()
print('\ncreated file: {}'.format(outpath))
outfile = 'xsc_replaced.json'
json.dump(replaced, open(outfile,'w'))
print("\ncreated file: {}".format(outfile))
print("\nremoved / added")
for k,v in replaced.iteritems():
print k.split('/')[-1], v['old'], v['new']
m = np.mean([i['old']/float(i['new']) for i in replaced.values()])
print("average ratio: {}".format(m))
print("\nK mag and r_ext of sources with NaN photometry:")
for i in find_files(bcdphot_out_path, "*xsc_nan_phot.csv"):
reg = i.split('/')[-1]
rec = np.recfromcsv(i)
bad_id = rec.designation.tolist()
bad_k = rec.k_m_k20fe.tolist()
bad_r_ext = rec.r_ext.tolist()
print reg
print ("\tid\t\t\tKmag\tr_ext")
if type(bad_id) is list:
seq = sorted(zip(bad_id, bad_k, bad_r_ext), key=lambda x: x[0])
for j,k,l in seq:
print("\t{}\t{}\t{}".format(j,k,l))
else:
print("\t{}\t{}\t{}".format(bad_id, bad_k, bad_r_ext))
def run_xsc_phot(bcdphot_out_path, mosaic_path):
replaced = {}
for cat in find_files(bcdphot_out_path, "*_combined_hdr_catalog.txt"):
print("\n======================================================")
print("\nadjusting photometry in: {}".format(cat.split('/')[-1]))
print("------------------------------------------------------")
outpath = cat.replace('combined_hdr_catalog.txt', '2mass_xsc.tbl')
# retrieve 2mass data if file doesn't already exist (from previous run)
if not os.path.isfile(outpath):
# get url and retrieve data
url = query_2mass_xsc_polygon(*get_region_corners(cat))
print("\ndownloading 2MASS photometry from: {}".format(url))
text = urllib2.urlopen(url).read()
# write to disk
with open(outpath, 'w') as f:
f.write(text)
print("\ncreated file: {}".format(outpath))
# read back in as recarray
print("\nreading: {}".format(outpath))
names = open(outpath).read().split('\n')[76].split('|')[1:-1]
da = np.recfromtxt(outpath, skip_header=80, names=names)
# write input file for xsc_phot.pro
infile_outpath = '/'.join(cat.split('/')[:-1]) + '/xsc.txt'
with open(infile_outpath, 'w') as w:
for i in range(da.shape[0]):
w.write("{} {} {} {}\n".format(da.designation[i], da.ra[i],
da.dec[i], da.r_ext[i]))
print(
"\ncreated input file for xsc_phot.pro: {}".format(infile_outpath))
# locate the FITS mosaic file for xsc_phot.pro to do photometry on
reg, ch = cat.split('/')[-1].split('_')[:2]
mosaicfile = filter(lambda x: 'dirbe{}/ch{}/long/full/Combine'\
.format(reg,ch) in x, find_files(mosaic_path, '*mosaic.fits'))[0]
print("\nfound mosaic file: {}".format(mosaicfile))
# spawn IDL subprocess running xsc_phot.pro and catch stdout in file
outpath = infile_outpath.replace('xsc.txt', 'xsc_phot_out.txt')
if not os.path.isfile(outpath):
outfile = open(outpath, 'w')
print("\nspawning xsc_phot.pro IDL subprocess")
cmd = "xsc_phot,'" + mosaicfile + "','" + infile_outpath + "','long'"
rc = subprocess.call(
['/usr/local/itt/idl71/bin/idl', '-quiet', '-e', cmd],
stderr=subprocess.PIPE,
stdout=outfile)
outfile.close()
# read in output to recarray
print("\nreading: {}".format(outpath))
phot = np.recfromtxt(outpath,
names=['id', 'flux', 'unc', 'sky', 'skyunc'])
# make sure rows are aligned
assert (da.designation == phot.id).all()
# ignore xsc sources we got a NaN or negative flux for
bad = np.isnan(phot.flux) | (phot.flux < 0)
print("\naper.pro returned NaN or negative flux for {} sources".format(
bad.sum()))
if bad.sum() > 0:
for i in phot[bad].id:
print(i)
outpath = cat.replace('combined_hdr_catalog.txt',
'xsc_nan_phot.csv')
with open(outpath, 'w') as f:
w = csv.writer(f)
w.writerow(da.dtype.names)
w.writerows(da[bad].tolist())
print('\ncreated file: {}'.format(outpath))
phot = phot[~bad]
da = da[~bad]
# read in pipeline catalog
print("\nreading: {}".format(cat))
names = open(cat).readline().split()[1:]
c = np.recfromtxt(cat, names=names)
# loop through xsc sources and find matches in pipeline catalog
print(
"\nfinding records associated with XSC sources in pipeline catalog"
)
c_flux_total = []
n_in_aper = []
c_idx = []
coords = radec_to_coords(c.ra, c.dec)
kdt = KDT(coords)
for i in range(phot.size):
radius = da.r_ext[i] / 3600.
# idx1, idx2, ds = spherematch(da.ra[i], da.dec[i],
# c.ra, c.dec, tolerance=radius)
idx, ds = spherematch2(da.ra[i],
da.dec[i],
c.ra,
c.dec,
kdt,
tolerance=radius,
k=500)
# c_flux_total.append(c.flux[idx2].sum())
# n_in_aper.append(c.flux[idx2].size)
# c_idx.append(idx2.tolist())
c_flux_total.append(c.flux[idx].sum())
n_in_aper.append(ds.size)
c_idx.append(idx.tolist())
print("\nhistogram of source counts in r_ext aperture")
for i in [(i, n_in_aper.count(i)) for i in set(n_in_aper)]:
print i
# create new version of catalog file with xsc-associated entries replaced
c_idx = np.array(flatten(c_idx))
print("\nremoving {}, adding {}".format(c_idx.size, phot.size))
replaced[cat] = {'old': c_idx.size, 'new': phot.size}
replaced[cat]['hist'] = [(i, n_in_aper.count(i))
for i in set(n_in_aper)]
c = np.delete(c, c_idx)
newrows = np.rec.array([(-i, da.ra[i], da.dec[i],
phot.flux[i], phot.unc[i], 1) for i in \
range(phot.size)], dtype=c.dtype)
newcat = np.hstack((c, newrows))
# write new version of catalog to disk
fmt = ['%i'] + ['%0.8f'] * 2 + ['%.4e'] * 2 + ['%i']
outpath = cat.replace('catalog.txt', 'catalog_xsc_cor.txt')
np.savetxt(outpath, newcat, fmt=fmt, header=' '.join(names))
print('\ncreated file: {}'.format(outpath))
# make plot of total old vs. new flux
plt.scatter(c_flux_total, phot.flux)
ylim = plt.gca().get_ylim()
plt.xlim(*ylim)
max_y = ylim[1]
plt.plot(ylim, ylim, 'r-')
plt.xlabel('old flux [mJy]')
plt.ylabel('new flux [mJy]')
name = ' '.join(cat.split('/')[-1].split('_')[:2])
plt.title(name)
outpath = cat.replace('combined_hdr_catalog.txt',
'xsc_new_vs_old_phot.png')
plt.savefig(outpath, dpi=200)
plt.close()
print('\ncreated file: {}'.format(outpath))
outfile = 'xsc_replaced.json'
json.dump(replaced, open(outfile, 'w'))
print("\ncreated file: {}".format(outfile))
print("\nremoved / added")
for k, v in replaced.iteritems():
print k.split('/')[-1], v['old'], v['new']
m = np.mean([i['old'] / float(i['new']) for i in replaced.values()])
print("average ratio: {}".format(m))
print("\nK mag and r_ext of sources with NaN photometry:")
for i in find_files(bcdphot_out_path, "*xsc_nan_phot.csv"):
reg = i.split('/')[-1]
rec = np.recfromcsv(i)
bad_id = rec.designation.tolist()
bad_k = rec.k_m_k20fe.tolist()
bad_r_ext = rec.r_ext.tolist()
print reg
print("\tid\t\t\tKmag\tr_ext")
if type(bad_id) is list:
seq = sorted(zip(bad_id, bad_k, bad_r_ext), key=lambda x: x[0])
for j, k, l in seq:
print("\t{}\t{}\t{}".format(j, k, l))
else:
print("\t{}\t{}\t{}".format(bad_id, bad_k, bad_r_ext))