コード例 #1
0
ファイル: wise.py プロジェクト: chaorun/bcdphot
def plot_wise(cat_path):

    for catfile in find_files(cat_path, "*merged+wise.csv"):

        print("\nreading catalog: {}".format(catfile))
        df = pd.read_csv(catfile)

        # convert to magnitudes
        nbadflux = (df.flux <= 0).sum()
        try:
            assert nbadflux == 0
        except:
            print("warning: {} negative flux source(s)".format(nbadflux))
        ch = catfile.split('/')[-1].split('_')[1]
        mags = spz_jy_to_mags(df.flux * 1e-3, float(ch))
        if ch == '1':
            plt.scatter(df.W1mag, mags)
            plt.xlabel('W1 [mag]')
            plt.ylabel('I1 [mag]')
        elif ch == '2':
            plt.scatter(df.W2mag, mags)
            plt.xlabel('W2 [mag]')
            plt.ylabel('I2 [mag]')
        ax = plt.gca()
        xlim, ylim = ax.get_xlim(), ax.get_ylim()
        plt.plot([-5, ylim[1] * 2], [-5, ylim[1] * 2], 'r-')
        ax.set_xlim(xlim)
        ax.set_ylim(ylim)
        reg = catfile.split('/')[-1].split('_')[0]
        name = '{}_{}_IRAC_vs_WISE.png'.format(reg, ch)
        outpath = '/'.join(catfile.split('/')[:-1] + [name])
        plt.savefig(outpath, dpi=120)
        plt.close()
コード例 #2
0
ファイル: wise.py プロジェクト: chaorun/bcdphot
def plot(x, y, outpath, xlabel, ylabel, plot_style, plot_type):
    if plot_type == 'mag-mag':
        xlim = (10, 18)
        ylim = (10, 18)
    elif plot_type == 'color-mag':
        xlim = (10, 18)
        ylim = (-1, 1)
    elif plot_type == 'color-color':
        xlim = (-1, 1)
        ylim = (-1, 1)
    else:
        raise (ValueError(
            "plot_type should be one of ['mag-mag', 'color-mag', 'color-color'] "
        ))
    isinrange = lambda a, b: (a >= b[0]) & (a <= b[1])
    g = isinrange(x, xlim) & isinrange(y, ylim)
    if plot_style == 'scatter':
        plt.scatter(x[g], y[g])
    elif plot_style == 'hexbin':
        plt.hexbin(x[g], y[g])
    elif plot_style == 'hist2d':
        plt.hist2d(x[g], y[g], bins=100)
    else:
        raise (ValueError(
            "plot_style should be one of ['scatter', 'hexbin', 'hist2d'] "))
    plt.xlabel(xlabel)
    plt.ylabel(ylabel)
    ax = plt.gca()
    ax.set_xlim(xlim)
    ax.set_ylim(ylim)
    plt.savefig(outpath, dpi=120)
    plt.close()
    print("created file: {}".format(outpath))
コード例 #3
0
ファイル: wise.py プロジェクト: john-livingston/bcdphot
def plot(x, y, outpath, xlabel, ylabel, plot_style, plot_type):
	if plot_type == 'mag-mag':
		xlim = (10, 18)
		ylim = (10, 18)
	elif plot_type == 'color-mag':
		xlim = (10, 18)
		ylim = (-1, 1)
	elif plot_type == 'color-color':
		xlim = (-1, 1)
		ylim = (-1, 1)
	else:
		raise(ValueError("plot_type should be one of ['mag-mag', 'color-mag', 'color-color'] "))
	isinrange = lambda a,b: (a>=b[0]) & (a<=b[1])
	g = isinrange(x, xlim) & isinrange(y, ylim)
	if plot_style == 'scatter':
		plt.scatter(x[g], y[g])
	elif plot_style == 'hexbin':
		plt.hexbin(x[g], y[g])
	elif plot_style == 'hist2d':
		plt.hist2d(x[g], y[g], bins=100)
	else:
		raise(ValueError("plot_style should be one of ['scatter', 'hexbin', 'hist2d'] "))
	plt.xlabel(xlabel)
	plt.ylabel(ylabel)
	ax = plt.gca()
	ax.set_xlim(xlim)
	ax.set_ylim(ylim)
	plt.savefig(outpath, dpi=120)
	plt.close()
	print("created file: {}".format(outpath))
コード例 #4
0
ファイル: wise.py プロジェクト: john-livingston/bcdphot
def plot_wise(cat_path):

	for catfile in find_files(cat_path, "*merged+wise.csv"):

		print("\nreading catalog: {}".format(catfile))
		df = pd.read_csv(catfile)

		# convert to magnitudes
		nbadflux = (df.flux <= 0).sum()
		try:
			assert nbadflux == 0
		except:
			print("warning: {} negative flux source(s)".format(nbadflux))
		ch = catfile.split('/')[-1].split('_')[1]
		mags = spz_jy_to_mags(df.flux*1e-3, float(ch))
		if ch == '1':
			plt.scatter(df.W1mag, mags)
			plt.xlabel('W1 [mag]')
			plt.ylabel('I1 [mag]')
		elif ch == '2':
			plt.scatter(df.W2mag, mags)
			plt.xlabel('W2 [mag]')
			plt.ylabel('I2 [mag]')
		ax = plt.gca()
		xlim, ylim = ax.get_xlim(), ax.get_ylim()
		plt.plot([-5, ylim[1]*2], [-5, ylim[1]*2], 'r-')
		ax.set_xlim(xlim) ; ax.set_ylim(ylim)
		reg = catfile.split('/')[-1].split('_')[0]
		name = '{}_{}_IRAC_vs_WISE.png'.format(reg, ch)
		outpath = '/'.join(catfile.split('/')[:-1]+[name])
		plt.savefig(outpath, dpi=120)
		plt.close()
コード例 #5
0
ファイル: cluster.py プロジェクト: stephentu/vldbtrends
def scipy_fcluster_merge_strategy(assignments):

    # Z-matrix helpers
    def groups(assignments):
        cluster_map = {}
        for idx, gid in enumerate(assignments):
            lst = cluster_map.get(gid, [])
            lst.append(idx)
            cluster_map[gid] = lst
        return tuple(sorted(map(tuple, cluster_map.values()), key=len, reverse=True))

    def zmatrix(assignments_samples):
        n = len(assignments_samples[0])
        # should make sparse matrix
        zmat = np.zeros((n, n), dtype=np.float32)
        for assignments in assignments_samples:
            clusters = groups(assignments)
            for cluster in clusters:
                for i, j in it.product(cluster, repeat=2):
                    zmat[i, j] += 1
        zmat /= float(len(assignments_samples))
        return zmat

    def reorder_zmat(zmat, order):
        zmat = zmat[order]
        zmat = zmat[:,order]
        return zmat

    def linkage(zmat):
        assert zmat.shape[0] == zmat.shape[1]
        zmat0 = np.array(zmat[np.triu_indices(zmat.shape[0], k=1)])
        zmat0 = 1. - zmat0
        return scipy.cluster.hierarchy.linkage(zmat0)

    def ordering(l):
        return np.array(scipy.cluster.hierarchy.leaves_list(l))

    zmat = zmatrix(assignments)
    li = linkage(zmat)

    # diagnostic: draw z-matrix
    indices = ordering(li)
    plt.imshow(reorder_zmat(zmat, indices),
           cmap=plt.cm.binary, interpolation='nearest')
    plt.savefig("zmat-before.pdf")
    plt.close()

    fassignment = scipy.cluster.hierarchy.fcluster(li, 0.001)
    clustering = groups(fassignment)
    sorted_ordering = list(it.chain.from_iterable(clustering))

    # draw post fcluster() ordered z-matrix
    plt.imshow(reorder_zmat(zmat, sorted_ordering),
           cmap=plt.cm.binary, interpolation='nearest')
    plt.savefig("zmat-after.pdf")
    plt.close()

    return fassignment
コード例 #6
0
ファイル: sdss.py プロジェクト: chaorun/bcdphot
def plot_sdss(cat_path):
    for catfile in find_files(cat_path, "*merged+sdss.txt"):

        # for now ignore the channel 2 files
        if catfile.split('/')[-1].split('_')[1] != '1':
            continue

        print("\nreading catalog: {}".format(catfile))
        df = pd.read_table(catfile, sep=' ')

        # get rid of negative flux sources, if any
        df = df[df.flux > 0]

        # convert to magnitudes
        mags = spz_jy_to_mags(df.flux * 1e-3, 1)

        # print counts per magnitude bin
        for i in range(10, 15):
            sc = ((df.cl == 3) & (mags > i) & (mags < i + 1)).sum()
            xc = ((df.xsc == 1) & (mags > i) & (mags < i + 1)).sum()
            msg = "{}th to {}th mag: {} SDSS galaxy sources, {} 2MASS XSC sources"
            print(msg.format(i, i + 1, sc, xc))

        # print number of sources agreed upon
        agree = ((df.xsc == 1) & (df.cl == 3)).sum()
        disagree = ((df.xsc == 1) & (df.cl == 6)).sum()
        na = ((df.xsc == 1) & (df.cl == 0)).sum()
        msg = "{} 2MASS XSC sources classified as galaxies by SDSS"
        print(msg.format(agree))
        msg = "{} 2MASS XSC sources classified as stars by SDSS"
        print(msg.format(disagree))
        msg = "{} 2MASS XSC sources not matched to SDSS"
        print(msg.format(na))

        # plot normed histograms of 2MASS XSC and SDSS galaxy magnitudes
        xsc_gals = (mags > 10) & (mags < 15) & (df.xsc == 1)
        sdss_gals = (mags > 10) & (mags < 15) & (df.cl == 3)
        # mags[xsc_gals].hist(label='2MASS XSC', normed=True)
        # mags[sdss_gals].hist(label='SDSS galaxies', normed=True)
        plt.hist([mags[xsc_gals].values, mags[sdss_gals].values],
                 bins=5,
                 label=['2MASS', 'SDSS'])
        plt.xlabel('IRAC1 [mag]')
        plt.ylabel('Number Count')
        reg = catfile.split('/')[-1].split('_')[0]
        plt.title('{} Extended Sources / Galaxies'.format(reg))
        plt.legend(loc=2)
        name = '{}_2mass_xsc_vs_sdss_hist.png'.format(reg)
        outpath = '/'.join(catfile.split('/')[:-1] + [name])
        plt.savefig(outpath, dpi=100)
        plt.close()
        print("created file: {}".format(outpath))
コード例 #7
0
ファイル: sdss.py プロジェクト: john-livingston/bcdphot
def plot_sdss(cat_path):
	for catfile in find_files(cat_path, "*merged+sdss.txt"):

		# for now ignore the channel 2 files
		if catfile.split('/')[-1].split('_')[1] != '1':
			continue

		print("\nreading catalog: {}".format(catfile))
		df = pd.read_table(catfile, sep=' ')

		# get rid of negative flux sources, if any
		df = df[df.flux > 0]

		# convert to magnitudes
		mags = spz_jy_to_mags(df.flux*1e-3, 1)

		# print counts per magnitude bin
		for i in range(10,15):
			sc = ((df.cl == 3) & (mags > i) & (mags < i+1)).sum()
			xc = ((df.xsc == 1) & (mags > i) & (mags < i+1)).sum() 
			msg = "{}th to {}th mag: {} SDSS galaxy sources, {} 2MASS XSC sources"
			print(msg.format(i, i+1, sc, xc))

		# print number of sources agreed upon
		agree = ((df.xsc == 1) & (df.cl == 3)).sum()
		disagree = ((df.xsc == 1) & (df.cl == 6)).sum()
		na = ((df.xsc == 1) & (df.cl == 0)).sum()
		msg = "{} 2MASS XSC sources classified as galaxies by SDSS"
		print(msg.format(agree))
		msg = "{} 2MASS XSC sources classified as stars by SDSS"
		print(msg.format(disagree))
		msg = "{} 2MASS XSC sources not matched to SDSS"
		print(msg.format(na))

		# plot normed histograms of 2MASS XSC and SDSS galaxy magnitudes
		xsc_gals = (mags > 10) & (mags < 15) & (df.xsc == 1)
		sdss_gals = (mags > 10) & (mags < 15) & (df.cl == 3)
		# mags[xsc_gals].hist(label='2MASS XSC', normed=True)
		# mags[sdss_gals].hist(label='SDSS galaxies', normed=True)
		plt.hist([mags[xsc_gals].values, mags[sdss_gals].values],
			bins=5, label=['2MASS', 'SDSS'])
		plt.xlabel('IRAC1 [mag]')
		plt.ylabel('Number Count')
		reg = catfile.split('/')[-1].split('_')[0]
		plt.title('{} Extended Sources / Galaxies'.format(reg))
		plt.legend(loc=2)
		name = '{}_2mass_xsc_vs_sdss_hist.png'.format(reg)
		outpath = '/'.join(catfile.split('/')[:-1]+[name])
		plt.savefig(outpath, dpi=100)
		plt.close()
		print("created file: {}".format(outpath))
コード例 #8
0
def plot_signal(signal, shape=None, name=None):
    """
    Plot a numeric signal as an image.
    """
    s = np.copy(signal)
    if s.max() != 0.:
        s *= 255.0/s.max()
    s = s.astype(int)
    if shape:
        s = s.reshape(shape)

    plt.figure()
    plt.imshow(s, cmap=plt.cm.gray, interpolation='nearest')
    if name:
        plt.savefig(name)
    plt.close()
コード例 #9
0
def layerActivations(model, data, labels):
    print('Visualizing activations with tSNE...')

    if not os.path.exists(cc.cfg['plots']['layer_activations_dir']):
        os.makedirs(cc.cfg['plots']['layer_activations_dir'])


    numLabels = cc.cfg['plots']['layer_activations_label_cap']

    data = data[:cc.cfg['plots']['layer_activations_points_cap']]
    labels = labels[:numLabels,:cc.cfg['plots']['layer_activations_points_cap']]

    subplotCols = numLabels
    subplotRows = len(model.layers)-1
    subplotIdx = 1

    plt.figure(figsize=(5*subplotCols,5*subplotRows))

    for i in range(1,len(model.layers)):
        print('Running tSNE for layer {}/{}'.format(i+1,len(model.layers)))

        func = K.function([model.layers[0].input], [model.layers[i].output])
        out = func([data])[0]

        tsneModel = TSNE(n_components = 2, random_state = 0)
        tsneOut = tsneModel.fit_transform(out).T

        # labeledTsneOut = np.hstack((tsneOut, labels[0].reshape(-1,1)))

        for j in range(numLabels):
            plt.subplot(subplotRows, subplotCols, subplotIdx)
            plt.title('{} / {}'.format(model.layers[i].name,cc.exp['params']['data']['labels'][j]))
            plt.scatter(tsneOut[0],tsneOut[1],c=labels[j],cmap = 'plasma')

            subplotIdx += 1

        # tsneDF = pd.DataFrame(labeledTsneOut, columns = ('a', 'b', 'c'))
        # plot = tsneDF.plot.scatter(x = 'a', y = 'b', c = 'c', cmap = 'plasma')


    plt.tight_layout()
    plt.savefig('{}/activations.png'.format(cc.cfg['plots']['layer_activations_dir']))
    plt.close()

    print('...done')
コード例 #10
0
def histograms(modelLogger):
    if not cc.cfg['plots']['histograms']:
        return

    if not os.path.exists(cc.cfg['plots']['histograms_dir']):
        os.makedirs(cc.cfg['plots']['histograms_dir'])

    cntEpochs = len(modelLogger.epochLogs)
    cntLayers = len(modelLogger.epochLogs[-1]['weights'])

    logVals = [
        {'name':'weights','color':'blue'},
        {'name':'updates','color':'red'},
        {'name':'ratios','color':'green'}
    ]

    subplotRows = len(logVals)
    subplotCols = cntLayers

    for x in range(cntEpochs):
        subplotIdx = 1

        plt.figure(figsize=(5*subplotCols,5*subplotRows))
        plt.suptitle('Histograms per layer, epoch {}/{}'.format(x,cntEpochs-1), fontsize=14)

        for logVal in logVals:
            for i,layer in enumerate(modelLogger.epochLogs[x][logVal['name']]):
                histmin = layer.min()
                histmax = layer.max()

                plt.subplot(subplotRows, subplotCols, subplotIdx)
                plt.title('{}, {}'.format(modelLogger.model.layers[modelLogger.loggedLayers[i]].name,logVal['name']))
                plt.hist(layer, range = (histmin, histmax), bins = 30, color = logVal['color'])

                subplotIdx+=1

        plt.tight_layout()
        plt.subplots_adjust(top=0.9)
        plt.savefig('{}/hist_{e:03d}.png'.format(cc.cfg['plots']['histograms_dir'], e = x))
        plt.close()
コード例 #11
0
env.reset(np.array([0.0]))
code.reset()

[mp,varp,spsp,sp,msep,parts,ws] = pf.particle_filter(code, env, timewindow=timewindow,
                                                     dt=dt, nparticles=nparticles,
                                                     mode='v', testf=(lambda x:x))
if gaussian:
    print "MSE of gaussian filter %f"% mseg
print "MSE of particle filter %f"% msep

if plotting:
    
    #matplotlib.rcParams['font.size']=10
    
    plt.close()    
    plt.figure()
    ax1 = plt.gcf().add_subplot(2,1,1)
    times = np.arange(0.0,dt*timewindow,dt)
    if gaussian:    
        ax1.plot(times,sg,'r',label='Signal')
        if sum(sum(spsg)) !=0:
            (ts,neurs) = np.where(spsg == 1)
            spiketimes = times[ts]
            thetas = [code.neurons[i].theta for i in neurs]
        else:
            spiketimes = []
            thetas = []
        
        ax1.plot(spiketimes,thetas,'yo',label='Spike times')
        ax1.plot(times,mg,'b',label='Mean prediction')
コード例 #12
0
[78,  376],
[79,  164],
[80,  194],
[81,  342],
[82,  314],
[83,  68],
[84,  54],
[85,  131],
[86,  52],
[87,  220],
[88,  145],
[89,  63],
[90,  608],
[91,  68],
[92,  155],
[93,  49],
[94,  108],
[95,  59],
[96,  67],
[97,  58],
[98,  42],
[99,  210],
[100, 174]])



plt.figure(figsize=(5,5))
plt.scatter(histdata.T[0],histdata.T[1])
plt.savefig('hist.png')
plt.close()
コード例 #13
0
ファイル: xsc.py プロジェクト: john-livingston/bcdphot
def run_xsc_phot(bcdphot_out_path, mosaic_path):
	replaced = {}
	for cat in find_files(bcdphot_out_path, "*_combined_hdr_catalog.txt"):

		print("\n======================================================")
		print("\nadjusting photometry in: {}".format(cat.split('/')[-1]))
		print("------------------------------------------------------")
		outpath = cat.replace('combined_hdr_catalog.txt','2mass_xsc.tbl')

		# retrieve 2mass data if file doesn't already exist (from previous run)
		if not os.path.isfile(outpath):
			# get url and retrieve data
			url = query_2mass_xsc_polygon(*get_region_corners(cat))
			print("\ndownloading 2MASS photometry from: {}".format(url))
			text = urllib2.urlopen(url).read()
			# write to disk
			with open(outpath, 'w') as f:
				f.write(text)
			print("\ncreated file: {}".format(outpath))

		# read back in as recarray	
		print("\nreading: {}".format(outpath))
		names = open(outpath).read().split('\n')[76].split('|')[1:-1]
		da = np.recfromtxt(outpath, skip_header=80, names=names)

		# write input file for xsc_phot.pro
		infile_outpath = '/'.join(cat.split('/')[:-1])+'/xsc.txt'
		with open(infile_outpath,'w') as w:
			for i in range(da.shape[0]):
				w.write("{} {} {} {}\n".format(da.designation[i], da.ra[i], da.dec[i], da.r_ext[i]))
		print("\ncreated input file for xsc_phot.pro: {}".format(infile_outpath))

		# locate the FITS mosaic file for xsc_phot.pro to do photometry on
		reg, ch = cat.split('/')[-1].split('_')[:2]
		mosaicfile = filter(lambda x: 'dirbe{}/ch{}/long/full/Combine'\
			.format(reg,ch) in x, find_files(mosaic_path, '*mosaic.fits'))[0]
		print("\nfound mosaic file: {}".format(mosaicfile))

		# spawn IDL subprocess running xsc_phot.pro and catch stdout in file
		outpath = infile_outpath.replace('xsc.txt', 'xsc_phot_out.txt')
		if not os.path.isfile(outpath):
			outfile = open(outpath,'w')
			print("\nspawning xsc_phot.pro IDL subprocess")
			cmd = "xsc_phot,'"+mosaicfile+"','"+infile_outpath+"','long'"
			rc = subprocess.call(['/usr/local/itt/idl71/bin/idl','-quiet','-e',cmd], 
				stderr = subprocess.PIPE, stdout = outfile)
			outfile.close()

		# read in output to recarray
		print("\nreading: {}".format(outpath))
		phot = np.recfromtxt(outpath, names=['id','flux','unc','sky','skyunc'])

		# make sure rows are aligned
		assert (da.designation == phot.id).all()

		# ignore xsc sources we got a NaN or negative flux for
		bad = np.isnan(phot.flux) | (phot.flux < 0)
		print("\naper.pro returned NaN or negative flux for {} sources".format(bad.sum()))
		if bad.sum() > 0:
			for i in phot[bad].id:
				print(i)
			outpath = cat.replace('combined_hdr_catalog.txt','xsc_nan_phot.csv')
			with open(outpath,'w') as f:
				w = csv.writer(f)
				w.writerow(da.dtype.names)
				w.writerows(da[bad].tolist())
			print('\ncreated file: {}'.format(outpath))
		phot = phot[~bad]
		da = da[~bad]

		# read in pipeline catalog
		print("\nreading: {}".format(cat))
		names = open(cat).readline().split()[1:]
		c = np.recfromtxt(cat, names=names)

		# loop through xsc sources and find matches in pipeline catalog
		print("\nfinding records associated with XSC sources in pipeline catalog")
		c_flux_total = []
		n_in_aper = []
		c_idx = []
		coords = radec_to_coords(c.ra, c.dec)
		kdt = KDT(coords)
		for i in range(phot.size):
			radius = da.r_ext[i]/3600.
			# idx1, idx2, ds = spherematch(da.ra[i], da.dec[i], 
			# 	c.ra, c.dec, tolerance=radius)
			idx, ds = spherematch2(da.ra[i], da.dec[i], c.ra, c.dec,
				kdt, tolerance=radius, k=500)
			# c_flux_total.append(c.flux[idx2].sum())
			# n_in_aper.append(c.flux[idx2].size)
			# c_idx.append(idx2.tolist())
			c_flux_total.append(c.flux[idx].sum())
			n_in_aper.append(ds.size)
			c_idx.append(idx.tolist())
		print("\nhistogram of source counts in r_ext aperture")
		for i in [(i,n_in_aper.count(i)) for i in set(n_in_aper)]:
			print i

		# create new version of catalog file with xsc-associated entries replaced
		c_idx = np.array(flatten(c_idx))
		print("\nremoving {}, adding {}".format(c_idx.size, phot.size))
		replaced[cat] = {'old':c_idx.size, 'new':phot.size}
		replaced[cat]['hist'] = [(i,n_in_aper.count(i)) for i in set(n_in_aper)]
		c = np.delete(c, c_idx)
		newrows = np.rec.array([(-i, da.ra[i], da.dec[i], 
			phot.flux[i], phot.unc[i], 1) for i in \
			range(phot.size)], dtype=c.dtype)
		newcat = np.hstack((c, newrows))

		# write new version of catalog to disk
		fmt = ['%i']+['%0.8f']*2+['%.4e']*2+['%i']
		outpath = cat.replace('catalog.txt', 'catalog_xsc_cor.txt')
		np.savetxt(outpath, newcat, fmt = fmt, header = ' '.join(names))
		print('\ncreated file: {}'.format(outpath))

		# make plot of total old vs. new flux
		plt.scatter(c_flux_total, phot.flux)
		ylim = plt.gca().get_ylim()
		plt.xlim(*ylim)
		max_y = ylim[1]
		plt.plot(ylim, ylim, 'r-')
		plt.xlabel('old flux [mJy]')
		plt.ylabel('new flux [mJy]')
		name = ' '.join(cat.split('/')[-1].split('_')[:2])
		plt.title(name)
		outpath = cat.replace('combined_hdr_catalog.txt','xsc_new_vs_old_phot.png')
		plt.savefig(outpath, dpi=200)
		plt.close()
		print('\ncreated file: {}'.format(outpath))

	outfile = 'xsc_replaced.json'
	json.dump(replaced, open(outfile,'w'))
	print("\ncreated file: {}".format(outfile))
	print("\nremoved / added")
	for k,v in replaced.iteritems():
		print k.split('/')[-1], v['old'], v['new']
	m = np.mean([i['old']/float(i['new']) for i in replaced.values()])
	print("average ratio: {}".format(m))
	print("\nK mag and r_ext of sources with NaN photometry:")
	for i in find_files(bcdphot_out_path, "*xsc_nan_phot.csv"):
		reg = i.split('/')[-1]
		rec = np.recfromcsv(i)
		bad_id = rec.designation.tolist()
		bad_k = rec.k_m_k20fe.tolist()
		bad_r_ext = rec.r_ext.tolist()
		print reg
		print ("\tid\t\t\tKmag\tr_ext")
		if type(bad_id) is list:
			seq = sorted(zip(bad_id, bad_k, bad_r_ext), key=lambda x: x[0])
			for j,k,l in seq:
				print("\t{}\t{}\t{}".format(j,k,l))
		else:
			print("\t{}\t{}\t{}".format(bad_id, bad_k, bad_r_ext))
コード例 #14
0
def run_xsc_phot(bcdphot_out_path, mosaic_path):
    replaced = {}
    for cat in find_files(bcdphot_out_path, "*_combined_hdr_catalog.txt"):

        print("\n======================================================")
        print("\nadjusting photometry in: {}".format(cat.split('/')[-1]))
        print("------------------------------------------------------")
        outpath = cat.replace('combined_hdr_catalog.txt', '2mass_xsc.tbl')

        # retrieve 2mass data if file doesn't already exist (from previous run)
        if not os.path.isfile(outpath):
            # get url and retrieve data
            url = query_2mass_xsc_polygon(*get_region_corners(cat))
            print("\ndownloading 2MASS photometry from: {}".format(url))
            text = urllib2.urlopen(url).read()
            # write to disk
            with open(outpath, 'w') as f:
                f.write(text)
            print("\ncreated file: {}".format(outpath))

        # read back in as recarray
        print("\nreading: {}".format(outpath))
        names = open(outpath).read().split('\n')[76].split('|')[1:-1]
        da = np.recfromtxt(outpath, skip_header=80, names=names)

        # write input file for xsc_phot.pro
        infile_outpath = '/'.join(cat.split('/')[:-1]) + '/xsc.txt'
        with open(infile_outpath, 'w') as w:
            for i in range(da.shape[0]):
                w.write("{} {} {} {}\n".format(da.designation[i], da.ra[i],
                                               da.dec[i], da.r_ext[i]))
        print(
            "\ncreated input file for xsc_phot.pro: {}".format(infile_outpath))

        # locate the FITS mosaic file for xsc_phot.pro to do photometry on
        reg, ch = cat.split('/')[-1].split('_')[:2]
        mosaicfile = filter(lambda x: 'dirbe{}/ch{}/long/full/Combine'\
         .format(reg,ch) in x, find_files(mosaic_path, '*mosaic.fits'))[0]
        print("\nfound mosaic file: {}".format(mosaicfile))

        # spawn IDL subprocess running xsc_phot.pro and catch stdout in file
        outpath = infile_outpath.replace('xsc.txt', 'xsc_phot_out.txt')
        if not os.path.isfile(outpath):
            outfile = open(outpath, 'w')
            print("\nspawning xsc_phot.pro IDL subprocess")
            cmd = "xsc_phot,'" + mosaicfile + "','" + infile_outpath + "','long'"
            rc = subprocess.call(
                ['/usr/local/itt/idl71/bin/idl', '-quiet', '-e', cmd],
                stderr=subprocess.PIPE,
                stdout=outfile)
            outfile.close()

        # read in output to recarray
        print("\nreading: {}".format(outpath))
        phot = np.recfromtxt(outpath,
                             names=['id', 'flux', 'unc', 'sky', 'skyunc'])

        # make sure rows are aligned
        assert (da.designation == phot.id).all()

        # ignore xsc sources we got a NaN or negative flux for
        bad = np.isnan(phot.flux) | (phot.flux < 0)
        print("\naper.pro returned NaN or negative flux for {} sources".format(
            bad.sum()))
        if bad.sum() > 0:
            for i in phot[bad].id:
                print(i)
            outpath = cat.replace('combined_hdr_catalog.txt',
                                  'xsc_nan_phot.csv')
            with open(outpath, 'w') as f:
                w = csv.writer(f)
                w.writerow(da.dtype.names)
                w.writerows(da[bad].tolist())
            print('\ncreated file: {}'.format(outpath))
        phot = phot[~bad]
        da = da[~bad]

        # read in pipeline catalog
        print("\nreading: {}".format(cat))
        names = open(cat).readline().split()[1:]
        c = np.recfromtxt(cat, names=names)

        # loop through xsc sources and find matches in pipeline catalog
        print(
            "\nfinding records associated with XSC sources in pipeline catalog"
        )
        c_flux_total = []
        n_in_aper = []
        c_idx = []
        coords = radec_to_coords(c.ra, c.dec)
        kdt = KDT(coords)
        for i in range(phot.size):
            radius = da.r_ext[i] / 3600.
            # idx1, idx2, ds = spherematch(da.ra[i], da.dec[i],
            # 	c.ra, c.dec, tolerance=radius)
            idx, ds = spherematch2(da.ra[i],
                                   da.dec[i],
                                   c.ra,
                                   c.dec,
                                   kdt,
                                   tolerance=radius,
                                   k=500)
            # c_flux_total.append(c.flux[idx2].sum())
            # n_in_aper.append(c.flux[idx2].size)
            # c_idx.append(idx2.tolist())
            c_flux_total.append(c.flux[idx].sum())
            n_in_aper.append(ds.size)
            c_idx.append(idx.tolist())
        print("\nhistogram of source counts in r_ext aperture")
        for i in [(i, n_in_aper.count(i)) for i in set(n_in_aper)]:
            print i

        # create new version of catalog file with xsc-associated entries replaced
        c_idx = np.array(flatten(c_idx))
        print("\nremoving {}, adding {}".format(c_idx.size, phot.size))
        replaced[cat] = {'old': c_idx.size, 'new': phot.size}
        replaced[cat]['hist'] = [(i, n_in_aper.count(i))
                                 for i in set(n_in_aper)]
        c = np.delete(c, c_idx)
        newrows = np.rec.array([(-i, da.ra[i], da.dec[i],
         phot.flux[i], phot.unc[i], 1) for i in \
         range(phot.size)], dtype=c.dtype)
        newcat = np.hstack((c, newrows))

        # write new version of catalog to disk
        fmt = ['%i'] + ['%0.8f'] * 2 + ['%.4e'] * 2 + ['%i']
        outpath = cat.replace('catalog.txt', 'catalog_xsc_cor.txt')
        np.savetxt(outpath, newcat, fmt=fmt, header=' '.join(names))
        print('\ncreated file: {}'.format(outpath))

        # make plot of total old vs. new flux
        plt.scatter(c_flux_total, phot.flux)
        ylim = plt.gca().get_ylim()
        plt.xlim(*ylim)
        max_y = ylim[1]
        plt.plot(ylim, ylim, 'r-')
        plt.xlabel('old flux [mJy]')
        plt.ylabel('new flux [mJy]')
        name = ' '.join(cat.split('/')[-1].split('_')[:2])
        plt.title(name)
        outpath = cat.replace('combined_hdr_catalog.txt',
                              'xsc_new_vs_old_phot.png')
        plt.savefig(outpath, dpi=200)
        plt.close()
        print('\ncreated file: {}'.format(outpath))

    outfile = 'xsc_replaced.json'
    json.dump(replaced, open(outfile, 'w'))
    print("\ncreated file: {}".format(outfile))
    print("\nremoved / added")
    for k, v in replaced.iteritems():
        print k.split('/')[-1], v['old'], v['new']
    m = np.mean([i['old'] / float(i['new']) for i in replaced.values()])
    print("average ratio: {}".format(m))
    print("\nK mag and r_ext of sources with NaN photometry:")
    for i in find_files(bcdphot_out_path, "*xsc_nan_phot.csv"):
        reg = i.split('/')[-1]
        rec = np.recfromcsv(i)
        bad_id = rec.designation.tolist()
        bad_k = rec.k_m_k20fe.tolist()
        bad_r_ext = rec.r_ext.tolist()
        print reg
        print("\tid\t\t\tKmag\tr_ext")
        if type(bad_id) is list:
            seq = sorted(zip(bad_id, bad_k, bad_r_ext), key=lambda x: x[0])
            for j, k, l in seq:
                print("\t{}\t{}\t{}".format(j, k, l))
        else:
            print("\t{}\t{}\t{}".format(bad_id, bad_k, bad_r_ext))