def plot_wise(cat_path): for catfile in find_files(cat_path, "*merged+wise.csv"): print("\nreading catalog: {}".format(catfile)) df = pd.read_csv(catfile) # convert to magnitudes nbadflux = (df.flux <= 0).sum() try: assert nbadflux == 0 except: print("warning: {} negative flux source(s)".format(nbadflux)) ch = catfile.split('/')[-1].split('_')[1] mags = spz_jy_to_mags(df.flux * 1e-3, float(ch)) if ch == '1': plt.scatter(df.W1mag, mags) plt.xlabel('W1 [mag]') plt.ylabel('I1 [mag]') elif ch == '2': plt.scatter(df.W2mag, mags) plt.xlabel('W2 [mag]') plt.ylabel('I2 [mag]') ax = plt.gca() xlim, ylim = ax.get_xlim(), ax.get_ylim() plt.plot([-5, ylim[1] * 2], [-5, ylim[1] * 2], 'r-') ax.set_xlim(xlim) ax.set_ylim(ylim) reg = catfile.split('/')[-1].split('_')[0] name = '{}_{}_IRAC_vs_WISE.png'.format(reg, ch) outpath = '/'.join(catfile.split('/')[:-1] + [name]) plt.savefig(outpath, dpi=120) plt.close()
def plot(x, y, outpath, xlabel, ylabel, plot_style, plot_type): if plot_type == 'mag-mag': xlim = (10, 18) ylim = (10, 18) elif plot_type == 'color-mag': xlim = (10, 18) ylim = (-1, 1) elif plot_type == 'color-color': xlim = (-1, 1) ylim = (-1, 1) else: raise (ValueError( "plot_type should be one of ['mag-mag', 'color-mag', 'color-color'] " )) isinrange = lambda a, b: (a >= b[0]) & (a <= b[1]) g = isinrange(x, xlim) & isinrange(y, ylim) if plot_style == 'scatter': plt.scatter(x[g], y[g]) elif plot_style == 'hexbin': plt.hexbin(x[g], y[g]) elif plot_style == 'hist2d': plt.hist2d(x[g], y[g], bins=100) else: raise (ValueError( "plot_style should be one of ['scatter', 'hexbin', 'hist2d'] ")) plt.xlabel(xlabel) plt.ylabel(ylabel) ax = plt.gca() ax.set_xlim(xlim) ax.set_ylim(ylim) plt.savefig(outpath, dpi=120) plt.close() print("created file: {}".format(outpath))
def plot(x, y, outpath, xlabel, ylabel, plot_style, plot_type): if plot_type == 'mag-mag': xlim = (10, 18) ylim = (10, 18) elif plot_type == 'color-mag': xlim = (10, 18) ylim = (-1, 1) elif plot_type == 'color-color': xlim = (-1, 1) ylim = (-1, 1) else: raise(ValueError("plot_type should be one of ['mag-mag', 'color-mag', 'color-color'] ")) isinrange = lambda a,b: (a>=b[0]) & (a<=b[1]) g = isinrange(x, xlim) & isinrange(y, ylim) if plot_style == 'scatter': plt.scatter(x[g], y[g]) elif plot_style == 'hexbin': plt.hexbin(x[g], y[g]) elif plot_style == 'hist2d': plt.hist2d(x[g], y[g], bins=100) else: raise(ValueError("plot_style should be one of ['scatter', 'hexbin', 'hist2d'] ")) plt.xlabel(xlabel) plt.ylabel(ylabel) ax = plt.gca() ax.set_xlim(xlim) ax.set_ylim(ylim) plt.savefig(outpath, dpi=120) plt.close() print("created file: {}".format(outpath))
def plot_wise(cat_path): for catfile in find_files(cat_path, "*merged+wise.csv"): print("\nreading catalog: {}".format(catfile)) df = pd.read_csv(catfile) # convert to magnitudes nbadflux = (df.flux <= 0).sum() try: assert nbadflux == 0 except: print("warning: {} negative flux source(s)".format(nbadflux)) ch = catfile.split('/')[-1].split('_')[1] mags = spz_jy_to_mags(df.flux*1e-3, float(ch)) if ch == '1': plt.scatter(df.W1mag, mags) plt.xlabel('W1 [mag]') plt.ylabel('I1 [mag]') elif ch == '2': plt.scatter(df.W2mag, mags) plt.xlabel('W2 [mag]') plt.ylabel('I2 [mag]') ax = plt.gca() xlim, ylim = ax.get_xlim(), ax.get_ylim() plt.plot([-5, ylim[1]*2], [-5, ylim[1]*2], 'r-') ax.set_xlim(xlim) ; ax.set_ylim(ylim) reg = catfile.split('/')[-1].split('_')[0] name = '{}_{}_IRAC_vs_WISE.png'.format(reg, ch) outpath = '/'.join(catfile.split('/')[:-1]+[name]) plt.savefig(outpath, dpi=120) plt.close()
def plot_3d_scatter(self, fig, ax, x, y, z, title='', xlabel='', ylabel='', zlabel='', title_font={}, axis_font={}, tick_font={}): if not title_font: title_font = title_font_default if not axis_font: axis_font = axis_font_default cmap = plt.cm.jet h = plt.scatter(x, y, c=z, cmap=cmap) ax.set_aspect(1./ax.get_data_ratio()) # make axes square cbar = plt.colorbar(h, orientation='vertical', aspect=30, shrink=0.9) if xlabel: ax.set_xlabel(xlabel.replace("_", " "), labelpad=10, **axis_font) if ylabel: ax.set_ylabel(ylabel.replace("_", " "), labelpad=10, **axis_font) if zlabel: cbar.ax.set_ylabel(zlabel.replace("_", " "), labelpad=10, **axis_font) if tick_font: ax.tick_params(**tick_font) if title: ax.set_title(title.replace("_", " "), **title_font) ax.grid(True) plt.tight_layout()
def layerActivations(model, data, labels): print('Visualizing activations with tSNE...') if not os.path.exists(cc.cfg['plots']['layer_activations_dir']): os.makedirs(cc.cfg['plots']['layer_activations_dir']) numLabels = cc.cfg['plots']['layer_activations_label_cap'] data = data[:cc.cfg['plots']['layer_activations_points_cap']] labels = labels[:numLabels,:cc.cfg['plots']['layer_activations_points_cap']] subplotCols = numLabels subplotRows = len(model.layers)-1 subplotIdx = 1 plt.figure(figsize=(5*subplotCols,5*subplotRows)) for i in range(1,len(model.layers)): print('Running tSNE for layer {}/{}'.format(i+1,len(model.layers))) func = K.function([model.layers[0].input], [model.layers[i].output]) out = func([data])[0] tsneModel = TSNE(n_components = 2, random_state = 0) tsneOut = tsneModel.fit_transform(out).T # labeledTsneOut = np.hstack((tsneOut, labels[0].reshape(-1,1))) for j in range(numLabels): plt.subplot(subplotRows, subplotCols, subplotIdx) plt.title('{} / {}'.format(model.layers[i].name,cc.exp['params']['data']['labels'][j])) plt.scatter(tsneOut[0],tsneOut[1],c=labels[j],cmap = 'plasma') subplotIdx += 1 # tsneDF = pd.DataFrame(labeledTsneOut, columns = ('a', 'b', 'c')) # plot = tsneDF.plot.scatter(x = 'a', y = 'b', c = 'c', cmap = 'plasma') plt.tight_layout() plt.savefig('{}/activations.png'.format(cc.cfg['plots']['layer_activations_dir'])) plt.close() print('...done')
def plot_3d_scatter(self, fig, ax, x, y, z, title='', xlabel='', ylabel='', zlabel='', title_font={}, axis_font={}, tick_font={}): if not title_font: title_font = title_font_default if not axis_font: axis_font = axis_font_default cmap = plt.cm.jet h = plt.scatter(x, y, c=z, cmap=cmap) ax.set_aspect(1. / ax.get_data_ratio()) # make axes square cbar = plt.colorbar(h, orientation='vertical', aspect=30, shrink=0.9) if xlabel: ax.set_xlabel(xlabel.replace("_", " "), labelpad=10, **axis_font) if ylabel: ax.set_ylabel(ylabel.replace("_", " "), labelpad=10, **axis_font) if zlabel: cbar.ax.set_ylabel(zlabel.replace("_", " "), labelpad=10, **axis_font) if tick_font: ax.tick_params(**tick_font) if title: ax.set_title(title.replace("_", " "), **title_font) ax.grid(True) plt.tight_layout()
[78, 376], [79, 164], [80, 194], [81, 342], [82, 314], [83, 68], [84, 54], [85, 131], [86, 52], [87, 220], [88, 145], [89, 63], [90, 608], [91, 68], [92, 155], [93, 49], [94, 108], [95, 59], [96, 67], [97, 58], [98, 42], [99, 210], [100, 174]]) plt.figure(figsize=(5,5)) plt.scatter(histdata.T[0],histdata.T[1]) plt.savefig('hist.png') plt.close()
def run_xsc_phot(bcdphot_out_path, mosaic_path): replaced = {} for cat in find_files(bcdphot_out_path, "*_combined_hdr_catalog.txt"): print("\n======================================================") print("\nadjusting photometry in: {}".format(cat.split('/')[-1])) print("------------------------------------------------------") outpath = cat.replace('combined_hdr_catalog.txt','2mass_xsc.tbl') # retrieve 2mass data if file doesn't already exist (from previous run) if not os.path.isfile(outpath): # get url and retrieve data url = query_2mass_xsc_polygon(*get_region_corners(cat)) print("\ndownloading 2MASS photometry from: {}".format(url)) text = urllib2.urlopen(url).read() # write to disk with open(outpath, 'w') as f: f.write(text) print("\ncreated file: {}".format(outpath)) # read back in as recarray print("\nreading: {}".format(outpath)) names = open(outpath).read().split('\n')[76].split('|')[1:-1] da = np.recfromtxt(outpath, skip_header=80, names=names) # write input file for xsc_phot.pro infile_outpath = '/'.join(cat.split('/')[:-1])+'/xsc.txt' with open(infile_outpath,'w') as w: for i in range(da.shape[0]): w.write("{} {} {} {}\n".format(da.designation[i], da.ra[i], da.dec[i], da.r_ext[i])) print("\ncreated input file for xsc_phot.pro: {}".format(infile_outpath)) # locate the FITS mosaic file for xsc_phot.pro to do photometry on reg, ch = cat.split('/')[-1].split('_')[:2] mosaicfile = filter(lambda x: 'dirbe{}/ch{}/long/full/Combine'\ .format(reg,ch) in x, find_files(mosaic_path, '*mosaic.fits'))[0] print("\nfound mosaic file: {}".format(mosaicfile)) # spawn IDL subprocess running xsc_phot.pro and catch stdout in file outpath = infile_outpath.replace('xsc.txt', 'xsc_phot_out.txt') if not os.path.isfile(outpath): outfile = open(outpath,'w') print("\nspawning xsc_phot.pro IDL subprocess") cmd = "xsc_phot,'"+mosaicfile+"','"+infile_outpath+"','long'" rc = subprocess.call(['/usr/local/itt/idl71/bin/idl','-quiet','-e',cmd], stderr = subprocess.PIPE, stdout = outfile) outfile.close() # read in output to recarray print("\nreading: {}".format(outpath)) phot = np.recfromtxt(outpath, names=['id','flux','unc','sky','skyunc']) # make sure rows are aligned assert (da.designation == phot.id).all() # ignore xsc sources we got a NaN or negative flux for bad = np.isnan(phot.flux) | (phot.flux < 0) print("\naper.pro returned NaN or negative flux for {} sources".format(bad.sum())) if bad.sum() > 0: for i in phot[bad].id: print(i) outpath = cat.replace('combined_hdr_catalog.txt','xsc_nan_phot.csv') with open(outpath,'w') as f: w = csv.writer(f) w.writerow(da.dtype.names) w.writerows(da[bad].tolist()) print('\ncreated file: {}'.format(outpath)) phot = phot[~bad] da = da[~bad] # read in pipeline catalog print("\nreading: {}".format(cat)) names = open(cat).readline().split()[1:] c = np.recfromtxt(cat, names=names) # loop through xsc sources and find matches in pipeline catalog print("\nfinding records associated with XSC sources in pipeline catalog") c_flux_total = [] n_in_aper = [] c_idx = [] coords = radec_to_coords(c.ra, c.dec) kdt = KDT(coords) for i in range(phot.size): radius = da.r_ext[i]/3600. # idx1, idx2, ds = spherematch(da.ra[i], da.dec[i], # c.ra, c.dec, tolerance=radius) idx, ds = spherematch2(da.ra[i], da.dec[i], c.ra, c.dec, kdt, tolerance=radius, k=500) # c_flux_total.append(c.flux[idx2].sum()) # n_in_aper.append(c.flux[idx2].size) # c_idx.append(idx2.tolist()) c_flux_total.append(c.flux[idx].sum()) n_in_aper.append(ds.size) c_idx.append(idx.tolist()) print("\nhistogram of source counts in r_ext aperture") for i in [(i,n_in_aper.count(i)) for i in set(n_in_aper)]: print i # create new version of catalog file with xsc-associated entries replaced c_idx = np.array(flatten(c_idx)) print("\nremoving {}, adding {}".format(c_idx.size, phot.size)) replaced[cat] = {'old':c_idx.size, 'new':phot.size} replaced[cat]['hist'] = [(i,n_in_aper.count(i)) for i in set(n_in_aper)] c = np.delete(c, c_idx) newrows = np.rec.array([(-i, da.ra[i], da.dec[i], phot.flux[i], phot.unc[i], 1) for i in \ range(phot.size)], dtype=c.dtype) newcat = np.hstack((c, newrows)) # write new version of catalog to disk fmt = ['%i']+['%0.8f']*2+['%.4e']*2+['%i'] outpath = cat.replace('catalog.txt', 'catalog_xsc_cor.txt') np.savetxt(outpath, newcat, fmt = fmt, header = ' '.join(names)) print('\ncreated file: {}'.format(outpath)) # make plot of total old vs. new flux plt.scatter(c_flux_total, phot.flux) ylim = plt.gca().get_ylim() plt.xlim(*ylim) max_y = ylim[1] plt.plot(ylim, ylim, 'r-') plt.xlabel('old flux [mJy]') plt.ylabel('new flux [mJy]') name = ' '.join(cat.split('/')[-1].split('_')[:2]) plt.title(name) outpath = cat.replace('combined_hdr_catalog.txt','xsc_new_vs_old_phot.png') plt.savefig(outpath, dpi=200) plt.close() print('\ncreated file: {}'.format(outpath)) outfile = 'xsc_replaced.json' json.dump(replaced, open(outfile,'w')) print("\ncreated file: {}".format(outfile)) print("\nremoved / added") for k,v in replaced.iteritems(): print k.split('/')[-1], v['old'], v['new'] m = np.mean([i['old']/float(i['new']) for i in replaced.values()]) print("average ratio: {}".format(m)) print("\nK mag and r_ext of sources with NaN photometry:") for i in find_files(bcdphot_out_path, "*xsc_nan_phot.csv"): reg = i.split('/')[-1] rec = np.recfromcsv(i) bad_id = rec.designation.tolist() bad_k = rec.k_m_k20fe.tolist() bad_r_ext = rec.r_ext.tolist() print reg print ("\tid\t\t\tKmag\tr_ext") if type(bad_id) is list: seq = sorted(zip(bad_id, bad_k, bad_r_ext), key=lambda x: x[0]) for j,k,l in seq: print("\t{}\t{}\t{}".format(j,k,l)) else: print("\t{}\t{}\t{}".format(bad_id, bad_k, bad_r_ext))
# Plot subplot(211) plt.plot(C * rho, linewidth=3) for i, e in enumerate(epi): plt.plot(range(e[0], e[0] + len(prI[i])), prI[i], color=colours[1], linewidth=3) # axvline(len(te), color="r", linewidth=2) # te = np.append(te, range(len(e)-1)) title("Per-Epidemic Long Predictions") legend(["Observed", "Predicted"], loc=2) subplot(212) plt.scatter(actualsizes, predictedsizes, s=20 * np.array(range(len(actualsizes))), c=(t[starts[:-1]] % periodicity)) plt.plot(actualsizes, sizeline.predict(actualsizes), linewidth=3) plt.gray() title( "Predicted vs Actual Epidemic Sizes. Lighter = Later in Year; Bigger = Later in Time" ) xlabel("Actual Size") ylabel("Predicted Size") legend([ r"R$^2$ = " + str( sizeline.score(actualsizes.reshape(len(actualsizes), 1), predictedsizes)) ], loc=0) xlim([0, 1.05 * np.max(actualsizes)])
# Plot subplot(211) plt.plot(C*rho, linewidth=3) for i, e in enumerate(epi) : plt.plot(range(e[0], e[0] + len(prI[i])) , prI[i], color=colours[1], linewidth=3) # axvline(len(te), color="r", linewidth=2) # te = np.append(te, range(len(e)-1)) title("Per-Epidemic Long Predictions") legend(["Observed", "Predicted"], loc=2) subplot(212) plt.scatter(actualsizes, predictedsizes, s=20*np.array(range(len(actualsizes))), c=(t[starts[:-1]] % periodicity)) plt.plot(actualsizes, sizeline.predict(actualsizes), linewidth=3) plt.gray() title("Predicted vs Actual Epidemic Sizes. Lighter = Later in Year; Bigger = Later in Time") xlabel("Actual Size") ylabel("Predicted Size") legend([r"R$^2$ = " + str(sizeline.score(actualsizes.reshape(len(actualsizes),1), predictedsizes))], loc=0) xlim([0, 1.05*np.max(actualsizes)]) tight_layout() # <markdowncell> # **Fraction Infected per Epidemic** # # The fraction of the total susceptibles that get infected during an epidemic can be used as a diagnostic tool against certain numerical issues. This quantity should be between zero and one, and is defined for epidemic $i$ as :
def run_xsc_phot(bcdphot_out_path, mosaic_path): replaced = {} for cat in find_files(bcdphot_out_path, "*_combined_hdr_catalog.txt"): print("\n======================================================") print("\nadjusting photometry in: {}".format(cat.split('/')[-1])) print("------------------------------------------------------") outpath = cat.replace('combined_hdr_catalog.txt', '2mass_xsc.tbl') # retrieve 2mass data if file doesn't already exist (from previous run) if not os.path.isfile(outpath): # get url and retrieve data url = query_2mass_xsc_polygon(*get_region_corners(cat)) print("\ndownloading 2MASS photometry from: {}".format(url)) text = urllib2.urlopen(url).read() # write to disk with open(outpath, 'w') as f: f.write(text) print("\ncreated file: {}".format(outpath)) # read back in as recarray print("\nreading: {}".format(outpath)) names = open(outpath).read().split('\n')[76].split('|')[1:-1] da = np.recfromtxt(outpath, skip_header=80, names=names) # write input file for xsc_phot.pro infile_outpath = '/'.join(cat.split('/')[:-1]) + '/xsc.txt' with open(infile_outpath, 'w') as w: for i in range(da.shape[0]): w.write("{} {} {} {}\n".format(da.designation[i], da.ra[i], da.dec[i], da.r_ext[i])) print( "\ncreated input file for xsc_phot.pro: {}".format(infile_outpath)) # locate the FITS mosaic file for xsc_phot.pro to do photometry on reg, ch = cat.split('/')[-1].split('_')[:2] mosaicfile = filter(lambda x: 'dirbe{}/ch{}/long/full/Combine'\ .format(reg,ch) in x, find_files(mosaic_path, '*mosaic.fits'))[0] print("\nfound mosaic file: {}".format(mosaicfile)) # spawn IDL subprocess running xsc_phot.pro and catch stdout in file outpath = infile_outpath.replace('xsc.txt', 'xsc_phot_out.txt') if not os.path.isfile(outpath): outfile = open(outpath, 'w') print("\nspawning xsc_phot.pro IDL subprocess") cmd = "xsc_phot,'" + mosaicfile + "','" + infile_outpath + "','long'" rc = subprocess.call( ['/usr/local/itt/idl71/bin/idl', '-quiet', '-e', cmd], stderr=subprocess.PIPE, stdout=outfile) outfile.close() # read in output to recarray print("\nreading: {}".format(outpath)) phot = np.recfromtxt(outpath, names=['id', 'flux', 'unc', 'sky', 'skyunc']) # make sure rows are aligned assert (da.designation == phot.id).all() # ignore xsc sources we got a NaN or negative flux for bad = np.isnan(phot.flux) | (phot.flux < 0) print("\naper.pro returned NaN or negative flux for {} sources".format( bad.sum())) if bad.sum() > 0: for i in phot[bad].id: print(i) outpath = cat.replace('combined_hdr_catalog.txt', 'xsc_nan_phot.csv') with open(outpath, 'w') as f: w = csv.writer(f) w.writerow(da.dtype.names) w.writerows(da[bad].tolist()) print('\ncreated file: {}'.format(outpath)) phot = phot[~bad] da = da[~bad] # read in pipeline catalog print("\nreading: {}".format(cat)) names = open(cat).readline().split()[1:] c = np.recfromtxt(cat, names=names) # loop through xsc sources and find matches in pipeline catalog print( "\nfinding records associated with XSC sources in pipeline catalog" ) c_flux_total = [] n_in_aper = [] c_idx = [] coords = radec_to_coords(c.ra, c.dec) kdt = KDT(coords) for i in range(phot.size): radius = da.r_ext[i] / 3600. # idx1, idx2, ds = spherematch(da.ra[i], da.dec[i], # c.ra, c.dec, tolerance=radius) idx, ds = spherematch2(da.ra[i], da.dec[i], c.ra, c.dec, kdt, tolerance=radius, k=500) # c_flux_total.append(c.flux[idx2].sum()) # n_in_aper.append(c.flux[idx2].size) # c_idx.append(idx2.tolist()) c_flux_total.append(c.flux[idx].sum()) n_in_aper.append(ds.size) c_idx.append(idx.tolist()) print("\nhistogram of source counts in r_ext aperture") for i in [(i, n_in_aper.count(i)) for i in set(n_in_aper)]: print i # create new version of catalog file with xsc-associated entries replaced c_idx = np.array(flatten(c_idx)) print("\nremoving {}, adding {}".format(c_idx.size, phot.size)) replaced[cat] = {'old': c_idx.size, 'new': phot.size} replaced[cat]['hist'] = [(i, n_in_aper.count(i)) for i in set(n_in_aper)] c = np.delete(c, c_idx) newrows = np.rec.array([(-i, da.ra[i], da.dec[i], phot.flux[i], phot.unc[i], 1) for i in \ range(phot.size)], dtype=c.dtype) newcat = np.hstack((c, newrows)) # write new version of catalog to disk fmt = ['%i'] + ['%0.8f'] * 2 + ['%.4e'] * 2 + ['%i'] outpath = cat.replace('catalog.txt', 'catalog_xsc_cor.txt') np.savetxt(outpath, newcat, fmt=fmt, header=' '.join(names)) print('\ncreated file: {}'.format(outpath)) # make plot of total old vs. new flux plt.scatter(c_flux_total, phot.flux) ylim = plt.gca().get_ylim() plt.xlim(*ylim) max_y = ylim[1] plt.plot(ylim, ylim, 'r-') plt.xlabel('old flux [mJy]') plt.ylabel('new flux [mJy]') name = ' '.join(cat.split('/')[-1].split('_')[:2]) plt.title(name) outpath = cat.replace('combined_hdr_catalog.txt', 'xsc_new_vs_old_phot.png') plt.savefig(outpath, dpi=200) plt.close() print('\ncreated file: {}'.format(outpath)) outfile = 'xsc_replaced.json' json.dump(replaced, open(outfile, 'w')) print("\ncreated file: {}".format(outfile)) print("\nremoved / added") for k, v in replaced.iteritems(): print k.split('/')[-1], v['old'], v['new'] m = np.mean([i['old'] / float(i['new']) for i in replaced.values()]) print("average ratio: {}".format(m)) print("\nK mag and r_ext of sources with NaN photometry:") for i in find_files(bcdphot_out_path, "*xsc_nan_phot.csv"): reg = i.split('/')[-1] rec = np.recfromcsv(i) bad_id = rec.designation.tolist() bad_k = rec.k_m_k20fe.tolist() bad_r_ext = rec.r_ext.tolist() print reg print("\tid\t\t\tKmag\tr_ext") if type(bad_id) is list: seq = sorted(zip(bad_id, bad_k, bad_r_ext), key=lambda x: x[0]) for j, k, l in seq: print("\t{}\t{}\t{}".format(j, k, l)) else: print("\t{}\t{}\t{}".format(bad_id, bad_k, bad_r_ext))