def plot_wise(cat_path): for catfile in find_files(cat_path, "*merged+wise.csv"): print("\nreading catalog: {}".format(catfile)) df = pd.read_csv(catfile) # convert to magnitudes nbadflux = (df.flux <= 0).sum() try: assert nbadflux == 0 except: print("warning: {} negative flux source(s)".format(nbadflux)) ch = catfile.split('/')[-1].split('_')[1] mags = spz_jy_to_mags(df.flux*1e-3, float(ch)) if ch == '1': plt.scatter(df.W1mag, mags) plt.xlabel('W1 [mag]') plt.ylabel('I1 [mag]') elif ch == '2': plt.scatter(df.W2mag, mags) plt.xlabel('W2 [mag]') plt.ylabel('I2 [mag]') ax = plt.gca() xlim, ylim = ax.get_xlim(), ax.get_ylim() plt.plot([-5, ylim[1]*2], [-5, ylim[1]*2], 'r-') ax.set_xlim(xlim) ; ax.set_ylim(ylim) reg = catfile.split('/')[-1].split('_')[0] name = '{}_{}_IRAC_vs_WISE.png'.format(reg, ch) outpath = '/'.join(catfile.split('/')[:-1]+[name]) plt.savefig(outpath, dpi=120) plt.close()
def plot(x, y, outpath, xlabel, ylabel, plot_style, plot_type): if plot_type == 'mag-mag': xlim = (10, 18) ylim = (10, 18) elif plot_type == 'color-mag': xlim = (10, 18) ylim = (-1, 1) elif plot_type == 'color-color': xlim = (-1, 1) ylim = (-1, 1) else: raise(ValueError("plot_type should be one of ['mag-mag', 'color-mag', 'color-color'] ")) isinrange = lambda a,b: (a>=b[0]) & (a<=b[1]) g = isinrange(x, xlim) & isinrange(y, ylim) if plot_style == 'scatter': plt.scatter(x[g], y[g]) elif plot_style == 'hexbin': plt.hexbin(x[g], y[g]) elif plot_style == 'hist2d': plt.hist2d(x[g], y[g], bins=100) else: raise(ValueError("plot_style should be one of ['scatter', 'hexbin', 'hist2d'] ")) plt.xlabel(xlabel) plt.ylabel(ylabel) ax = plt.gca() ax.set_xlim(xlim) ax.set_ylim(ylim) plt.savefig(outpath, dpi=120) plt.close() print("created file: {}".format(outpath))
def plot_wise(cat_path): for catfile in find_files(cat_path, "*merged+wise.csv"): print("\nreading catalog: {}".format(catfile)) df = pd.read_csv(catfile) # convert to magnitudes nbadflux = (df.flux <= 0).sum() try: assert nbadflux == 0 except: print("warning: {} negative flux source(s)".format(nbadflux)) ch = catfile.split('/')[-1].split('_')[1] mags = spz_jy_to_mags(df.flux * 1e-3, float(ch)) if ch == '1': plt.scatter(df.W1mag, mags) plt.xlabel('W1 [mag]') plt.ylabel('I1 [mag]') elif ch == '2': plt.scatter(df.W2mag, mags) plt.xlabel('W2 [mag]') plt.ylabel('I2 [mag]') ax = plt.gca() xlim, ylim = ax.get_xlim(), ax.get_ylim() plt.plot([-5, ylim[1] * 2], [-5, ylim[1] * 2], 'r-') ax.set_xlim(xlim) ax.set_ylim(ylim) reg = catfile.split('/')[-1].split('_')[0] name = '{}_{}_IRAC_vs_WISE.png'.format(reg, ch) outpath = '/'.join(catfile.split('/')[:-1] + [name]) plt.savefig(outpath, dpi=120) plt.close()
def plot(x, y, outpath, xlabel, ylabel, plot_style, plot_type): if plot_type == 'mag-mag': xlim = (10, 18) ylim = (10, 18) elif plot_type == 'color-mag': xlim = (10, 18) ylim = (-1, 1) elif plot_type == 'color-color': xlim = (-1, 1) ylim = (-1, 1) else: raise (ValueError( "plot_type should be one of ['mag-mag', 'color-mag', 'color-color'] " )) isinrange = lambda a, b: (a >= b[0]) & (a <= b[1]) g = isinrange(x, xlim) & isinrange(y, ylim) if plot_style == 'scatter': plt.scatter(x[g], y[g]) elif plot_style == 'hexbin': plt.hexbin(x[g], y[g]) elif plot_style == 'hist2d': plt.hist2d(x[g], y[g], bins=100) else: raise (ValueError( "plot_style should be one of ['scatter', 'hexbin', 'hist2d'] ")) plt.xlabel(xlabel) plt.ylabel(ylabel) ax = plt.gca() ax.set_xlim(xlim) ax.set_ylim(ylim) plt.savefig(outpath, dpi=120) plt.close() print("created file: {}".format(outpath))
def plot_learning_curves(num_points, X_train, Y_train, X_test, Y_test, positive_class=1, negative_class=0): train_set_sizes = [len(X_train) / k for k in range(num_points + 1, 0, -1)] test_errors = [] training_errors = [] for training_set_size in train_set_sizes: model = train(X_train, Y_train, training_set_size) test_error = evaluate(model, X_test, Y_test, positive_class, negative_class) training_error = evaluate(model, X_train, Y_train, positive_class, negative_class) test_errors.append(test_error) training_errors.append(training_error) plt.plot(train_set_sizes, training_errors, 'bs-', label='Training accuracy') plt.plot(train_set_sizes, test_errors, 'g^-', label='Test accuracy') plt.ylabel('Accuracy') plt.xlabel('Number of training samples') plt.title('Augmented Logistic Regression Learning Curve') plt.legend(loc='lower right') plt.savefig('../Figures/accuracyPlotAugmented.png', dpi=100) pylab.show()
def plot_days(days): nplots = len(days) fig, subplots = plt.subplots(nplots, figsize=(10, 50)) for i, day in enumerate(days): subplot = subplots[i] plot_day(day, subplot) plt.tight_layout() plt.savefig('./plots/best.png')
def scipy_fcluster_merge_strategy(assignments): # Z-matrix helpers def groups(assignments): cluster_map = {} for idx, gid in enumerate(assignments): lst = cluster_map.get(gid, []) lst.append(idx) cluster_map[gid] = lst return tuple(sorted(map(tuple, cluster_map.values()), key=len, reverse=True)) def zmatrix(assignments_samples): n = len(assignments_samples[0]) # should make sparse matrix zmat = np.zeros((n, n), dtype=np.float32) for assignments in assignments_samples: clusters = groups(assignments) for cluster in clusters: for i, j in it.product(cluster, repeat=2): zmat[i, j] += 1 zmat /= float(len(assignments_samples)) return zmat def reorder_zmat(zmat, order): zmat = zmat[order] zmat = zmat[:,order] return zmat def linkage(zmat): assert zmat.shape[0] == zmat.shape[1] zmat0 = np.array(zmat[np.triu_indices(zmat.shape[0], k=1)]) zmat0 = 1. - zmat0 return scipy.cluster.hierarchy.linkage(zmat0) def ordering(l): return np.array(scipy.cluster.hierarchy.leaves_list(l)) zmat = zmatrix(assignments) li = linkage(zmat) # diagnostic: draw z-matrix indices = ordering(li) plt.imshow(reorder_zmat(zmat, indices), cmap=plt.cm.binary, interpolation='nearest') plt.savefig("zmat-before.pdf") plt.close() fassignment = scipy.cluster.hierarchy.fcluster(li, 0.001) clustering = groups(fassignment) sorted_ordering = list(it.chain.from_iterable(clustering)) # draw post fcluster() ordered z-matrix plt.imshow(reorder_zmat(zmat, sorted_ordering), cmap=plt.cm.binary, interpolation='nearest') plt.savefig("zmat-after.pdf") plt.close() return fassignment
def plot_sdss(cat_path): for catfile in find_files(cat_path, "*merged+sdss.txt"): # for now ignore the channel 2 files if catfile.split('/')[-1].split('_')[1] != '1': continue print("\nreading catalog: {}".format(catfile)) df = pd.read_table(catfile, sep=' ') # get rid of negative flux sources, if any df = df[df.flux > 0] # convert to magnitudes mags = spz_jy_to_mags(df.flux * 1e-3, 1) # print counts per magnitude bin for i in range(10, 15): sc = ((df.cl == 3) & (mags > i) & (mags < i + 1)).sum() xc = ((df.xsc == 1) & (mags > i) & (mags < i + 1)).sum() msg = "{}th to {}th mag: {} SDSS galaxy sources, {} 2MASS XSC sources" print(msg.format(i, i + 1, sc, xc)) # print number of sources agreed upon agree = ((df.xsc == 1) & (df.cl == 3)).sum() disagree = ((df.xsc == 1) & (df.cl == 6)).sum() na = ((df.xsc == 1) & (df.cl == 0)).sum() msg = "{} 2MASS XSC sources classified as galaxies by SDSS" print(msg.format(agree)) msg = "{} 2MASS XSC sources classified as stars by SDSS" print(msg.format(disagree)) msg = "{} 2MASS XSC sources not matched to SDSS" print(msg.format(na)) # plot normed histograms of 2MASS XSC and SDSS galaxy magnitudes xsc_gals = (mags > 10) & (mags < 15) & (df.xsc == 1) sdss_gals = (mags > 10) & (mags < 15) & (df.cl == 3) # mags[xsc_gals].hist(label='2MASS XSC', normed=True) # mags[sdss_gals].hist(label='SDSS galaxies', normed=True) plt.hist([mags[xsc_gals].values, mags[sdss_gals].values], bins=5, label=['2MASS', 'SDSS']) plt.xlabel('IRAC1 [mag]') plt.ylabel('Number Count') reg = catfile.split('/')[-1].split('_')[0] plt.title('{} Extended Sources / Galaxies'.format(reg)) plt.legend(loc=2) name = '{}_2mass_xsc_vs_sdss_hist.png'.format(reg) outpath = '/'.join(catfile.split('/')[:-1] + [name]) plt.savefig(outpath, dpi=100) plt.close() print("created file: {}".format(outpath))
def plot_sdss(cat_path): for catfile in find_files(cat_path, "*merged+sdss.txt"): # for now ignore the channel 2 files if catfile.split('/')[-1].split('_')[1] != '1': continue print("\nreading catalog: {}".format(catfile)) df = pd.read_table(catfile, sep=' ') # get rid of negative flux sources, if any df = df[df.flux > 0] # convert to magnitudes mags = spz_jy_to_mags(df.flux*1e-3, 1) # print counts per magnitude bin for i in range(10,15): sc = ((df.cl == 3) & (mags > i) & (mags < i+1)).sum() xc = ((df.xsc == 1) & (mags > i) & (mags < i+1)).sum() msg = "{}th to {}th mag: {} SDSS galaxy sources, {} 2MASS XSC sources" print(msg.format(i, i+1, sc, xc)) # print number of sources agreed upon agree = ((df.xsc == 1) & (df.cl == 3)).sum() disagree = ((df.xsc == 1) & (df.cl == 6)).sum() na = ((df.xsc == 1) & (df.cl == 0)).sum() msg = "{} 2MASS XSC sources classified as galaxies by SDSS" print(msg.format(agree)) msg = "{} 2MASS XSC sources classified as stars by SDSS" print(msg.format(disagree)) msg = "{} 2MASS XSC sources not matched to SDSS" print(msg.format(na)) # plot normed histograms of 2MASS XSC and SDSS galaxy magnitudes xsc_gals = (mags > 10) & (mags < 15) & (df.xsc == 1) sdss_gals = (mags > 10) & (mags < 15) & (df.cl == 3) # mags[xsc_gals].hist(label='2MASS XSC', normed=True) # mags[sdss_gals].hist(label='SDSS galaxies', normed=True) plt.hist([mags[xsc_gals].values, mags[sdss_gals].values], bins=5, label=['2MASS', 'SDSS']) plt.xlabel('IRAC1 [mag]') plt.ylabel('Number Count') reg = catfile.split('/')[-1].split('_')[0] plt.title('{} Extended Sources / Galaxies'.format(reg)) plt.legend(loc=2) name = '{}_2mass_xsc_vs_sdss_hist.png'.format(reg) outpath = '/'.join(catfile.split('/')[:-1]+[name]) plt.savefig(outpath, dpi=100) plt.close() print("created file: {}".format(outpath))
def plot_signal(signal, shape=None, name=None): """ Plot a numeric signal as an image. """ s = np.copy(signal) if s.max() != 0.: s *= 255.0/s.max() s = s.astype(int) if shape: s = s.reshape(shape) plt.figure() plt.imshow(s, cmap=plt.cm.gray, interpolation='nearest') if name: plt.savefig(name) plt.close()
def histogram(original, updated, bins=None, main="", save=None, log=False): """Plot a histogram of score improvements (updated-origianl) Input: original - list of original scores updated - list of updates scores in same order as original bins - number of bins to represent improvements """ #Lengths of score lists must be identical, assume in same order assert len(original) == len(original) #Set up bins: if bins is not None and bins > 0: imoprovements = {(-1,-1):0} for i in xrange(0, len(original), bins): improvements[(0,i+bins)] = 0 else: improvements = {(-1,-1):0, (-5,0):0, (0,1):0, (1,25):0, (25,50):0, (50,75):0, (75,100):0, (100,125):0, (125,150):0, (150,200):0, (200,300):0, (300,400):0, (500,10000):0} #defaultdict(int) #Calcualte improvements for o, u in izip(original, updated): if o>u: improvements[(-1,-1)] += 1 continue for lower, upper in improvements: if lower <= int(u-o) < upper: improvements[(lower,upper)] += 1 break keys = sorted(improvements.keys(), key=lambda x:x[0]) values = [improvements[r] for r in keys] fig, ax = plt.subplots() ax.set_title(main) ax.set_xlabel("Improvement (updated-original) bitscores") ax.set_ylabel("log(Frequency)") #ax.set_yscale('log') width = 1.0 #ax.set_xticks(np.arange(len(improvements))) #ax.set_xticklabels([l for l, u in keys]) bar(ax, np.arange(len(improvements)), values, log=log, annotate=True, grid='y', xticklabels=[l for l, u in keys]) if save is None: plt.show() else: plt.savefig(save)
def layerActivations(model, data, labels): print('Visualizing activations with tSNE...') if not os.path.exists(cc.cfg['plots']['layer_activations_dir']): os.makedirs(cc.cfg['plots']['layer_activations_dir']) numLabels = cc.cfg['plots']['layer_activations_label_cap'] data = data[:cc.cfg['plots']['layer_activations_points_cap']] labels = labels[:numLabels,:cc.cfg['plots']['layer_activations_points_cap']] subplotCols = numLabels subplotRows = len(model.layers)-1 subplotIdx = 1 plt.figure(figsize=(5*subplotCols,5*subplotRows)) for i in range(1,len(model.layers)): print('Running tSNE for layer {}/{}'.format(i+1,len(model.layers))) func = K.function([model.layers[0].input], [model.layers[i].output]) out = func([data])[0] tsneModel = TSNE(n_components = 2, random_state = 0) tsneOut = tsneModel.fit_transform(out).T # labeledTsneOut = np.hstack((tsneOut, labels[0].reshape(-1,1))) for j in range(numLabels): plt.subplot(subplotRows, subplotCols, subplotIdx) plt.title('{} / {}'.format(model.layers[i].name,cc.exp['params']['data']['labels'][j])) plt.scatter(tsneOut[0],tsneOut[1],c=labels[j],cmap = 'plasma') subplotIdx += 1 # tsneDF = pd.DataFrame(labeledTsneOut, columns = ('a', 'b', 'c')) # plot = tsneDF.plot.scatter(x = 'a', y = 'b', c = 'c', cmap = 'plasma') plt.tight_layout() plt.savefig('{}/activations.png'.format(cc.cfg['plots']['layer_activations_dir'])) plt.close() print('...done')
def histograms(modelLogger): if not cc.cfg['plots']['histograms']: return if not os.path.exists(cc.cfg['plots']['histograms_dir']): os.makedirs(cc.cfg['plots']['histograms_dir']) cntEpochs = len(modelLogger.epochLogs) cntLayers = len(modelLogger.epochLogs[-1]['weights']) logVals = [ {'name':'weights','color':'blue'}, {'name':'updates','color':'red'}, {'name':'ratios','color':'green'} ] subplotRows = len(logVals) subplotCols = cntLayers for x in range(cntEpochs): subplotIdx = 1 plt.figure(figsize=(5*subplotCols,5*subplotRows)) plt.suptitle('Histograms per layer, epoch {}/{}'.format(x,cntEpochs-1), fontsize=14) for logVal in logVals: for i,layer in enumerate(modelLogger.epochLogs[x][logVal['name']]): histmin = layer.min() histmax = layer.max() plt.subplot(subplotRows, subplotCols, subplotIdx) plt.title('{}, {}'.format(modelLogger.model.layers[modelLogger.loggedLayers[i]].name,logVal['name'])) plt.hist(layer, range = (histmin, histmax), bins = 30, color = logVal['color']) subplotIdx+=1 plt.tight_layout() plt.subplots_adjust(top=0.9) plt.savefig('{}/hist_{e:03d}.png'.format(cc.cfg['plots']['histograms_dir'], e = x)) plt.close()
def run_xsc_phot(bcdphot_out_path, mosaic_path): replaced = {} for cat in find_files(bcdphot_out_path, "*_combined_hdr_catalog.txt"): print("\n======================================================") print("\nadjusting photometry in: {}".format(cat.split('/')[-1])) print("------------------------------------------------------") outpath = cat.replace('combined_hdr_catalog.txt', '2mass_xsc.tbl') # retrieve 2mass data if file doesn't already exist (from previous run) if not os.path.isfile(outpath): # get url and retrieve data url = query_2mass_xsc_polygon(*get_region_corners(cat)) print("\ndownloading 2MASS photometry from: {}".format(url)) text = urllib2.urlopen(url).read() # write to disk with open(outpath, 'w') as f: f.write(text) print("\ncreated file: {}".format(outpath)) # read back in as recarray print("\nreading: {}".format(outpath)) names = open(outpath).read().split('\n')[76].split('|')[1:-1] da = np.recfromtxt(outpath, skip_header=80, names=names) # write input file for xsc_phot.pro infile_outpath = '/'.join(cat.split('/')[:-1]) + '/xsc.txt' with open(infile_outpath, 'w') as w: for i in range(da.shape[0]): w.write("{} {} {} {}\n".format(da.designation[i], da.ra[i], da.dec[i], da.r_ext[i])) print( "\ncreated input file for xsc_phot.pro: {}".format(infile_outpath)) # locate the FITS mosaic file for xsc_phot.pro to do photometry on reg, ch = cat.split('/')[-1].split('_')[:2] mosaicfile = filter(lambda x: 'dirbe{}/ch{}/long/full/Combine'\ .format(reg,ch) in x, find_files(mosaic_path, '*mosaic.fits'))[0] print("\nfound mosaic file: {}".format(mosaicfile)) # spawn IDL subprocess running xsc_phot.pro and catch stdout in file outpath = infile_outpath.replace('xsc.txt', 'xsc_phot_out.txt') if not os.path.isfile(outpath): outfile = open(outpath, 'w') print("\nspawning xsc_phot.pro IDL subprocess") cmd = "xsc_phot,'" + mosaicfile + "','" + infile_outpath + "','long'" rc = subprocess.call( ['/usr/local/itt/idl71/bin/idl', '-quiet', '-e', cmd], stderr=subprocess.PIPE, stdout=outfile) outfile.close() # read in output to recarray print("\nreading: {}".format(outpath)) phot = np.recfromtxt(outpath, names=['id', 'flux', 'unc', 'sky', 'skyunc']) # make sure rows are aligned assert (da.designation == phot.id).all() # ignore xsc sources we got a NaN or negative flux for bad = np.isnan(phot.flux) | (phot.flux < 0) print("\naper.pro returned NaN or negative flux for {} sources".format( bad.sum())) if bad.sum() > 0: for i in phot[bad].id: print(i) outpath = cat.replace('combined_hdr_catalog.txt', 'xsc_nan_phot.csv') with open(outpath, 'w') as f: w = csv.writer(f) w.writerow(da.dtype.names) w.writerows(da[bad].tolist()) print('\ncreated file: {}'.format(outpath)) phot = phot[~bad] da = da[~bad] # read in pipeline catalog print("\nreading: {}".format(cat)) names = open(cat).readline().split()[1:] c = np.recfromtxt(cat, names=names) # loop through xsc sources and find matches in pipeline catalog print( "\nfinding records associated with XSC sources in pipeline catalog" ) c_flux_total = [] n_in_aper = [] c_idx = [] coords = radec_to_coords(c.ra, c.dec) kdt = KDT(coords) for i in range(phot.size): radius = da.r_ext[i] / 3600. # idx1, idx2, ds = spherematch(da.ra[i], da.dec[i], # c.ra, c.dec, tolerance=radius) idx, ds = spherematch2(da.ra[i], da.dec[i], c.ra, c.dec, kdt, tolerance=radius, k=500) # c_flux_total.append(c.flux[idx2].sum()) # n_in_aper.append(c.flux[idx2].size) # c_idx.append(idx2.tolist()) c_flux_total.append(c.flux[idx].sum()) n_in_aper.append(ds.size) c_idx.append(idx.tolist()) print("\nhistogram of source counts in r_ext aperture") for i in [(i, n_in_aper.count(i)) for i in set(n_in_aper)]: print i # create new version of catalog file with xsc-associated entries replaced c_idx = np.array(flatten(c_idx)) print("\nremoving {}, adding {}".format(c_idx.size, phot.size)) replaced[cat] = {'old': c_idx.size, 'new': phot.size} replaced[cat]['hist'] = [(i, n_in_aper.count(i)) for i in set(n_in_aper)] c = np.delete(c, c_idx) newrows = np.rec.array([(-i, da.ra[i], da.dec[i], phot.flux[i], phot.unc[i], 1) for i in \ range(phot.size)], dtype=c.dtype) newcat = np.hstack((c, newrows)) # write new version of catalog to disk fmt = ['%i'] + ['%0.8f'] * 2 + ['%.4e'] * 2 + ['%i'] outpath = cat.replace('catalog.txt', 'catalog_xsc_cor.txt') np.savetxt(outpath, newcat, fmt=fmt, header=' '.join(names)) print('\ncreated file: {}'.format(outpath)) # make plot of total old vs. new flux plt.scatter(c_flux_total, phot.flux) ylim = plt.gca().get_ylim() plt.xlim(*ylim) max_y = ylim[1] plt.plot(ylim, ylim, 'r-') plt.xlabel('old flux [mJy]') plt.ylabel('new flux [mJy]') name = ' '.join(cat.split('/')[-1].split('_')[:2]) plt.title(name) outpath = cat.replace('combined_hdr_catalog.txt', 'xsc_new_vs_old_phot.png') plt.savefig(outpath, dpi=200) plt.close() print('\ncreated file: {}'.format(outpath)) outfile = 'xsc_replaced.json' json.dump(replaced, open(outfile, 'w')) print("\ncreated file: {}".format(outfile)) print("\nremoved / added") for k, v in replaced.iteritems(): print k.split('/')[-1], v['old'], v['new'] m = np.mean([i['old'] / float(i['new']) for i in replaced.values()]) print("average ratio: {}".format(m)) print("\nK mag and r_ext of sources with NaN photometry:") for i in find_files(bcdphot_out_path, "*xsc_nan_phot.csv"): reg = i.split('/')[-1] rec = np.recfromcsv(i) bad_id = rec.designation.tolist() bad_k = rec.k_m_k20fe.tolist() bad_r_ext = rec.r_ext.tolist() print reg print("\tid\t\t\tKmag\tr_ext") if type(bad_id) is list: seq = sorted(zip(bad_id, bad_k, bad_r_ext), key=lambda x: x[0]) for j, k, l in seq: print("\t{}\t{}\t{}".format(j, k, l)) else: print("\t{}\t{}\t{}".format(bad_id, bad_k, bad_r_ext))
# For all the spines, make their line thicker and return them to be black all_spines = ['top', 'left', 'bottom', 'right'] for spine in all_spines: ax.spines[spine].set_linewidth(0.5) #ax.spines[spine].set_color('black') # Get back the ticks. The position of the numbers is informative enough of # the position of the value. ax.xaxis.set_ticks_position('both') ax.yaxis.set_ticks_position('both') ax.tick_params('both', length=15, width=1, which='major') ax.tick_params('both', length=5, width=1, which='minor') plt.xlabel("SDSS "+band+" magnitude, mag") plt.ylabel("GC "+band+" magnitude, mag") plt.savefig('test/'+band+'_SDSS_vs_GC') #print band, "mean magnitude difference: ", np.mean(petroMags[:, i]) - np.mean(mags[:, i]) fig = plt.figure(figsize=(14, 14)) for i, band in enumerate(bands): lim_lo = limits[i][0] lim_hi = limits[i][1] x, m, b = makeDiagonalLine([lim_lo, lim_hi]) print i, lim_lo, lim_hi ax = plt.subplot(5, 1, i+1) c = ax.scatter(petroMags[:, i], mags[:, i], c='k', s=8, edgecolor='k') ax.axis([lim_lo, lim_hi, lim_lo, lim_hi]) ax.errorbar(petroMags[:, i], mags[:, i], yerr=mag_err[:, i], mew=0, linestyle = "none", color="black") plt.plot(x,m*x + b, c='k') ax.xaxis.set_major_locator(majorLocator)
eps = data[0] delta = 0.1 string = [r'$\tau\delta$ = %.2f' % float(delta*t) for t in taus] #string2 = [r'$\tau$ = '+str(data[3][i]) for i in range(5)] maxmmse = np.max(eps) #maxf = np.max(data[1]) def f(x): ppl.plot(alphas,eps[:,x]/maxmmse,label=string[x],ax=ax1) # ax2.plot(data[2],data[1][:,x]/maxf,label=string2[x]) #plt.close() #plt.figure() #ax1 = plt.gcf().add_subplot(1,1,1) fig, ax1 = ppl.subplots(1) plt.title(r'MMSE for a Reconstruction Task with Adaptive Neurons') #ax2 = plt.gcf().add_subplot(1,2,2) #plt.title(r'Firing rate of adaptive code') map(f,range(0,10,2)) ppl.legend(ax1) #ax2.legend() ax1.set_xlabel(r'$\alpha$') ax1.set_ylabel(r'MMSE') #ax2.set_xlabel(r'$\alpha$') #ax2.set_ylabel(r'Firing Rate') #plt.xlabel(r'Tuning width $\alpha$') #plt.ylabel(r'MMSE') plt.show() #plt.savefig("mmse_discrimination.png",dpi=600) plt.savefig("figure_5_8.png",dpi=600) plt.savefig("figure_5_8.eps")
def run_xsc_phot(bcdphot_out_path, mosaic_path): replaced = {} for cat in find_files(bcdphot_out_path, "*_combined_hdr_catalog.txt"): print("\n======================================================") print("\nadjusting photometry in: {}".format(cat.split('/')[-1])) print("------------------------------------------------------") outpath = cat.replace('combined_hdr_catalog.txt','2mass_xsc.tbl') # retrieve 2mass data if file doesn't already exist (from previous run) if not os.path.isfile(outpath): # get url and retrieve data url = query_2mass_xsc_polygon(*get_region_corners(cat)) print("\ndownloading 2MASS photometry from: {}".format(url)) text = urllib2.urlopen(url).read() # write to disk with open(outpath, 'w') as f: f.write(text) print("\ncreated file: {}".format(outpath)) # read back in as recarray print("\nreading: {}".format(outpath)) names = open(outpath).read().split('\n')[76].split('|')[1:-1] da = np.recfromtxt(outpath, skip_header=80, names=names) # write input file for xsc_phot.pro infile_outpath = '/'.join(cat.split('/')[:-1])+'/xsc.txt' with open(infile_outpath,'w') as w: for i in range(da.shape[0]): w.write("{} {} {} {}\n".format(da.designation[i], da.ra[i], da.dec[i], da.r_ext[i])) print("\ncreated input file for xsc_phot.pro: {}".format(infile_outpath)) # locate the FITS mosaic file for xsc_phot.pro to do photometry on reg, ch = cat.split('/')[-1].split('_')[:2] mosaicfile = filter(lambda x: 'dirbe{}/ch{}/long/full/Combine'\ .format(reg,ch) in x, find_files(mosaic_path, '*mosaic.fits'))[0] print("\nfound mosaic file: {}".format(mosaicfile)) # spawn IDL subprocess running xsc_phot.pro and catch stdout in file outpath = infile_outpath.replace('xsc.txt', 'xsc_phot_out.txt') if not os.path.isfile(outpath): outfile = open(outpath,'w') print("\nspawning xsc_phot.pro IDL subprocess") cmd = "xsc_phot,'"+mosaicfile+"','"+infile_outpath+"','long'" rc = subprocess.call(['/usr/local/itt/idl71/bin/idl','-quiet','-e',cmd], stderr = subprocess.PIPE, stdout = outfile) outfile.close() # read in output to recarray print("\nreading: {}".format(outpath)) phot = np.recfromtxt(outpath, names=['id','flux','unc','sky','skyunc']) # make sure rows are aligned assert (da.designation == phot.id).all() # ignore xsc sources we got a NaN or negative flux for bad = np.isnan(phot.flux) | (phot.flux < 0) print("\naper.pro returned NaN or negative flux for {} sources".format(bad.sum())) if bad.sum() > 0: for i in phot[bad].id: print(i) outpath = cat.replace('combined_hdr_catalog.txt','xsc_nan_phot.csv') with open(outpath,'w') as f: w = csv.writer(f) w.writerow(da.dtype.names) w.writerows(da[bad].tolist()) print('\ncreated file: {}'.format(outpath)) phot = phot[~bad] da = da[~bad] # read in pipeline catalog print("\nreading: {}".format(cat)) names = open(cat).readline().split()[1:] c = np.recfromtxt(cat, names=names) # loop through xsc sources and find matches in pipeline catalog print("\nfinding records associated with XSC sources in pipeline catalog") c_flux_total = [] n_in_aper = [] c_idx = [] coords = radec_to_coords(c.ra, c.dec) kdt = KDT(coords) for i in range(phot.size): radius = da.r_ext[i]/3600. # idx1, idx2, ds = spherematch(da.ra[i], da.dec[i], # c.ra, c.dec, tolerance=radius) idx, ds = spherematch2(da.ra[i], da.dec[i], c.ra, c.dec, kdt, tolerance=radius, k=500) # c_flux_total.append(c.flux[idx2].sum()) # n_in_aper.append(c.flux[idx2].size) # c_idx.append(idx2.tolist()) c_flux_total.append(c.flux[idx].sum()) n_in_aper.append(ds.size) c_idx.append(idx.tolist()) print("\nhistogram of source counts in r_ext aperture") for i in [(i,n_in_aper.count(i)) for i in set(n_in_aper)]: print i # create new version of catalog file with xsc-associated entries replaced c_idx = np.array(flatten(c_idx)) print("\nremoving {}, adding {}".format(c_idx.size, phot.size)) replaced[cat] = {'old':c_idx.size, 'new':phot.size} replaced[cat]['hist'] = [(i,n_in_aper.count(i)) for i in set(n_in_aper)] c = np.delete(c, c_idx) newrows = np.rec.array([(-i, da.ra[i], da.dec[i], phot.flux[i], phot.unc[i], 1) for i in \ range(phot.size)], dtype=c.dtype) newcat = np.hstack((c, newrows)) # write new version of catalog to disk fmt = ['%i']+['%0.8f']*2+['%.4e']*2+['%i'] outpath = cat.replace('catalog.txt', 'catalog_xsc_cor.txt') np.savetxt(outpath, newcat, fmt = fmt, header = ' '.join(names)) print('\ncreated file: {}'.format(outpath)) # make plot of total old vs. new flux plt.scatter(c_flux_total, phot.flux) ylim = plt.gca().get_ylim() plt.xlim(*ylim) max_y = ylim[1] plt.plot(ylim, ylim, 'r-') plt.xlabel('old flux [mJy]') plt.ylabel('new flux [mJy]') name = ' '.join(cat.split('/')[-1].split('_')[:2]) plt.title(name) outpath = cat.replace('combined_hdr_catalog.txt','xsc_new_vs_old_phot.png') plt.savefig(outpath, dpi=200) plt.close() print('\ncreated file: {}'.format(outpath)) outfile = 'xsc_replaced.json' json.dump(replaced, open(outfile,'w')) print("\ncreated file: {}".format(outfile)) print("\nremoved / added") for k,v in replaced.iteritems(): print k.split('/')[-1], v['old'], v['new'] m = np.mean([i['old']/float(i['new']) for i in replaced.values()]) print("average ratio: {}".format(m)) print("\nK mag and r_ext of sources with NaN photometry:") for i in find_files(bcdphot_out_path, "*xsc_nan_phot.csv"): reg = i.split('/')[-1] rec = np.recfromcsv(i) bad_id = rec.designation.tolist() bad_k = rec.k_m_k20fe.tolist() bad_r_ext = rec.r_ext.tolist() print reg print ("\tid\t\t\tKmag\tr_ext") if type(bad_id) is list: seq = sorted(zip(bad_id, bad_k, bad_r_ext), key=lambda x: x[0]) for j,k,l in seq: print("\t{}\t{}\t{}".format(j,k,l)) else: print("\t{}\t{}\t{}".format(bad_id, bad_k, bad_r_ext))
[78, 376], [79, 164], [80, 194], [81, 342], [82, 314], [83, 68], [84, 54], [85, 131], [86, 52], [87, 220], [88, 145], [89, 63], [90, 608], [91, 68], [92, 155], [93, 49], [94, 108], [95, 59], [96, 67], [97, 58], [98, 42], [99, 210], [100, 174]]) plt.figure(figsize=(5,5)) plt.scatter(histdata.T[0],histdata.T[1]) plt.savefig('hist.png') plt.close()
sys.stderr.write('{} files contain {} bytes over {} files\n'.format(ftype.upper(), sum(fsize), len(files[ftype]))) ### # To upload to the correct spot on S3 # gzip *.paths # s3cmd put --acl-public *.paths.gz s3://aws-publicdatasets/common-crawl/crawl-data/CC-MAIN-YYYY-WW/ ### # Plot for ftype, fsize in size.items(): if not fsize: continue plt.hist(fsize, bins=50) plt.xlabel('Size (bytes)') plt.ylabel('Count') plt.title('Distribution for {}'.format(ftype.upper())) plt.savefig(prefix + '{}_dist.pdf'.format(ftype)) #plt.show(block=True) plt.close() ### # Find missing WAT / WET files warc = set([x.strip() for x in open(prefix + 'warc.paths').readlines()]) wat = [x.strip() for x in open(prefix + 'wat.paths').readlines()] wat = set([x.replace('.warc.wat.', '.warc.').replace('/wat/', '/warc/') for x in wat]) # Work out the missing files and segments missing = sorted(warc - wat) missing_segments = defaultdict(list) for fn in missing: start, suffix = fn.split('/warc/') segment = start.split('/')[-1] missing_segments[segment].append(fn)
width = 0.5 font = {'family' : 'normal', 'weight' : 'normal', 'size' : 10} # k c b m r o char vals = [ 45174 , 48810 , 41719 , 255017 , 642090 , 3053979 , 3066135 , 1731655 , 582226 , 126045 , 6738 ] total = float(sum(vals)) print(vals) vals = [(i / total)*100 for i in vals] print(vals) colors = ['LightGray','Gray','LightSkyBlue','RoyalBlue','LightSeaGreen','MediumSeaGreen','Khaki',\ 'Goldenrod','Tomato','MediumVioletRed','DarkViolet'] #plt.bar(ind, vals, width) ppl.bar(ax, ind, vals, color=colors) plt.xticks(ind+width/2.,\ ('1e-7','1e-6','1e-5','1e-4','0.001','0.005','0.01', '0.05', '0.1', '0.2', '0.5')\ ,fontsize=11) plt.yticks(fontsize=11) plt.title("Distribution of cells by mutation rate across resistant populations",fontsize=11) #plt.ylim((0,100)) plt.ylabel("Percent of Total",fontsize=11) plt.xlabel("Initial Mutation rate: u",fontsize=11) plt.savefig(opt.filename+"_bar") plt.clf() ###
y = eta**2/(2*gamma) print nd, " of ",len(deltas) for nphi,phi in enumerate(phis): fun = lambda x: f(x,gamma,eta,phi,alpha,thetas,tau,delta) y = opt.fsolve(fun, y) epss[nd,nphi] = y/(eta**2/(2*gamma)) ls[nd,nphi] = lambdabar(thetas,gamma,eta,phi,alpha,tau,delta) strings = [r'$\tau\delta$ = '+str(i*tau) for i in deltas] #f = lambda i : plt.plot(alphas,epss[i,:],label=strings[i]) #map(f,range(len(deltas))) fig, ax = ppl.subplots(1,figsize=(7,7)) ppl.plot(ls[0,:],epss[0,:],label = strings[0], ax=ax) ppl.plot(ls[1,:],epss[1,:],label = strings[1], ax=ax) ppl.plot(ls[2,:],epss[2,:],label = strings[2], ax=ax) ppl.plot(ls[3,:],epss[3,:],label = strings[3], ax=ax) ppl.plot(ls[4,:],epss[4,:],label = strings[4], ax=ax) np.savez("../data/mean_field_ratedist.npz", eps = epss, ls=ls, phis = phis, deltas = deltas, tau = tau) ax.set_xlim([0.0,25.0]) ax.set_xlabel(r'$\lambda$ in spikes/sec') ax.set_ylabel(r'MMSE/$\epsilon_0$') plt.title("Rate-Distortion Curve for Adaptive Neurons") ppl.legend(ax) plt.savefig("rate_distortion_mf_alpha.png") plt.savefig("rate_distortion_mf_alpha.eps") plt.show()
else: spiketimes = [] thetas = [] ax1.plot(spiketimes,thetas,'yo',label='Spike times') ax1.plot(times,mg,'b',label='Mean prediction') ax1.set_title('Gaussian Filter') ax1.set_ylabel('Signal space') ax1.legend() ax2 = plt.gcf().add_subplot(2,1,2) ax2.plot(times,sp,'r',label='Signal') if sum(sum(spsp)) !=0: (tsp,neursp) = np.where(spsp == 1) spiketimesp = times[tsp] thetasp = [code.neurons[i].theta for i in neursp] else: spiketimesp = [] thetasp = [] ax2.plot(times,mp,label='Mean prediction') ppl.fill_between(times,mp-np.sqrt(varp),mp+np.sqrt(varp),ax=ax2) ax2.plot(spiketimesp,thetasp,'.',label='Spike times') ax2.set_ylabel('Signal space') ax2.set_xlabel('Time') ax2.legend() ax2.set_title('Particle Filter') plt.savefig('filtering.png',dpi=150)
ax1.set_xlabel(r'$\alpha$') ax2.set_ylabel(r'$\phi$') l1, = ppl.plot(alphas, eps[:,1], label=r'$\phi = '+str(phis[1])+r'$',ax=ax2) c1 = l1.get_color() ppl.plot(alphas,stoc_eps_02, '.-', color=c1) indmin = numpy.argmin(eps[:,1]) ppl.plot(alphas[indmin],eps[indmin,1],'o',color=c1) l2, = ppl.plot(alphas, eps[:,9], label=r'$\phi = '+str(phis[9])+r'$',ax=ax2) c2 = l2.get_color() ppl.plot(alphas,stoc_eps_10, '.-', color=c2) indmin = numpy.argmin(eps[:,9]) ppl.plot(alphas[indmin],eps[indmin,9],'o',color=c2) l3, = ppl.plot(alphas, eps[:,-1], label=r'$\phi = '+str(phis[-1])+r'$',ax=ax2) c3 = l3.get_color() ppl.plot(alphas,stoc_eps_20, '.-', color=c3) indmin = numpy.argmin(eps[:,-1]) ppl.plot(alphas[indmin],eps[indmin,-1],'o',color=c3) ppl.legend(ax2) ax2.set_xlabel(r'$\alpha$') ax2.set_ylabel(r'$\epsilon$') plt.savefig("figure_5_7.eps")
l1, = ax1.plot(spiketimes,thetas,'o',label='Observed Spikes') l3 = ppl.fill_between(times,mg-stg,mg+stg,ax=ax1,alpha=0.2) c1 = l1.get_color() c2 = l2.get_color() c3 = l3.get_facecolor() c4 = l4.get_color() ax1.set_title('Gaussian Assumed Density Filter') ax1.set_ylabel('Position [cm] (Preferred Stimulus)') ax1.set_xlabel('Time [s]') ppl.legend(ax1).get_frame().set_alpha(0.6) thetas = [code.neurons[i].theta for i in sptrain] ax2.plot(times,s,color=c4,label = 'True Sate') ax2.plot(times,m,color=c2,label='Posterior Mean') ax2.plot(times[sptimes],thetas,'o',color=c1,label='Observed Spikes') ppl.fill_between(times,m-st,m+st,ax=ax2,facecolor=c3,alpha=0.2) ax2.set_ylabel('Position [cm] (Preferred Stimulus)') ax2.set_xlabel('Time [s]') ppl.legend(ax2).get_frame().set_alpha(0.6) ax2.set_title('Particle Filter') ax1.set_ylim([(m-st).min()-0.5,(m+st).max()+0.5]) ax2.set_ylim([(m-st).min()-0.5,(m+st).max()+0.5]) #plt.show() plt.savefig('filtering_both.pdf') plt.savefig('filtering_both.eps') plt.savefig('filtering_both.png')
## import sys if __name__ == '__main__': # Accumulate the counts fed into standard in (stdin) counts = [] for line in sys.stdin: topic, count = line.split() # Shift the ones up by a tiny amount to allow them to be visible on the graph counts.append(int(count) + 0.05) print 'Total page view counts: {}'.format(len(counts)) # Display the hourly page view distribution on a log-log plot # This matches our friendly and well understood Zipf distribution fig = plt.figure(figsize=(9, 5)) ax = fig.add_subplot(1, 1, 1) plt.plot(xrange(len(counts)), counts, linewidth=3, label='Pageviews per page') # ax.set_xscale('log') ax.set_yscale('log') # plt.title('Log-log plot of hourly Wikipedia page view distribution') plt.xlabel('Rank order') plt.ylabel('Frequency') plt.grid(color='black') plt.legend() # plt.savefig('hourly_wikipedia_zipf.pdf') plt.show(block=True)
thetas = np.arange(-7.0,7.0,dtheta) xs = np.arange(-2.2,2.2,0.01) rates1 = [[rate(x,0.3,phi,t) for x in xs] for t in thetas] rates2 = [[rate(x,1.0,phi,t) for x in xs] for t in thetas] rates3 = [[rate(x,2.0,phi,t) for x in xs] for t in thetas] plt.rcParams['text.usetex']=True ax1.plot( alphas, eps ) ax1.set_xlabel(r'$\alpha$') ax1.set_ylabel(r'$MMSE$') map(lambda x : ax2.plot(xs,x), rates1) map(lambda x : ax3.plot(xs,x), rates2) map(lambda x : ax4.plot(xs,x), rates3) cm.getMaternSample(gamma=gamma,eta=eta,order=1,phi=phi,dtheta=dtheta,plot=True,ax=ax5,alpha=0.3, timewindow=20000) cm.getMaternSample(gamma=gamma,eta=eta,order=1,phi=phi,dtheta=dtheta,plot=True,ax=ax6,alpha=1.0, timewindow=20000) cm.getMaternSample(gamma=gamma,eta=eta,order=1,phi=phi,dtheta=dtheta,plot=True,ax=ax7,alpha=2.0, timewindow=20000) map(lambda x : x.set_ylabel(r'$Rate$'),[ax2,ax3,ax4]) map(lambda x : x.set_ylim([-0.01,1.7*phi]),[ax2,ax3,ax4]) ax2.set_title('Coding Strategies') ax2.legend([r'$\alpha=0.3$']) ax3.legend([r'$\alpha=1.0$']) ax4.legend([r'$\alpha=2.0$']) ax4.set_xlabel(r'$x$') plt.savefig('estimation_uni.png',dpi=300) plt.savefig('estimation_uni.eps')
import numpy from prettyplotlib import plt if __name__=='__main__': xs = numpy.arange(-5.0,5.0,0.001) def f(x): return 1.0*numpy.exp(-x**2/2) def fdash(x): return -x*numpy.exp(-x**2/2) tuning = f(xs) info = fdash(xs)**2/f(xs) plt.plot(xs,tuning,label='Tuning function') plt.plot(xs,info,label='Fisher Information') plt.gca().set_xlabel('x') plt.legend() plt.savefig('../figures/figure_3_5.eps')
matern0 = matern1 matern1 = IterateOnce(matern0, K_matern, dx,alpha =0.1) dist = SquaredDistance(matern0,matern1) print dist OU0 = K_ou(xs) OU1 = IterateOnce(OU0, K_ou, dx,alpha=0.1) dist = SquaredDistance(OU0,OU1) while dist > 1e-10: OU0 = OU1 OU1 = IterateOnce(OU0, K_ou, dx,alpha =0.1) dist = SquaredDistance(OU0,OU1) print dist ax1 = plt.subplot(311) ax2 = plt.subplot(312) ax3 = plt.subplot(313) plt.gcf().suptitle("Posterior kernels") ax1.plot(xs,K_ou(xs),label=r'OU prior kernel') ax1.plot(xs,OU1,label = r'OU posterior kernel') ax1.legend() ax2.plot(xs,K_matern(xs),label=r'Matern prior kernel') ax2.plot(xs,matern1, label=r'Matern posterior kernel') ax2.legend() ax3.plot(xs,K_rbf(xs),label=r'RBF prior kernel') ax3.plot(xs,rbf1,label=r'RBF posterior kernel') ax3.legend() plt.savefig("figure_3_3.eps")
ppl.plot(ts, Xcrit2[0,:,0], label=r'$\omega$ = 2.0', ax=axcrit) ppl.plot(ts, Xcrit4[0,:,0], label=r'$\omega$ = 4.0', ax=axcrit) p = ppl.legend(axcrit) fr = p.get_frame() fr.set_alpha(0.4) axcrit.set_title("Critical") A_over = numpy.array([[0.0,1.0],[-2.0,-5.0]]) H_over = numpy.array([[0.0,0.0],[0.0,1.5]]) ts,Xover2 = sample(X0,A_over, H_over, dt, N, 1) ts,Xover1 = sample(X0,numpy.array([[0.0,1.0],[-1.0,-2.5]]), 0.5*H_over, dt,N,1) ts,Xover4 = sample(X0,numpy.array([[0.0,1.0],[-4.0,-10.0]]), 2.0*H_over, dt,N,1) ppl.plot(ts, Xover1[0,:,0], label=r'$\omega$ = 1.0', ax=axover) ppl.plot(ts, Xover2[0,:,0], label=r'$\omega$ = 2.0', ax=axover) ppl.plot(ts, Xover4[0,:,0], label=r'$\omega$ = 4.0', ax=axover) p = ppl.legend(axover) fr = p.get_frame() fr.set_alpha(0.4) axover.set_title("Overdamped") axover.set_xlabel('Time [s]') axcrit.set_ylabel('Position [cm]') axou.set_ylabel('Position [cm]') axcrit.set_xlabel('Time [s]') plt.gcf().suptitle('Linear Stochastic Processes') plt.savefig('../figures/figure_5_2.eps') plt.savefig('../figures/figure_5_2.pdf')
ticklabels = np.round(ticks,decimals=2) cb1 = plt.colorbar(p1,ax=ax2) cb1.set_ticks(ticks) cb1.set_ticklabels(ticklabels) cb2 = plt.colorbar(p2,ax=ax3) cb2.set_ticks(ticks) cb2.set_ticklabels(ticklabels) cb3 = plt.colorbar(p3,ax=ax4) cb3.set_ticks(ticks) cb3.set_ticklabels(ticklabels) ax4.set_xlabel(r'$\Delta\theta$') ax4.set_ylabel(r'$\alpha$') ax3.set_ylabel(r'$\alpha$') ax2.set_ylabel(r'$\alpha$') ax3.axes.get_xaxis().set_ticks([]) ax2.axes.get_xaxis().set_ticks([]) #ax3.tick_params(axis='x',which='both',bottom='off') #ax2.tick_params(axis='x',which='both',bottom='off') ax2.set_title('ADF with dense coding assumption') ax3.set_title('Full ADF') ax4.set_title('Particle Filter') [p.set_clim(vmin=mintotal,vmax=maxtotal) for p in [p1,p2,p3]] plt.savefig('figure_5_14.pdf')
plt.plot(X, Y, label='iSVD u={}'.format(num)) """ print 'Testing raw SVD => exact reconstruction' svT = scipy.linalg.diagsvd(s, u.shape[0], vT.shape[1]).dot(vT) for y in xrange(train.shape[0]): for x in xrange(train.shape[1]): colU = u[y, :] rowV = svT[:, x] assert np.allclose(train[y, x], single_dot(u, svT, x, y)) """ ## plt.title('SVD reconstruction error on {}x{} matrix'.format(*train.shape)) plt.xlabel('Low rank approximation (k)') plt.ylabel('Frobenius norm') plt.ylim(0, max(svdY)) plt.legend(loc='best') plt.savefig('reconstruct_fro_{}x{}.pdf'.format(*train.shape)) plt.show(block=True) ## plt.plot(orthoX, orthoY, label="SVD", color='black', linewidth=2, linestyle='--') for label, X, Y in incr_ortho: plt.plot(X, Y, label=label) plt.title('SVD orthogonality error on {}x{} matrix'.format(*train.shape)) plt.xlabel('Low rank approximation (k)') plt.ylabel('Deviation from orthogonality') plt.semilogy() #plt.ylim(0, max(orthoY)) plt.legend(loc='best') plt.savefig('reconstruct_ortho_{}x{}.pdf'.format(*train.shape)) plt.show(block=True)
print "stringing" string = [r'$\phi$ = %.2f' % float(i) for i in data[3]] maxmmse = np.max(data[0]) def f(x): line, = ppl.plot(data[2],data[0][:,x],label=string[x],axes=ax1) return line.get_color() def get_min(data): minim = np.min(data) indmin = np.argmin(data) return minim,indmin print "subplotting" fig, ax1 = ppl.subplots(1) ax1.set_title(r'Classification') colors = map(f,range(data[3].size)) for i in range(data[3].size): mi,ind = get_min(data[0][:,i]) ppl.plot(data[2][ind],mi,'o',color=colors[i],axes=ax1) ppl.legend(ax1) ax1.set_xlabel(r'$\alpha$') ax1.set_ylabel(r'MMSE') plt.savefig("mmse_classification.eps")
np.random.shuffle(nodes) print 'Pageviews from "real" edge weights' print '-=-=-=-=-' display_graph(G) print print 'Pageviews from evenly distributed edge weights' print '-=-=-=-=-' display_graph(approxG) plt.plot(np.arange(0, len(rerrs)), rerrs, label='Relative error over time') plt.xlabel('Iteration') plt.ylabel('Average pageview relative error per node') plt.legend() plt.savefig('error_over_time.pdf') plt.show(block=True) plt.plot(np.arange(0, len(werrs)), werrs, label='Weight error over time') plt.xlabel('Iteration') plt.ylabel('Average weight error per edge') plt.legend() plt.savefig('weight_over_time.pdf') plt.show(block=True) fig, ax = plt.subplots(1) ppl.bar(ax, *orig_weight_data, alpha=0.5, color='black', label='Weight error before') ppl.bar(ax, np.arange(0, G.number_of_edges()), [abs(G[u][v]['weight'] - approxG[u][v]['weight']) for u, v in G.edges()], alpha=0.8, label='Weight error after') #plt.ylim(-1, 1) plt.legend(loc='best') plt.show(block=True)
""" print 'Testing raw SVD => exact reconstruction' svT = scipy.linalg.diagsvd(s, u.shape[0], vT.shape[1]).dot(vT) for y in xrange(train.shape[0]): for x in xrange(train.shape[1]): colU = u[y, :] rowV = svT[:, x] assert np.allclose(train[y, x], single_dot(u, svT, x, y)) """ ## plt.title('SVD reconstruction error on {}x{} matrix'.format(*train.shape)) plt.xlabel('Low rank approximation (k)') plt.ylabel('Frobenius norm') plt.ylim(0, max(svdY)) plt.legend(loc='best') plt.savefig('reconstruct_fro_{}x{}.pdf'.format(*train.shape)) plt.show(block=True) ## plt.plot(orthoX, orthoY, label="SVD", color='black', linewidth=2, linestyle='--') for label, X, Y in incr_ortho: plt.plot(X, Y, label=label) plt.title('SVD orthogonality error on {}x{} matrix'.format(*train.shape)) plt.xlabel('Low rank approximation (k)') plt.ylabel('Deviation from orthogonality') plt.semilogy() #plt.ylim(0, max(orthoY))
svdX = [] svdY = [] u, s, vT = scipy.linalg.svd(train) for k in xrange(1, 100): low_s = [s[i] for i in xrange(k)] # + (min(u.shape[0], vT.shape[1]) - k) * [0] print 'Exact SVD with low-rank approximation {}'.format(k) svdX.append(k) svdY.append(get_error(u, np.diag(low_s), vT, train, test)) plt.plot(svdX, svdY, label="SVD", color='black', linewidth='2', linestyle='--') """ print print 'Testing incremental SVD' for num in xrange(400, 1001, 300): print '... with block size of {}'.format(num) X, Y = [], [] for k in xrange(1, 91, 10): print k u, s, vT = incremental_SVD(train, k, num) X.append(k) Y.append(get_error(u, s, vT, train, test, prod_avg)) plt.plot(X, Y, label='iSVD u={}'.format(num)) ## plt.title('Recommendation system RMSE on {}x{} matrix'.format(*train.shape)) plt.xlabel('Low rank approximation (k)') plt.ylabel('Root Mean Squared Error') #plt.ylim(0, max(svdY)) plt.legend(loc='best') plt.savefig('recommend_rmse_{}x{}.pdf'.format(*train.shape)) plt.show(block=True)
OU1 = IterateOnce(OU0, K_ou, dx,alpha=0.1) dist = SquaredDistance(OU0,OU1) while dist > 1e-10: OU0 = OU1 OU1 = IterateOnce(OU0, K_ou, dx,alpha =0.1) dist = SquaredDistance(OU0,OU1) print dist fig, (ax1,ax2,ax3) = ppl.subplots(3,1) #ax1 = plt.subplot(311) #ax2 = plt.subplot(312) #ax3 = plt.subplot(313) plt.gcf().suptitle("Posterior kernels") xs = xs[:6000] OU1 = OU1[:6000] matern1 = matern1[:6000] rbf1 = rbf1[:6000] ppl.plot(xs,K_ou(xs),label=r'OU prior kernel',ax=ax1) ppl.plot(xs,OU1,label = r'OU posterior kernel',ax=ax1) ppl.legend(ax1) ppl.plot(xs,K_matern(xs),label=r'Matern prior kernel',ax=ax2) ppl.plot(xs,matern1, label=r'Matern posterior kernel',ax=ax2) ppl.legend(ax2) ppl.plot(xs,K_rbf(xs),label=r'RBF prior kernel',ax=ax3) ppl.plot(xs,rbf1,label=r'RBF posterior kernel',ax=ax3) ppl.legend(ax3) plt.savefig("../figures/figure_3_4.eps")
import prettyplotlib as ppl from prettyplotlib import plt def drift(x,eq): return 4.0*x*(eq-x**2) def bistablesample(x0,eq,eta,T,dt,NSamples): xs = numpy.zeros((NSamples,T)) xs[:,0] = x0 s_dt = numpy.sqrt(dt) for i in range(1,T): rands = numpy.random.normal(0,1,(NSamples,)) xs[:,i] = xs[:,i-1]+dt*drift(xs[:,i-1],eq) + s_dt*eta*rands return xs if __name__=='__main__': samples = bistablesample(0.0,1,0.9,300000,0.001,1) ts = numpy.arange(0.0,30000*0.001,0.001) print ts.shape print samples.shape fig, ax = ppl.subplots(1) ppl.plot(ts,samples,ax=ax) ax.set_xlabel('Time [s]') ax.set_ylabel('Position [cm]') plt.savefig('../figures/figure_5_10.eps') plt.savefig('../figures/figure_5_10.png',dpi=300) plt.show()
#plt.xticks(xticks,xlabels,axes=ax1) #plt.yticks(yticks,ylabels,axes=ax1) cb = plt.colorbar(p, ax=ax1) cb.set_ticks(np.array([0.3,0.4,0.5])) cb.set_ticklabels(np.array([0.3,0.4,0.5])) ax1.set_xlabel(r'$\alpha$') ax1.set_ylabel(r'$\phi$') ax1.set_title(r'MMSE ($\epsilon$)') l1,= ppl.plot( alphas, eps[:,10], label=r'$\phi = '+ str(phis[10]-0.001) + r'$',ax=ax2) ppl.plot( alphas[np.argmin(eps[:,10])], np.min(eps[:,10]), 'o',color=l1.get_color(),ax=ax2) l2, = ppl.plot( alphas, eps[:,20], label=r'$\phi = '+ str(phis[20]-0.001) + r'$',ax=ax2) ppl.plot( alphas[np.argmin(eps[:,20])] , np.min(eps[:,20]), 'o',color=l2.get_color(),ax=ax2) l3, = ppl.plot( alphas, eps[:,30], label=r'$\phi = '+ str(phis[30]-0.001) + r'$',ax=ax2) ppl.plot( alphas[np.argmin(eps[:,30])] , np.min(eps[:,30]), 'o',color=l3.get_color(),ax=ax2) l4, = ppl.plot( alphas, eps[:,40], label=r'$\phi = '+ str(phis[40]-0.001) + r'$',ax=ax2) ppl.plot( alphas[np.argmin(eps[:,40])] , np.min(eps[:,40]), 'o',color=l4.get_color(),ax=ax2) ppl.plot( alphas , stoc_eps[:,10], '-.',ax=ax2, color = l1.get_color()) ppl.plot( alphas , stoc_eps[:,20], '-.',ax=ax2, color = l2.get_color()) ppl.plot( alphas , stoc_eps[:,30], '-.',ax=ax2, color = l3.get_color()) ppl.plot( alphas , stoc_eps[:,40], '-.',ax=ax2, color = l4.get_color()) ax2.set_xlabel(r'$\alpha$') ax2.set_ylabel(r'$\epsilon$') ax2.set_title(r'MMSE as a function of $\alpha$') ppl.legend(ax2).get_frame().set_alpha(0.7) plt.savefig('../figures/figure_5_3.png',dpi=300) plt.savefig('../figures/figure_5_3.pdf') os.system("echo \"all done\" | mutt -a \"../figures/figure_5_3.eps\" -s \"Plot\" -- [email protected]")
svdY = [] u, s, vT = scipy.linalg.svd(train) for k in xrange(1, 100): low_s = [s[i] for i in xrange(k)] # + (min(u.shape[0], vT.shape[1]) - k) * [0] print 'Exact SVD with low-rank approximation {}'.format(k) svdX.append(k) svdY.append(get_error(u, np.diag(low_s), vT, train, test)) plt.plot(svdX, svdY, label="SVD", color='black', linewidth='2', linestyle='--') """ print print 'Testing incremental SVD' for num in xrange(400, 1001, 300): print '... with block size of {}'.format(num) X, Y = [], [] for k in xrange(1, 91, 10): print k u, s, vT = incremental_SVD(train, k, num) X.append(k) Y.append(get_error(u, s, vT, train, test, prod_avg)) plt.plot(X, Y, label='iSVD u={}'.format(num)) ## plt.title( 'Recommendation system RMSE on {}x{} matrix'.format(*train.shape)) plt.xlabel('Low rank approximation (k)') plt.ylabel('Root Mean Squared Error') #plt.ylim(0, max(svdY)) plt.legend(loc='best') plt.savefig('recommend_rmse_{}x{}.pdf'.format(*train.shape)) plt.show(block=True)