def plot_wise(cat_path):
for catfile in find_files(cat_path, "*merged+wise.csv"):
print("\nreading catalog: {}".format(catfile))
df = pd.read_csv(catfile)
# convert to magnitudes
nbadflux = (df.flux <= 0).sum()
try:
assert nbadflux == 0
except:
print("warning: {} negative flux source(s)".format(nbadflux))
ch = catfile.split('/')[-1].split('_')[1]
mags = spz_jy_to_mags(df.flux*1e-3, float(ch))
if ch == '1':
plt.scatter(df.W1mag, mags)
plt.xlabel('W1 [mag]')
plt.ylabel('I1 [mag]')
elif ch == '2':
plt.scatter(df.W2mag, mags)
plt.xlabel('W2 [mag]')
plt.ylabel('I2 [mag]')
ax = plt.gca()
xlim, ylim = ax.get_xlim(), ax.get_ylim()
plt.plot([-5, ylim[1]*2], [-5, ylim[1]*2], 'r-')
ax.set_xlim(xlim) ; ax.set_ylim(ylim)
reg = catfile.split('/')[-1].split('_')[0]
name = '{}_{}_IRAC_vs_WISE.png'.format(reg, ch)
outpath = '/'.join(catfile.split('/')[:-1]+[name])
plt.savefig(outpath, dpi=120)
plt.close()
def plot(x, y, outpath, xlabel, ylabel, plot_style, plot_type):
if plot_type == 'mag-mag':
xlim = (10, 18)
ylim = (10, 18)
elif plot_type == 'color-mag':
xlim = (10, 18)
ylim = (-1, 1)
elif plot_type == 'color-color':
xlim = (-1, 1)
ylim = (-1, 1)
else:
raise(ValueError("plot_type should be one of ['mag-mag', 'color-mag', 'color-color'] "))
isinrange = lambda a,b: (a>=b[0]) & (a<=b[1])
g = isinrange(x, xlim) & isinrange(y, ylim)
if plot_style == 'scatter':
plt.scatter(x[g], y[g])
elif plot_style == 'hexbin':
plt.hexbin(x[g], y[g])
elif plot_style == 'hist2d':
plt.hist2d(x[g], y[g], bins=100)
else:
raise(ValueError("plot_style should be one of ['scatter', 'hexbin', 'hist2d'] "))
plt.xlabel(xlabel)
plt.ylabel(ylabel)
ax = plt.gca()
ax.set_xlim(xlim)
ax.set_ylim(ylim)
plt.savefig(outpath, dpi=120)
plt.close()
print("created file: {}".format(outpath))
def plot_wise(cat_path):
for catfile in find_files(cat_path, "*merged+wise.csv"):
print("\nreading catalog: {}".format(catfile))
df = pd.read_csv(catfile)
# convert to magnitudes
nbadflux = (df.flux <= 0).sum()
try:
assert nbadflux == 0
except:
print("warning: {} negative flux source(s)".format(nbadflux))
ch = catfile.split('/')[-1].split('_')[1]
mags = spz_jy_to_mags(df.flux * 1e-3, float(ch))
if ch == '1':
plt.scatter(df.W1mag, mags)
plt.xlabel('W1 [mag]')
plt.ylabel('I1 [mag]')
elif ch == '2':
plt.scatter(df.W2mag, mags)
plt.xlabel('W2 [mag]')
plt.ylabel('I2 [mag]')
ax = plt.gca()
xlim, ylim = ax.get_xlim(), ax.get_ylim()
plt.plot([-5, ylim[1] * 2], [-5, ylim[1] * 2], 'r-')
ax.set_xlim(xlim)
ax.set_ylim(ylim)
reg = catfile.split('/')[-1].split('_')[0]
name = '{}_{}_IRAC_vs_WISE.png'.format(reg, ch)
outpath = '/'.join(catfile.split('/')[:-1] + [name])
plt.savefig(outpath, dpi=120)
plt.close()
def plot(x, y, outpath, xlabel, ylabel, plot_style, plot_type):
if plot_type == 'mag-mag':
xlim = (10, 18)
ylim = (10, 18)
elif plot_type == 'color-mag':
xlim = (10, 18)
ylim = (-1, 1)
elif plot_type == 'color-color':
xlim = (-1, 1)
ylim = (-1, 1)
else:
raise (ValueError(
"plot_type should be one of ['mag-mag', 'color-mag', 'color-color'] "
))
isinrange = lambda a, b: (a >= b[0]) & (a <= b[1])
g = isinrange(x, xlim) & isinrange(y, ylim)
if plot_style == 'scatter':
plt.scatter(x[g], y[g])
elif plot_style == 'hexbin':
plt.hexbin(x[g], y[g])
elif plot_style == 'hist2d':
plt.hist2d(x[g], y[g], bins=100)
else:
raise (ValueError(
"plot_style should be one of ['scatter', 'hexbin', 'hist2d'] "))
plt.xlabel(xlabel)
plt.ylabel(ylabel)
ax = plt.gca()
ax.set_xlim(xlim)
ax.set_ylim(ylim)
plt.savefig(outpath, dpi=120)
plt.close()
print("created file: {}".format(outpath))
def plot_learning_curves(num_points,
X_train,
Y_train,
X_test,
Y_test,
positive_class=1,
negative_class=0):
train_set_sizes = [len(X_train) / k for k in range(num_points + 1, 0, -1)]
test_errors = []
training_errors = []
for training_set_size in train_set_sizes:
model = train(X_train, Y_train, training_set_size)
test_error = evaluate(model, X_test, Y_test, positive_class,
negative_class)
training_error = evaluate(model, X_train, Y_train, positive_class,
negative_class)
test_errors.append(test_error)
training_errors.append(training_error)
plt.plot(train_set_sizes,
training_errors,
'bs-',
label='Training accuracy')
plt.plot(train_set_sizes, test_errors, 'g^-', label='Test accuracy')
plt.ylabel('Accuracy')
plt.xlabel('Number of training samples')
plt.title('Augmented Logistic Regression Learning Curve')
plt.legend(loc='lower right')
plt.savefig('../Figures/accuracyPlotAugmented.png', dpi=100)
pylab.show()
def plot_days(days):
nplots = len(days)
fig, subplots = plt.subplots(nplots, figsize=(10, 50))
for i, day in enumerate(days):
subplot = subplots[i]
plot_day(day, subplot)
plt.tight_layout()
plt.savefig('./plots/best.png')
def plot_days(days):
nplots = len(days)
fig, subplots = plt.subplots(nplots, figsize=(10, 50))
for i, day in enumerate(days):
subplot = subplots[i]
plot_day(day, subplot)
plt.tight_layout()
plt.savefig('./plots/best.png')
def scipy_fcluster_merge_strategy(assignments):
# Z-matrix helpers
def groups(assignments):
cluster_map = {}
for idx, gid in enumerate(assignments):
lst = cluster_map.get(gid, [])
lst.append(idx)
cluster_map[gid] = lst
return tuple(sorted(map(tuple, cluster_map.values()), key=len, reverse=True))
def zmatrix(assignments_samples):
n = len(assignments_samples[0])
# should make sparse matrix
zmat = np.zeros((n, n), dtype=np.float32)
for assignments in assignments_samples:
clusters = groups(assignments)
for cluster in clusters:
for i, j in it.product(cluster, repeat=2):
zmat[i, j] += 1
zmat /= float(len(assignments_samples))
return zmat
def reorder_zmat(zmat, order):
zmat = zmat[order]
zmat = zmat[:,order]
return zmat
def linkage(zmat):
assert zmat.shape[0] == zmat.shape[1]
zmat0 = np.array(zmat[np.triu_indices(zmat.shape[0], k=1)])
zmat0 = 1. - zmat0
return scipy.cluster.hierarchy.linkage(zmat0)
def ordering(l):
return np.array(scipy.cluster.hierarchy.leaves_list(l))
zmat = zmatrix(assignments)
li = linkage(zmat)
# diagnostic: draw z-matrix
indices = ordering(li)
plt.imshow(reorder_zmat(zmat, indices),
cmap=plt.cm.binary, interpolation='nearest')
plt.savefig("zmat-before.pdf")
plt.close()
fassignment = scipy.cluster.hierarchy.fcluster(li, 0.001)
clustering = groups(fassignment)
sorted_ordering = list(it.chain.from_iterable(clustering))
# draw post fcluster() ordered z-matrix
plt.imshow(reorder_zmat(zmat, sorted_ordering),
cmap=plt.cm.binary, interpolation='nearest')
plt.savefig("zmat-after.pdf")
plt.close()
return fassignment
def plot_sdss(cat_path):
for catfile in find_files(cat_path, "*merged+sdss.txt"):
# for now ignore the channel 2 files
if catfile.split('/')[-1].split('_')[1] != '1':
continue
print("\nreading catalog: {}".format(catfile))
df = pd.read_table(catfile, sep=' ')
# get rid of negative flux sources, if any
df = df[df.flux > 0]
# convert to magnitudes
mags = spz_jy_to_mags(df.flux * 1e-3, 1)
# print counts per magnitude bin
for i in range(10, 15):
sc = ((df.cl == 3) & (mags > i) & (mags < i + 1)).sum()
xc = ((df.xsc == 1) & (mags > i) & (mags < i + 1)).sum()
msg = "{}th to {}th mag: {} SDSS galaxy sources, {} 2MASS XSC sources"
print(msg.format(i, i + 1, sc, xc))
# print number of sources agreed upon
agree = ((df.xsc == 1) & (df.cl == 3)).sum()
disagree = ((df.xsc == 1) & (df.cl == 6)).sum()
na = ((df.xsc == 1) & (df.cl == 0)).sum()
msg = "{} 2MASS XSC sources classified as galaxies by SDSS"
print(msg.format(agree))
msg = "{} 2MASS XSC sources classified as stars by SDSS"
print(msg.format(disagree))
msg = "{} 2MASS XSC sources not matched to SDSS"
print(msg.format(na))
# plot normed histograms of 2MASS XSC and SDSS galaxy magnitudes
xsc_gals = (mags > 10) & (mags < 15) & (df.xsc == 1)
sdss_gals = (mags > 10) & (mags < 15) & (df.cl == 3)
# mags[xsc_gals].hist(label='2MASS XSC', normed=True)
# mags[sdss_gals].hist(label='SDSS galaxies', normed=True)
plt.hist([mags[xsc_gals].values, mags[sdss_gals].values],
bins=5,
label=['2MASS', 'SDSS'])
plt.xlabel('IRAC1 [mag]')
plt.ylabel('Number Count')
reg = catfile.split('/')[-1].split('_')[0]
plt.title('{} Extended Sources / Galaxies'.format(reg))
plt.legend(loc=2)
name = '{}_2mass_xsc_vs_sdss_hist.png'.format(reg)
outpath = '/'.join(catfile.split('/')[:-1] + [name])
plt.savefig(outpath, dpi=100)
plt.close()
print("created file: {}".format(outpath))
def plot_sdss(cat_path):
for catfile in find_files(cat_path, "*merged+sdss.txt"):
# for now ignore the channel 2 files
if catfile.split('/')[-1].split('_')[1] != '1':
continue
print("\nreading catalog: {}".format(catfile))
df = pd.read_table(catfile, sep=' ')
# get rid of negative flux sources, if any
df = df[df.flux > 0]
# convert to magnitudes
mags = spz_jy_to_mags(df.flux*1e-3, 1)
# print counts per magnitude bin
for i in range(10,15):
sc = ((df.cl == 3) & (mags > i) & (mags < i+1)).sum()
xc = ((df.xsc == 1) & (mags > i) & (mags < i+1)).sum()
msg = "{}th to {}th mag: {} SDSS galaxy sources, {} 2MASS XSC sources"
print(msg.format(i, i+1, sc, xc))
# print number of sources agreed upon
agree = ((df.xsc == 1) & (df.cl == 3)).sum()
disagree = ((df.xsc == 1) & (df.cl == 6)).sum()
na = ((df.xsc == 1) & (df.cl == 0)).sum()
msg = "{} 2MASS XSC sources classified as galaxies by SDSS"
print(msg.format(agree))
msg = "{} 2MASS XSC sources classified as stars by SDSS"
print(msg.format(disagree))
msg = "{} 2MASS XSC sources not matched to SDSS"
print(msg.format(na))
# plot normed histograms of 2MASS XSC and SDSS galaxy magnitudes
xsc_gals = (mags > 10) & (mags < 15) & (df.xsc == 1)
sdss_gals = (mags > 10) & (mags < 15) & (df.cl == 3)
# mags[xsc_gals].hist(label='2MASS XSC', normed=True)
# mags[sdss_gals].hist(label='SDSS galaxies', normed=True)
plt.hist([mags[xsc_gals].values, mags[sdss_gals].values],
bins=5, label=['2MASS', 'SDSS'])
plt.xlabel('IRAC1 [mag]')
plt.ylabel('Number Count')
reg = catfile.split('/')[-1].split('_')[0]
plt.title('{} Extended Sources / Galaxies'.format(reg))
plt.legend(loc=2)
name = '{}_2mass_xsc_vs_sdss_hist.png'.format(reg)
outpath = '/'.join(catfile.split('/')[:-1]+[name])
plt.savefig(outpath, dpi=100)
plt.close()
print("created file: {}".format(outpath))
def plot_signal(signal, shape=None, name=None):
"""
Plot a numeric signal as an image.
"""
s = np.copy(signal)
if s.max() != 0.:
s *= 255.0/s.max()
s = s.astype(int)
if shape:
s = s.reshape(shape)
plt.figure()
plt.imshow(s, cmap=plt.cm.gray, interpolation='nearest')
if name:
plt.savefig(name)
plt.close()
def histogram(original, updated, bins=None, main="", save=None, log=False):
"""Plot a histogram of score improvements (updated-origianl)
Input:
original - list of original scores
updated - list of updates scores in same order as original
bins - number of bins to represent improvements
"""
#Lengths of score lists must be identical, assume in same order
assert len(original) == len(original)
#Set up bins:
if bins is not None and bins > 0:
imoprovements = {(-1,-1):0}
for i in xrange(0, len(original), bins):
improvements[(0,i+bins)] = 0
else:
improvements = {(-1,-1):0, (-5,0):0, (0,1):0, (1,25):0, (25,50):0, (50,75):0, (75,100):0, (100,125):0, (125,150):0, (150,200):0, (200,300):0, (300,400):0, (500,10000):0} #defaultdict(int)
#Calcualte improvements
for o, u in izip(original, updated):
if o>u:
improvements[(-1,-1)] += 1
continue
for lower, upper in improvements:
if lower <= int(u-o) < upper:
improvements[(lower,upper)] += 1
break
keys = sorted(improvements.keys(), key=lambda x:x[0])
values = [improvements[r] for r in keys]
fig, ax = plt.subplots()
ax.set_title(main)
ax.set_xlabel("Improvement (updated-original) bitscores")
ax.set_ylabel("log(Frequency)")
#ax.set_yscale('log')
width = 1.0
#ax.set_xticks(np.arange(len(improvements)))
#ax.set_xticklabels([l for l, u in keys])
bar(ax, np.arange(len(improvements)), values, log=log,
annotate=True, grid='y', xticklabels=[l for l, u in keys])
if save is None:
plt.show()
else:
plt.savefig(save)
def layerActivations(model, data, labels):
print('Visualizing activations with tSNE...')
if not os.path.exists(cc.cfg['plots']['layer_activations_dir']):
os.makedirs(cc.cfg['plots']['layer_activations_dir'])
numLabels = cc.cfg['plots']['layer_activations_label_cap']
data = data[:cc.cfg['plots']['layer_activations_points_cap']]
labels = labels[:numLabels,:cc.cfg['plots']['layer_activations_points_cap']]
subplotCols = numLabels
subplotRows = len(model.layers)-1
subplotIdx = 1
plt.figure(figsize=(5*subplotCols,5*subplotRows))
for i in range(1,len(model.layers)):
print('Running tSNE for layer {}/{}'.format(i+1,len(model.layers)))
func = K.function([model.layers[0].input], [model.layers[i].output])
out = func([data])[0]
tsneModel = TSNE(n_components = 2, random_state = 0)
tsneOut = tsneModel.fit_transform(out).T
# labeledTsneOut = np.hstack((tsneOut, labels[0].reshape(-1,1)))
for j in range(numLabels):
plt.subplot(subplotRows, subplotCols, subplotIdx)
plt.title('{} / {}'.format(model.layers[i].name,cc.exp['params']['data']['labels'][j]))
plt.scatter(tsneOut[0],tsneOut[1],c=labels[j],cmap = 'plasma')
subplotIdx += 1
# tsneDF = pd.DataFrame(labeledTsneOut, columns = ('a', 'b', 'c'))
# plot = tsneDF.plot.scatter(x = 'a', y = 'b', c = 'c', cmap = 'plasma')
plt.tight_layout()
plt.savefig('{}/activations.png'.format(cc.cfg['plots']['layer_activations_dir']))
plt.close()
print('...done')
def histograms(modelLogger):
if not cc.cfg['plots']['histograms']:
return
if not os.path.exists(cc.cfg['plots']['histograms_dir']):
os.makedirs(cc.cfg['plots']['histograms_dir'])
cntEpochs = len(modelLogger.epochLogs)
cntLayers = len(modelLogger.epochLogs[-1]['weights'])
logVals = [
{'name':'weights','color':'blue'},
{'name':'updates','color':'red'},
{'name':'ratios','color':'green'}
]
subplotRows = len(logVals)
subplotCols = cntLayers
for x in range(cntEpochs):
subplotIdx = 1
plt.figure(figsize=(5*subplotCols,5*subplotRows))
plt.suptitle('Histograms per layer, epoch {}/{}'.format(x,cntEpochs-1), fontsize=14)
for logVal in logVals:
for i,layer in enumerate(modelLogger.epochLogs[x][logVal['name']]):
histmin = layer.min()
histmax = layer.max()
plt.subplot(subplotRows, subplotCols, subplotIdx)
plt.title('{}, {}'.format(modelLogger.model.layers[modelLogger.loggedLayers[i]].name,logVal['name']))
plt.hist(layer, range = (histmin, histmax), bins = 30, color = logVal['color'])
subplotIdx+=1
plt.tight_layout()
plt.subplots_adjust(top=0.9)
plt.savefig('{}/hist_{e:03d}.png'.format(cc.cfg['plots']['histograms_dir'], e = x))
plt.close()
def run_xsc_phot(bcdphot_out_path, mosaic_path):
replaced = {}
for cat in find_files(bcdphot_out_path, "*_combined_hdr_catalog.txt"):
print("\n======================================================")
print("\nadjusting photometry in: {}".format(cat.split('/')[-1]))
print("------------------------------------------------------")
outpath = cat.replace('combined_hdr_catalog.txt', '2mass_xsc.tbl')
# retrieve 2mass data if file doesn't already exist (from previous run)
if not os.path.isfile(outpath):
# get url and retrieve data
url = query_2mass_xsc_polygon(*get_region_corners(cat))
print("\ndownloading 2MASS photometry from: {}".format(url))
text = urllib2.urlopen(url).read()
# write to disk
with open(outpath, 'w') as f:
f.write(text)
print("\ncreated file: {}".format(outpath))
# read back in as recarray
print("\nreading: {}".format(outpath))
names = open(outpath).read().split('\n')[76].split('|')[1:-1]
da = np.recfromtxt(outpath, skip_header=80, names=names)
# write input file for xsc_phot.pro
infile_outpath = '/'.join(cat.split('/')[:-1]) + '/xsc.txt'
with open(infile_outpath, 'w') as w:
for i in range(da.shape[0]):
w.write("{} {} {} {}\n".format(da.designation[i], da.ra[i],
da.dec[i], da.r_ext[i]))
print(
"\ncreated input file for xsc_phot.pro: {}".format(infile_outpath))
# locate the FITS mosaic file for xsc_phot.pro to do photometry on
reg, ch = cat.split('/')[-1].split('_')[:2]
mosaicfile = filter(lambda x: 'dirbe{}/ch{}/long/full/Combine'\
.format(reg,ch) in x, find_files(mosaic_path, '*mosaic.fits'))[0]
print("\nfound mosaic file: {}".format(mosaicfile))
# spawn IDL subprocess running xsc_phot.pro and catch stdout in file
outpath = infile_outpath.replace('xsc.txt', 'xsc_phot_out.txt')
if not os.path.isfile(outpath):
outfile = open(outpath, 'w')
print("\nspawning xsc_phot.pro IDL subprocess")
cmd = "xsc_phot,'" + mosaicfile + "','" + infile_outpath + "','long'"
rc = subprocess.call(
['/usr/local/itt/idl71/bin/idl', '-quiet', '-e', cmd],
stderr=subprocess.PIPE,
stdout=outfile)
outfile.close()
# read in output to recarray
print("\nreading: {}".format(outpath))
phot = np.recfromtxt(outpath,
names=['id', 'flux', 'unc', 'sky', 'skyunc'])
# make sure rows are aligned
assert (da.designation == phot.id).all()
# ignore xsc sources we got a NaN or negative flux for
bad = np.isnan(phot.flux) | (phot.flux < 0)
print("\naper.pro returned NaN or negative flux for {} sources".format(
bad.sum()))
if bad.sum() > 0:
for i in phot[bad].id:
print(i)
outpath = cat.replace('combined_hdr_catalog.txt',
'xsc_nan_phot.csv')
with open(outpath, 'w') as f:
w = csv.writer(f)
w.writerow(da.dtype.names)
w.writerows(da[bad].tolist())
print('\ncreated file: {}'.format(outpath))
phot = phot[~bad]
da = da[~bad]
# read in pipeline catalog
print("\nreading: {}".format(cat))
names = open(cat).readline().split()[1:]
c = np.recfromtxt(cat, names=names)
# loop through xsc sources and find matches in pipeline catalog
print(
"\nfinding records associated with XSC sources in pipeline catalog"
)
c_flux_total = []
n_in_aper = []
c_idx = []
coords = radec_to_coords(c.ra, c.dec)
kdt = KDT(coords)
for i in range(phot.size):
radius = da.r_ext[i] / 3600.
# idx1, idx2, ds = spherematch(da.ra[i], da.dec[i],
# c.ra, c.dec, tolerance=radius)
idx, ds = spherematch2(da.ra[i],
da.dec[i],
c.ra,
c.dec,
kdt,
tolerance=radius,
k=500)
# c_flux_total.append(c.flux[idx2].sum())
# n_in_aper.append(c.flux[idx2].size)
# c_idx.append(idx2.tolist())
c_flux_total.append(c.flux[idx].sum())
n_in_aper.append(ds.size)
c_idx.append(idx.tolist())
print("\nhistogram of source counts in r_ext aperture")
for i in [(i, n_in_aper.count(i)) for i in set(n_in_aper)]:
print i
# create new version of catalog file with xsc-associated entries replaced
c_idx = np.array(flatten(c_idx))
print("\nremoving {}, adding {}".format(c_idx.size, phot.size))
replaced[cat] = {'old': c_idx.size, 'new': phot.size}
replaced[cat]['hist'] = [(i, n_in_aper.count(i))
for i in set(n_in_aper)]
c = np.delete(c, c_idx)
newrows = np.rec.array([(-i, da.ra[i], da.dec[i],
phot.flux[i], phot.unc[i], 1) for i in \
range(phot.size)], dtype=c.dtype)
newcat = np.hstack((c, newrows))
# write new version of catalog to disk
fmt = ['%i'] + ['%0.8f'] * 2 + ['%.4e'] * 2 + ['%i']
outpath = cat.replace('catalog.txt', 'catalog_xsc_cor.txt')
np.savetxt(outpath, newcat, fmt=fmt, header=' '.join(names))
print('\ncreated file: {}'.format(outpath))
# make plot of total old vs. new flux
plt.scatter(c_flux_total, phot.flux)
ylim = plt.gca().get_ylim()
plt.xlim(*ylim)
max_y = ylim[1]
plt.plot(ylim, ylim, 'r-')
plt.xlabel('old flux [mJy]')
plt.ylabel('new flux [mJy]')
name = ' '.join(cat.split('/')[-1].split('_')[:2])
plt.title(name)
outpath = cat.replace('combined_hdr_catalog.txt',
'xsc_new_vs_old_phot.png')
plt.savefig(outpath, dpi=200)
plt.close()
print('\ncreated file: {}'.format(outpath))
outfile = 'xsc_replaced.json'
json.dump(replaced, open(outfile, 'w'))
print("\ncreated file: {}".format(outfile))
print("\nremoved / added")
for k, v in replaced.iteritems():
print k.split('/')[-1], v['old'], v['new']
m = np.mean([i['old'] / float(i['new']) for i in replaced.values()])
print("average ratio: {}".format(m))
print("\nK mag and r_ext of sources with NaN photometry:")
for i in find_files(bcdphot_out_path, "*xsc_nan_phot.csv"):
reg = i.split('/')[-1]
rec = np.recfromcsv(i)
bad_id = rec.designation.tolist()
bad_k = rec.k_m_k20fe.tolist()
bad_r_ext = rec.r_ext.tolist()
print reg
print("\tid\t\t\tKmag\tr_ext")
if type(bad_id) is list:
seq = sorted(zip(bad_id, bad_k, bad_r_ext), key=lambda x: x[0])
for j, k, l in seq:
print("\t{}\t{}\t{}".format(j, k, l))
else:
print("\t{}\t{}\t{}".format(bad_id, bad_k, bad_r_ext))
# For all the spines, make their line thicker and return them to be black
all_spines = ['top', 'left', 'bottom', 'right']
for spine in all_spines:
ax.spines[spine].set_linewidth(0.5)
#ax.spines[spine].set_color('black')
# Get back the ticks. The position of the numbers is informative enough of
# the position of the value.
ax.xaxis.set_ticks_position('both')
ax.yaxis.set_ticks_position('both')
ax.tick_params('both', length=15, width=1, which='major')
ax.tick_params('both', length=5, width=1, which='minor')
plt.xlabel("SDSS "+band+" magnitude, mag")
plt.ylabel("GC "+band+" magnitude, mag")
plt.savefig('test/'+band+'_SDSS_vs_GC')
#print band, "mean magnitude difference: ", np.mean(petroMags[:, i]) - np.mean(mags[:, i])
fig = plt.figure(figsize=(14, 14))
for i, band in enumerate(bands):
lim_lo = limits[i][0]
lim_hi = limits[i][1]
x, m, b = makeDiagonalLine([lim_lo, lim_hi])
print i, lim_lo, lim_hi
ax = plt.subplot(5, 1, i+1)
c = ax.scatter(petroMags[:, i], mags[:, i], c='k', s=8, edgecolor='k')
ax.axis([lim_lo, lim_hi, lim_lo, lim_hi])
ax.errorbar(petroMags[:, i], mags[:, i], yerr=mag_err[:, i], mew=0, linestyle = "none", color="black")
plt.plot(x,m*x + b, c='k')
ax.xaxis.set_major_locator(majorLocator)
eps = data[0]
delta = 0.1
string = [r'$\tau\delta$ = %.2f' % float(delta*t) for t in taus]
#string2 = [r'$\tau$ = '+str(data[3][i]) for i in range(5)]
maxmmse = np.max(eps)
#maxf = np.max(data[1])
def f(x):
ppl.plot(alphas,eps[:,x]/maxmmse,label=string[x],ax=ax1)
# ax2.plot(data[2],data[1][:,x]/maxf,label=string2[x])
#plt.close()
#plt.figure()
#ax1 = plt.gcf().add_subplot(1,1,1)
fig, ax1 = ppl.subplots(1)
plt.title(r'MMSE for a Reconstruction Task with Adaptive Neurons')
#ax2 = plt.gcf().add_subplot(1,2,2)
#plt.title(r'Firing rate of adaptive code')
map(f,range(0,10,2))
ppl.legend(ax1)
#ax2.legend()
ax1.set_xlabel(r'$\alpha$')
ax1.set_ylabel(r'MMSE')
#ax2.set_xlabel(r'$\alpha$')
#ax2.set_ylabel(r'Firing Rate')
#plt.xlabel(r'Tuning width $\alpha$')
#plt.ylabel(r'MMSE')
plt.show()
#plt.savefig("mmse_discrimination.png",dpi=600)
plt.savefig("figure_5_8.png",dpi=600)
plt.savefig("figure_5_8.eps")
def run_xsc_phot(bcdphot_out_path, mosaic_path):
replaced = {}
for cat in find_files(bcdphot_out_path, "*_combined_hdr_catalog.txt"):
print("\n======================================================")
print("\nadjusting photometry in: {}".format(cat.split('/')[-1]))
print("------------------------------------------------------")
outpath = cat.replace('combined_hdr_catalog.txt','2mass_xsc.tbl')
# retrieve 2mass data if file doesn't already exist (from previous run)
if not os.path.isfile(outpath):
# get url and retrieve data
url = query_2mass_xsc_polygon(*get_region_corners(cat))
print("\ndownloading 2MASS photometry from: {}".format(url))
text = urllib2.urlopen(url).read()
# write to disk
with open(outpath, 'w') as f:
f.write(text)
print("\ncreated file: {}".format(outpath))
# read back in as recarray
print("\nreading: {}".format(outpath))
names = open(outpath).read().split('\n')[76].split('|')[1:-1]
da = np.recfromtxt(outpath, skip_header=80, names=names)
# write input file for xsc_phot.pro
infile_outpath = '/'.join(cat.split('/')[:-1])+'/xsc.txt'
with open(infile_outpath,'w') as w:
for i in range(da.shape[0]):
w.write("{} {} {} {}\n".format(da.designation[i], da.ra[i], da.dec[i], da.r_ext[i]))
print("\ncreated input file for xsc_phot.pro: {}".format(infile_outpath))
# locate the FITS mosaic file for xsc_phot.pro to do photometry on
reg, ch = cat.split('/')[-1].split('_')[:2]
mosaicfile = filter(lambda x: 'dirbe{}/ch{}/long/full/Combine'\
.format(reg,ch) in x, find_files(mosaic_path, '*mosaic.fits'))[0]
print("\nfound mosaic file: {}".format(mosaicfile))
# spawn IDL subprocess running xsc_phot.pro and catch stdout in file
outpath = infile_outpath.replace('xsc.txt', 'xsc_phot_out.txt')
if not os.path.isfile(outpath):
outfile = open(outpath,'w')
print("\nspawning xsc_phot.pro IDL subprocess")
cmd = "xsc_phot,'"+mosaicfile+"','"+infile_outpath+"','long'"
rc = subprocess.call(['/usr/local/itt/idl71/bin/idl','-quiet','-e',cmd],
stderr = subprocess.PIPE, stdout = outfile)
outfile.close()
# read in output to recarray
print("\nreading: {}".format(outpath))
phot = np.recfromtxt(outpath, names=['id','flux','unc','sky','skyunc'])
# make sure rows are aligned
assert (da.designation == phot.id).all()
# ignore xsc sources we got a NaN or negative flux for
bad = np.isnan(phot.flux) | (phot.flux < 0)
print("\naper.pro returned NaN or negative flux for {} sources".format(bad.sum()))
if bad.sum() > 0:
for i in phot[bad].id:
print(i)
outpath = cat.replace('combined_hdr_catalog.txt','xsc_nan_phot.csv')
with open(outpath,'w') as f:
w = csv.writer(f)
w.writerow(da.dtype.names)
w.writerows(da[bad].tolist())
print('\ncreated file: {}'.format(outpath))
phot = phot[~bad]
da = da[~bad]
# read in pipeline catalog
print("\nreading: {}".format(cat))
names = open(cat).readline().split()[1:]
c = np.recfromtxt(cat, names=names)
# loop through xsc sources and find matches in pipeline catalog
print("\nfinding records associated with XSC sources in pipeline catalog")
c_flux_total = []
n_in_aper = []
c_idx = []
coords = radec_to_coords(c.ra, c.dec)
kdt = KDT(coords)
for i in range(phot.size):
radius = da.r_ext[i]/3600.
# idx1, idx2, ds = spherematch(da.ra[i], da.dec[i],
# c.ra, c.dec, tolerance=radius)
idx, ds = spherematch2(da.ra[i], da.dec[i], c.ra, c.dec,
kdt, tolerance=radius, k=500)
# c_flux_total.append(c.flux[idx2].sum())
# n_in_aper.append(c.flux[idx2].size)
# c_idx.append(idx2.tolist())
c_flux_total.append(c.flux[idx].sum())
n_in_aper.append(ds.size)
c_idx.append(idx.tolist())
print("\nhistogram of source counts in r_ext aperture")
for i in [(i,n_in_aper.count(i)) for i in set(n_in_aper)]:
print i
# create new version of catalog file with xsc-associated entries replaced
c_idx = np.array(flatten(c_idx))
print("\nremoving {}, adding {}".format(c_idx.size, phot.size))
replaced[cat] = {'old':c_idx.size, 'new':phot.size}
replaced[cat]['hist'] = [(i,n_in_aper.count(i)) for i in set(n_in_aper)]
c = np.delete(c, c_idx)
newrows = np.rec.array([(-i, da.ra[i], da.dec[i],
phot.flux[i], phot.unc[i], 1) for i in \
range(phot.size)], dtype=c.dtype)
newcat = np.hstack((c, newrows))
# write new version of catalog to disk
fmt = ['%i']+['%0.8f']*2+['%.4e']*2+['%i']
outpath = cat.replace('catalog.txt', 'catalog_xsc_cor.txt')
np.savetxt(outpath, newcat, fmt = fmt, header = ' '.join(names))
print('\ncreated file: {}'.format(outpath))
# make plot of total old vs. new flux
plt.scatter(c_flux_total, phot.flux)
ylim = plt.gca().get_ylim()
plt.xlim(*ylim)
max_y = ylim[1]
plt.plot(ylim, ylim, 'r-')
plt.xlabel('old flux [mJy]')
plt.ylabel('new flux [mJy]')
name = ' '.join(cat.split('/')[-1].split('_')[:2])
plt.title(name)
outpath = cat.replace('combined_hdr_catalog.txt','xsc_new_vs_old_phot.png')
plt.savefig(outpath, dpi=200)
plt.close()
print('\ncreated file: {}'.format(outpath))
outfile = 'xsc_replaced.json'
json.dump(replaced, open(outfile,'w'))
print("\ncreated file: {}".format(outfile))
print("\nremoved / added")
for k,v in replaced.iteritems():
print k.split('/')[-1], v['old'], v['new']
m = np.mean([i['old']/float(i['new']) for i in replaced.values()])
print("average ratio: {}".format(m))
print("\nK mag and r_ext of sources with NaN photometry:")
for i in find_files(bcdphot_out_path, "*xsc_nan_phot.csv"):
reg = i.split('/')[-1]
rec = np.recfromcsv(i)
bad_id = rec.designation.tolist()
bad_k = rec.k_m_k20fe.tolist()
bad_r_ext = rec.r_ext.tolist()
print reg
print ("\tid\t\t\tKmag\tr_ext")
if type(bad_id) is list:
seq = sorted(zip(bad_id, bad_k, bad_r_ext), key=lambda x: x[0])
for j,k,l in seq:
print("\t{}\t{}\t{}".format(j,k,l))
else:
print("\t{}\t{}\t{}".format(bad_id, bad_k, bad_r_ext))
[78, 376],
[79, 164],
[80, 194],
[81, 342],
[82, 314],
[83, 68],
[84, 54],
[85, 131],
[86, 52],
[87, 220],
[88, 145],
[89, 63],
[90, 608],
[91, 68],
[92, 155],
[93, 49],
[94, 108],
[95, 59],
[96, 67],
[97, 58],
[98, 42],
[99, 210],
[100, 174]])
plt.figure(figsize=(5,5))
plt.scatter(histdata.T[0],histdata.T[1])
plt.savefig('hist.png')
plt.close()
sys.stderr.write('{} files contain {} bytes over {} files\n'.format(ftype.upper(), sum(fsize), len(files[ftype])))
###
# To upload to the correct spot on S3
# gzip *.paths
# s3cmd put --acl-public *.paths.gz s3://aws-publicdatasets/common-crawl/crawl-data/CC-MAIN-YYYY-WW/
###
# Plot
for ftype, fsize in size.items():
if not fsize:
continue
plt.hist(fsize, bins=50)
plt.xlabel('Size (bytes)')
plt.ylabel('Count')
plt.title('Distribution for {}'.format(ftype.upper()))
plt.savefig(prefix + '{}_dist.pdf'.format(ftype))
#plt.show(block=True)
plt.close()
###
# Find missing WAT / WET files
warc = set([x.strip() for x in open(prefix + 'warc.paths').readlines()])
wat = [x.strip() for x in open(prefix + 'wat.paths').readlines()]
wat = set([x.replace('.warc.wat.', '.warc.').replace('/wat/', '/warc/') for x in wat])
# Work out the missing files and segments
missing = sorted(warc - wat)
missing_segments = defaultdict(list)
for fn in missing:
start, suffix = fn.split('/warc/')
segment = start.split('/')[-1]
missing_segments[segment].append(fn)
width = 0.5
font = {'family' : 'normal',
'weight' : 'normal',
'size' : 10}
# k c b m r o char
vals = [ 45174 , 48810 , 41719 , 255017 , 642090 , 3053979 , 3066135 , 1731655 , 582226 , 126045 , 6738 ]
total = float(sum(vals))
print(vals)
vals = [(i / total)*100 for i in vals]
print(vals)
colors = ['LightGray','Gray','LightSkyBlue','RoyalBlue','LightSeaGreen','MediumSeaGreen','Khaki',\
'Goldenrod','Tomato','MediumVioletRed','DarkViolet']
#plt.bar(ind, vals, width)
ppl.bar(ax, ind, vals, color=colors)
plt.xticks(ind+width/2.,\
('1e-7','1e-6','1e-5','1e-4','0.001','0.005','0.01', '0.05', '0.1', '0.2', '0.5')\
,fontsize=11)
plt.yticks(fontsize=11)
plt.title("Distribution of cells by mutation rate across resistant populations",fontsize=11)
#plt.ylim((0,100))
plt.ylabel("Percent of Total",fontsize=11)
plt.xlabel("Initial Mutation rate: u",fontsize=11)
plt.savefig(opt.filename+"_bar")
plt.clf()
###
y = eta**2/(2*gamma)
print nd, " of ",len(deltas)
for nphi,phi in enumerate(phis):
fun = lambda x: f(x,gamma,eta,phi,alpha,thetas,tau,delta)
y = opt.fsolve(fun, y)
epss[nd,nphi] = y/(eta**2/(2*gamma))
ls[nd,nphi] = lambdabar(thetas,gamma,eta,phi,alpha,tau,delta)
strings = [r'$\tau\delta$ = '+str(i*tau) for i in deltas]
#f = lambda i : plt.plot(alphas,epss[i,:],label=strings[i])
#map(f,range(len(deltas)))
fig, ax = ppl.subplots(1,figsize=(7,7))
ppl.plot(ls[0,:],epss[0,:],label = strings[0], ax=ax)
ppl.plot(ls[1,:],epss[1,:],label = strings[1], ax=ax)
ppl.plot(ls[2,:],epss[2,:],label = strings[2], ax=ax)
ppl.plot(ls[3,:],epss[3,:],label = strings[3], ax=ax)
ppl.plot(ls[4,:],epss[4,:],label = strings[4], ax=ax)
np.savez("../data/mean_field_ratedist.npz", eps = epss, ls=ls, phis = phis, deltas = deltas, tau = tau)
ax.set_xlim([0.0,25.0])
ax.set_xlabel(r'$\lambda$ in spikes/sec')
ax.set_ylabel(r'MMSE/$\epsilon_0$')
plt.title("Rate-Distortion Curve for Adaptive Neurons")
ppl.legend(ax)
plt.savefig("rate_distortion_mf_alpha.png")
plt.savefig("rate_distortion_mf_alpha.eps")
plt.show()
else:
spiketimes = []
thetas = []
ax1.plot(spiketimes,thetas,'yo',label='Spike times')
ax1.plot(times,mg,'b',label='Mean prediction')
ax1.set_title('Gaussian Filter')
ax1.set_ylabel('Signal space')
ax1.legend()
ax2 = plt.gcf().add_subplot(2,1,2)
ax2.plot(times,sp,'r',label='Signal')
if sum(sum(spsp)) !=0:
(tsp,neursp) = np.where(spsp == 1)
spiketimesp = times[tsp]
thetasp = [code.neurons[i].theta for i in neursp]
else:
spiketimesp = []
thetasp = []
ax2.plot(times,mp,label='Mean prediction')
ppl.fill_between(times,mp-np.sqrt(varp),mp+np.sqrt(varp),ax=ax2)
ax2.plot(spiketimesp,thetasp,'.',label='Spike times')
ax2.set_ylabel('Signal space')
ax2.set_xlabel('Time')
ax2.legend()
ax2.set_title('Particle Filter')
plt.savefig('filtering.png',dpi=150)
ax1.set_xlabel(r'$\alpha$')
ax2.set_ylabel(r'$\phi$')
l1, = ppl.plot(alphas, eps[:,1], label=r'$\phi = '+str(phis[1])+r'$',ax=ax2)
c1 = l1.get_color()
ppl.plot(alphas,stoc_eps_02, '.-', color=c1)
indmin = numpy.argmin(eps[:,1])
ppl.plot(alphas[indmin],eps[indmin,1],'o',color=c1)
l2, = ppl.plot(alphas, eps[:,9], label=r'$\phi = '+str(phis[9])+r'$',ax=ax2)
c2 = l2.get_color()
ppl.plot(alphas,stoc_eps_10, '.-', color=c2)
indmin = numpy.argmin(eps[:,9])
ppl.plot(alphas[indmin],eps[indmin,9],'o',color=c2)
l3, = ppl.plot(alphas, eps[:,-1], label=r'$\phi = '+str(phis[-1])+r'$',ax=ax2)
c3 = l3.get_color()
ppl.plot(alphas,stoc_eps_20, '.-', color=c3)
indmin = numpy.argmin(eps[:,-1])
ppl.plot(alphas[indmin],eps[indmin,-1],'o',color=c3)
ppl.legend(ax2)
ax2.set_xlabel(r'$\alpha$')
ax2.set_ylabel(r'$\epsilon$')
plt.savefig("figure_5_7.eps")
l1, = ax1.plot(spiketimes,thetas,'o',label='Observed Spikes')
l3 = ppl.fill_between(times,mg-stg,mg+stg,ax=ax1,alpha=0.2)
c1 = l1.get_color()
c2 = l2.get_color()
c3 = l3.get_facecolor()
c4 = l4.get_color()
ax1.set_title('Gaussian Assumed Density Filter')
ax1.set_ylabel('Position [cm] (Preferred Stimulus)')
ax1.set_xlabel('Time [s]')
ppl.legend(ax1).get_frame().set_alpha(0.6)
thetas = [code.neurons[i].theta for i in sptrain]
ax2.plot(times,s,color=c4,label = 'True Sate')
ax2.plot(times,m,color=c2,label='Posterior Mean')
ax2.plot(times[sptimes],thetas,'o',color=c1,label='Observed Spikes')
ppl.fill_between(times,m-st,m+st,ax=ax2,facecolor=c3,alpha=0.2)
ax2.set_ylabel('Position [cm] (Preferred Stimulus)')
ax2.set_xlabel('Time [s]')
ppl.legend(ax2).get_frame().set_alpha(0.6)
ax2.set_title('Particle Filter')
ax1.set_ylim([(m-st).min()-0.5,(m+st).max()+0.5])
ax2.set_ylim([(m-st).min()-0.5,(m+st).max()+0.5])
#plt.show()
plt.savefig('filtering_both.pdf')
plt.savefig('filtering_both.eps')
plt.savefig('filtering_both.png')
##
import sys
if __name__ == '__main__':
# Accumulate the counts fed into standard in (stdin)
counts = []
for line in sys.stdin:
topic, count = line.split()
# Shift the ones up by a tiny amount to allow them to be visible on the graph
counts.append(int(count) + 0.05)
print 'Total page view counts: {}'.format(len(counts))
# Display the hourly page view distribution on a log-log plot
# This matches our friendly and well understood Zipf distribution
fig = plt.figure(figsize=(9, 5))
ax = fig.add_subplot(1, 1, 1)
plt.plot(xrange(len(counts)), counts, linewidth=3, label='Pageviews per page')
#
ax.set_xscale('log')
ax.set_yscale('log')
#
plt.title('Log-log plot of hourly Wikipedia page view distribution')
plt.xlabel('Rank order')
plt.ylabel('Frequency')
plt.grid(color='black')
plt.legend()
#
plt.savefig('hourly_wikipedia_zipf.pdf')
plt.show(block=True)
thetas = np.arange(-7.0,7.0,dtheta)
xs = np.arange(-2.2,2.2,0.01)
rates1 = [[rate(x,0.3,phi,t) for x in xs] for t in thetas]
rates2 = [[rate(x,1.0,phi,t) for x in xs] for t in thetas]
rates3 = [[rate(x,2.0,phi,t) for x in xs] for t in thetas]
plt.rcParams['text.usetex']=True
ax1.plot( alphas, eps )
ax1.set_xlabel(r'$\alpha$')
ax1.set_ylabel(r'$MMSE$')
map(lambda x : ax2.plot(xs,x), rates1)
map(lambda x : ax3.plot(xs,x), rates2)
map(lambda x : ax4.plot(xs,x), rates3)
cm.getMaternSample(gamma=gamma,eta=eta,order=1,phi=phi,dtheta=dtheta,plot=True,ax=ax5,alpha=0.3, timewindow=20000)
cm.getMaternSample(gamma=gamma,eta=eta,order=1,phi=phi,dtheta=dtheta,plot=True,ax=ax6,alpha=1.0, timewindow=20000)
cm.getMaternSample(gamma=gamma,eta=eta,order=1,phi=phi,dtheta=dtheta,plot=True,ax=ax7,alpha=2.0, timewindow=20000)
map(lambda x : x.set_ylabel(r'$Rate$'),[ax2,ax3,ax4])
map(lambda x : x.set_ylim([-0.01,1.7*phi]),[ax2,ax3,ax4])
ax2.set_title('Coding Strategies')
ax2.legend([r'$\alpha=0.3$'])
ax3.legend([r'$\alpha=1.0$'])
ax4.legend([r'$\alpha=2.0$'])
ax4.set_xlabel(r'$x$')
plt.savefig('estimation_uni.png',dpi=300)
plt.savefig('estimation_uni.eps')
import numpy
from prettyplotlib import plt
if __name__=='__main__':
xs = numpy.arange(-5.0,5.0,0.001)
def f(x):
return 1.0*numpy.exp(-x**2/2)
def fdash(x):
return -x*numpy.exp(-x**2/2)
tuning = f(xs)
info = fdash(xs)**2/f(xs)
plt.plot(xs,tuning,label='Tuning function')
plt.plot(xs,info,label='Fisher Information')
plt.gca().set_xlabel('x')
plt.legend()
plt.savefig('../figures/figure_3_5.eps')
matern0 = matern1
matern1 = IterateOnce(matern0, K_matern, dx,alpha =0.1)
dist = SquaredDistance(matern0,matern1)
print dist
OU0 = K_ou(xs)
OU1 = IterateOnce(OU0, K_ou, dx,alpha=0.1)
dist = SquaredDistance(OU0,OU1)
while dist > 1e-10:
OU0 = OU1
OU1 = IterateOnce(OU0, K_ou, dx,alpha =0.1)
dist = SquaredDistance(OU0,OU1)
print dist
ax1 = plt.subplot(311)
ax2 = plt.subplot(312)
ax3 = plt.subplot(313)
plt.gcf().suptitle("Posterior kernels")
ax1.plot(xs,K_ou(xs),label=r'OU prior kernel')
ax1.plot(xs,OU1,label = r'OU posterior kernel')
ax1.legend()
ax2.plot(xs,K_matern(xs),label=r'Matern prior kernel')
ax2.plot(xs,matern1, label=r'Matern posterior kernel')
ax2.legend()
ax3.plot(xs,K_rbf(xs),label=r'RBF prior kernel')
ax3.plot(xs,rbf1,label=r'RBF posterior kernel')
ax3.legend()
plt.savefig("figure_3_3.eps")
ppl.plot(ts, Xcrit2[0,:,0], label=r'$\omega$ = 2.0', ax=axcrit)
ppl.plot(ts, Xcrit4[0,:,0], label=r'$\omega$ = 4.0', ax=axcrit)
p = ppl.legend(axcrit)
fr = p.get_frame()
fr.set_alpha(0.4)
axcrit.set_title("Critical")
A_over = numpy.array([[0.0,1.0],[-2.0,-5.0]])
H_over = numpy.array([[0.0,0.0],[0.0,1.5]])
ts,Xover2 = sample(X0,A_over, H_over, dt, N, 1)
ts,Xover1 = sample(X0,numpy.array([[0.0,1.0],[-1.0,-2.5]]), 0.5*H_over, dt,N,1)
ts,Xover4 = sample(X0,numpy.array([[0.0,1.0],[-4.0,-10.0]]), 2.0*H_over, dt,N,1)
ppl.plot(ts, Xover1[0,:,0], label=r'$\omega$ = 1.0', ax=axover)
ppl.plot(ts, Xover2[0,:,0], label=r'$\omega$ = 2.0', ax=axover)
ppl.plot(ts, Xover4[0,:,0], label=r'$\omega$ = 4.0', ax=axover)
p = ppl.legend(axover)
fr = p.get_frame()
fr.set_alpha(0.4)
axover.set_title("Overdamped")
axover.set_xlabel('Time [s]')
axcrit.set_ylabel('Position [cm]')
axou.set_ylabel('Position [cm]')
axcrit.set_xlabel('Time [s]')
plt.gcf().suptitle('Linear Stochastic Processes')
plt.savefig('../figures/figure_5_2.eps')
plt.savefig('../figures/figure_5_2.pdf')
ticklabels = np.round(ticks,decimals=2)
cb1 = plt.colorbar(p1,ax=ax2)
cb1.set_ticks(ticks)
cb1.set_ticklabels(ticklabels)
cb2 = plt.colorbar(p2,ax=ax3)
cb2.set_ticks(ticks)
cb2.set_ticklabels(ticklabels)
cb3 = plt.colorbar(p3,ax=ax4)
cb3.set_ticks(ticks)
cb3.set_ticklabels(ticklabels)
ax4.set_xlabel(r'$\Delta\theta$')
ax4.set_ylabel(r'$\alpha$')
ax3.set_ylabel(r'$\alpha$')
ax2.set_ylabel(r'$\alpha$')
ax3.axes.get_xaxis().set_ticks([])
ax2.axes.get_xaxis().set_ticks([])
#ax3.tick_params(axis='x',which='both',bottom='off')
#ax2.tick_params(axis='x',which='both',bottom='off')
ax2.set_title('ADF with dense coding assumption')
ax3.set_title('Full ADF')
ax4.set_title('Particle Filter')
[p.set_clim(vmin=mintotal,vmax=maxtotal) for p in [p1,p2,p3]]
plt.savefig('figure_5_14.pdf')
plt.plot(X, Y, label='iSVD u={}'.format(num))
"""
print 'Testing raw SVD => exact reconstruction'
svT = scipy.linalg.diagsvd(s, u.shape[0], vT.shape[1]).dot(vT)
for y in xrange(train.shape[0]):
for x in xrange(train.shape[1]):
colU = u[y, :]
rowV = svT[:, x]
assert np.allclose(train[y, x], single_dot(u, svT, x, y))
"""
##
plt.title('SVD reconstruction error on {}x{} matrix'.format(*train.shape))
plt.xlabel('Low rank approximation (k)')
plt.ylabel('Frobenius norm')
plt.ylim(0, max(svdY))
plt.legend(loc='best')
plt.savefig('reconstruct_fro_{}x{}.pdf'.format(*train.shape))
plt.show(block=True)
##
plt.plot(orthoX, orthoY, label="SVD", color='black', linewidth=2, linestyle='--')
for label, X, Y in incr_ortho:
plt.plot(X, Y, label=label)
plt.title('SVD orthogonality error on {}x{} matrix'.format(*train.shape))
plt.xlabel('Low rank approximation (k)')
plt.ylabel('Deviation from orthogonality')
plt.semilogy()
#plt.ylim(0, max(orthoY))
plt.legend(loc='best')
plt.savefig('reconstruct_ortho_{}x{}.pdf'.format(*train.shape))
plt.show(block=True)
print "stringing"
string = [r'$\phi$ = %.2f' % float(i) for i in data[3]]
maxmmse = np.max(data[0])
def f(x):
line, = ppl.plot(data[2],data[0][:,x],label=string[x],axes=ax1)
return line.get_color()
def get_min(data):
minim = np.min(data)
indmin = np.argmin(data)
return minim,indmin
print "subplotting"
fig, ax1 = ppl.subplots(1)
ax1.set_title(r'Classification')
colors = map(f,range(data[3].size))
for i in range(data[3].size):
mi,ind = get_min(data[0][:,i])
ppl.plot(data[2][ind],mi,'o',color=colors[i],axes=ax1)
ppl.legend(ax1)
ax1.set_xlabel(r'$\alpha$')
ax1.set_ylabel(r'MMSE')
plt.savefig("mmse_classification.eps")
np.random.shuffle(nodes)
print 'Pageviews from "real" edge weights'
print '-=-=-=-=-'
display_graph(G)
print
print 'Pageviews from evenly distributed edge weights'
print '-=-=-=-=-'
display_graph(approxG)
plt.plot(np.arange(0, len(rerrs)), rerrs, label='Relative error over time')
plt.xlabel('Iteration')
plt.ylabel('Average pageview relative error per node')
plt.legend()
plt.savefig('error_over_time.pdf')
plt.show(block=True)
plt.plot(np.arange(0, len(werrs)), werrs, label='Weight error over time')
plt.xlabel('Iteration')
plt.ylabel('Average weight error per edge')
plt.legend()
plt.savefig('weight_over_time.pdf')
plt.show(block=True)
fig, ax = plt.subplots(1)
ppl.bar(ax, *orig_weight_data, alpha=0.5, color='black', label='Weight error before')
ppl.bar(ax, np.arange(0, G.number_of_edges()), [abs(G[u][v]['weight'] - approxG[u][v]['weight']) for u, v in G.edges()], alpha=0.8, label='Weight error after')
#plt.ylim(-1, 1)
plt.legend(loc='best')
plt.show(block=True)
"""
print 'Testing raw SVD => exact reconstruction'
svT = scipy.linalg.diagsvd(s, u.shape[0], vT.shape[1]).dot(vT)
for y in xrange(train.shape[0]):
for x in xrange(train.shape[1]):
colU = u[y, :]
rowV = svT[:, x]
assert np.allclose(train[y, x], single_dot(u, svT, x, y))
"""
##
plt.title('SVD reconstruction error on {}x{} matrix'.format(*train.shape))
plt.xlabel('Low rank approximation (k)')
plt.ylabel('Frobenius norm')
plt.ylim(0, max(svdY))
plt.legend(loc='best')
plt.savefig('reconstruct_fro_{}x{}.pdf'.format(*train.shape))
plt.show(block=True)
##
plt.plot(orthoX,
orthoY,
label="SVD",
color='black',
linewidth=2,
linestyle='--')
for label, X, Y in incr_ortho:
plt.plot(X, Y, label=label)
plt.title('SVD orthogonality error on {}x{} matrix'.format(*train.shape))
plt.xlabel('Low rank approximation (k)')
plt.ylabel('Deviation from orthogonality')
plt.semilogy()
#plt.ylim(0, max(orthoY))
svdX = []
svdY = []
u, s, vT = scipy.linalg.svd(train)
for k in xrange(1, 100):
low_s = [s[i] for i in xrange(k)] # + (min(u.shape[0], vT.shape[1]) - k) * [0]
print 'Exact SVD with low-rank approximation {}'.format(k)
svdX.append(k)
svdY.append(get_error(u, np.diag(low_s), vT, train, test))
plt.plot(svdX, svdY, label="SVD", color='black', linewidth='2', linestyle='--')
"""
print
print 'Testing incremental SVD'
for num in xrange(400, 1001, 300):
print '... with block size of {}'.format(num)
X, Y = [], []
for k in xrange(1, 91, 10):
print k
u, s, vT = incremental_SVD(train, k, num)
X.append(k)
Y.append(get_error(u, s, vT, train, test, prod_avg))
plt.plot(X, Y, label='iSVD u={}'.format(num))
##
plt.title('Recommendation system RMSE on {}x{} matrix'.format(*train.shape))
plt.xlabel('Low rank approximation (k)')
plt.ylabel('Root Mean Squared Error')
#plt.ylim(0, max(svdY))
plt.legend(loc='best')
plt.savefig('recommend_rmse_{}x{}.pdf'.format(*train.shape))
plt.show(block=True)
OU1 = IterateOnce(OU0, K_ou, dx,alpha=0.1)
dist = SquaredDistance(OU0,OU1)
while dist > 1e-10:
OU0 = OU1
OU1 = IterateOnce(OU0, K_ou, dx,alpha =0.1)
dist = SquaredDistance(OU0,OU1)
print dist
fig, (ax1,ax2,ax3) = ppl.subplots(3,1)
#ax1 = plt.subplot(311)
#ax2 = plt.subplot(312)
#ax3 = plt.subplot(313)
plt.gcf().suptitle("Posterior kernels")
xs = xs[:6000]
OU1 = OU1[:6000]
matern1 = matern1[:6000]
rbf1 = rbf1[:6000]
ppl.plot(xs,K_ou(xs),label=r'OU prior kernel',ax=ax1)
ppl.plot(xs,OU1,label = r'OU posterior kernel',ax=ax1)
ppl.legend(ax1)
ppl.plot(xs,K_matern(xs),label=r'Matern prior kernel',ax=ax2)
ppl.plot(xs,matern1, label=r'Matern posterior kernel',ax=ax2)
ppl.legend(ax2)
ppl.plot(xs,K_rbf(xs),label=r'RBF prior kernel',ax=ax3)
ppl.plot(xs,rbf1,label=r'RBF posterior kernel',ax=ax3)
ppl.legend(ax3)
plt.savefig("../figures/figure_3_4.eps")
import prettyplotlib as ppl
from prettyplotlib import plt
def drift(x,eq):
return 4.0*x*(eq-x**2)
def bistablesample(x0,eq,eta,T,dt,NSamples):
xs = numpy.zeros((NSamples,T))
xs[:,0] = x0
s_dt = numpy.sqrt(dt)
for i in range(1,T):
rands = numpy.random.normal(0,1,(NSamples,))
xs[:,i] = xs[:,i-1]+dt*drift(xs[:,i-1],eq) + s_dt*eta*rands
return xs
if __name__=='__main__':
samples = bistablesample(0.0,1,0.9,300000,0.001,1)
ts = numpy.arange(0.0,30000*0.001,0.001)
print ts.shape
print samples.shape
fig, ax = ppl.subplots(1)
ppl.plot(ts,samples,ax=ax)
ax.set_xlabel('Time [s]')
ax.set_ylabel('Position [cm]')
plt.savefig('../figures/figure_5_10.eps')
plt.savefig('../figures/figure_5_10.png',dpi=300)
plt.show()
#plt.xticks(xticks,xlabels,axes=ax1)
#plt.yticks(yticks,ylabels,axes=ax1)
cb = plt.colorbar(p, ax=ax1)
cb.set_ticks(np.array([0.3,0.4,0.5]))
cb.set_ticklabels(np.array([0.3,0.4,0.5]))
ax1.set_xlabel(r'$\alpha$')
ax1.set_ylabel(r'$\phi$')
ax1.set_title(r'MMSE ($\epsilon$)')
l1,= ppl.plot( alphas, eps[:,10], label=r'$\phi = '+ str(phis[10]-0.001) + r'$',ax=ax2)
ppl.plot( alphas[np.argmin(eps[:,10])], np.min(eps[:,10]), 'o',color=l1.get_color(),ax=ax2)
l2, = ppl.plot( alphas, eps[:,20], label=r'$\phi = '+ str(phis[20]-0.001) + r'$',ax=ax2)
ppl.plot( alphas[np.argmin(eps[:,20])] , np.min(eps[:,20]), 'o',color=l2.get_color(),ax=ax2)
l3, = ppl.plot( alphas, eps[:,30], label=r'$\phi = '+ str(phis[30]-0.001) + r'$',ax=ax2)
ppl.plot( alphas[np.argmin(eps[:,30])] , np.min(eps[:,30]), 'o',color=l3.get_color(),ax=ax2)
l4, = ppl.plot( alphas, eps[:,40], label=r'$\phi = '+ str(phis[40]-0.001) + r'$',ax=ax2)
ppl.plot( alphas[np.argmin(eps[:,40])] , np.min(eps[:,40]), 'o',color=l4.get_color(),ax=ax2)
ppl.plot( alphas , stoc_eps[:,10], '-.',ax=ax2, color = l1.get_color())
ppl.plot( alphas , stoc_eps[:,20], '-.',ax=ax2, color = l2.get_color())
ppl.plot( alphas , stoc_eps[:,30], '-.',ax=ax2, color = l3.get_color())
ppl.plot( alphas , stoc_eps[:,40], '-.',ax=ax2, color = l4.get_color())
ax2.set_xlabel(r'$\alpha$')
ax2.set_ylabel(r'$\epsilon$')
ax2.set_title(r'MMSE as a function of $\alpha$')
ppl.legend(ax2).get_frame().set_alpha(0.7)
plt.savefig('../figures/figure_5_3.png',dpi=300)
plt.savefig('../figures/figure_5_3.pdf')
os.system("echo \"all done\" | mutt -a \"../figures/figure_5_3.eps\" -s \"Plot\" -- [email protected]")
svdY = []
u, s, vT = scipy.linalg.svd(train)
for k in xrange(1, 100):
low_s = [s[i] for i in xrange(k)] # + (min(u.shape[0], vT.shape[1]) - k) * [0]
print 'Exact SVD with low-rank approximation {}'.format(k)
svdX.append(k)
svdY.append(get_error(u, np.diag(low_s), vT, train, test))
plt.plot(svdX, svdY, label="SVD", color='black', linewidth='2', linestyle='--')
"""
print
print 'Testing incremental SVD'
for num in xrange(400, 1001, 300):
print '... with block size of {}'.format(num)
X, Y = [], []
for k in xrange(1, 91, 10):
print k
u, s, vT = incremental_SVD(train, k, num)
X.append(k)
Y.append(get_error(u, s, vT, train, test, prod_avg))
plt.plot(X, Y, label='iSVD u={}'.format(num))
##
plt.title(
'Recommendation system RMSE on {}x{} matrix'.format(*train.shape))
plt.xlabel('Low rank approximation (k)')
plt.ylabel('Root Mean Squared Error')
#plt.ylim(0, max(svdY))
plt.legend(loc='best')
plt.savefig('recommend_rmse_{}x{}.pdf'.format(*train.shape))
plt.show(block=True)
