def run(n = 3000):
if n == "short":
n = 50
with model:
trace = sample(n, step, start)
pl.figure()
pl.hexbin(trace['x'], trace['y'])
# lets plot the samples vs. the actual distribution
from theano import function
xn = 1500
yn = 1000
xs = np.linspace(-3, .25, xn)[np.newaxis, :]
ys = np.linspace(-1.5, 1.5, yn)[:, np.newaxis]
like = (xs + ys ** 2) ** 2 * N
post = np.exp(-.5 * (xs ** 2 + ys ** 2 + like))
post = post
pl.figure()
extent = np.min(xs), np.max(xs), np.min(ys), np.max(ys)
pl.imshow(post, extent=extent)
def plot(name,samples_x, samples_y):
pylab.hexbin(samples_x, samples_y, gridsize=50, bins=1000)
pylab.axis([-1.0, 1.0, -1.0, 1.0])
cb = pylab.colorbar()
pylab.xlabel('x')
pylab.ylabel('y')
pylab.title(name)
pylab.savefig('{0}.png'.format(name))
def main():
if len(sys.argv) < 2:
sys.exit('Usage: ' + sys.argv[0] + ' [Accounting files]')
else:
joblist = sys.argv[1:]
jobs = jobstats.alljobs(joblist)
qts = list()
wallts = list()
corehrs = list()
allqts = list()
for job in jobs:
if job.cores > 1 and job.tiq > 0 and not np.isnan(job.tiq):
allqts.append(np.log10(job.tiq/3600.0))
corehrs.append(np.log10(job.cores*job.walltimereq/3600.0))
if job.cores == 1 and job.tiq > 0 and not np.isnan(job.tiq):
qts.append(np.log10(job.tiq/3600.0))
wallts.append(np.log10(job.walltimereq/3600.0))
print('max qt:' + str(max(qts)))
print('avg qt:' + str(np.mean(qts)) + '+/-' + str(np.std(qts)/np.sqrt(len(qts))))
subx = list()
suby = list()
for i,j in zip(wallts,qts):
if j > -1.0 and i > -1.0:
subx.append(i)
suby.append(j)
z = np.polyfit(subx,suby,1)
p = np.poly1d(z)
plt.cla()
plt.hexbin(wallts, qts, bins='log')
plt.plot([min(qts), max(qts)], [min(qts), max(qts)], 'k--')
plt.plot(subx, p(subx), 'k')
plt.xlabel('log(Walltime (hours))')
plt.ylabel('log(Queue Time (hours))')
plt.savefig('QueueTimes.png')
subx = list()
suby = list()
for i,j in zip(corehrs,allqts):
if j > -1.0 and i > -1.0:
subx.append(i)
suby.append(j)
z = np.polyfit(subx,suby,1)
p = np.poly1d(z)
plt.cla()
plt.hexbin(corehrs, allqts, bins='log')
plt.plot([min(allqts), max(allqts)], [min(allqts), max(allqts)], 'k--')
plt.plot(subx, p(subx),'k')
plt.xlabel('log(Core Hours)')
plt.ylabel('log(Queue Time (hours))')
plt.savefig('CHvsQT.png')
def plotHexbin(dists, projected_dists, n_components):
pl.figure()
pl.hexbin(dists, projected_dists, gridsize=100)
pl.xlabel("Pairwise squared distances in original space")
pl.ylabel("Pairwise squared distances in projected sapce")
pl.title("Pairwise distances distribution for n_components=%d" \
% n_components)
cb = pl.colorbar()
cb.set_label('Sample pairs counts')
def showellipticitydensity(self,gridsize):
x =self.stars["X_IMAGE"]
y =self.stars["Y_IMAGE"]
ellipticity = self.stars["ELLIPTICITY"]*pixelscale
pylab.hexbin( \
x, y, numpy.flipud(ellipticity), \
gridsize=gridsize, \
vmin=0.0, \
vmax=0.3
)
pylab.colorbar()
pylab.title("ellipticity= %.3lf(+/-)%.3lf" \
% (numpy.median(ellipticity), ellipticity.std()/numpy.sqrt(ellipticity.size)) )
def showfwhmdensity(self,gridsize):
x =self.stars["X_IMAGE"]
y =self.stars["Y_IMAGE"]
fwhm = self.stars["FWHM_IMAGE"]*pixelscale
pylab.hexbin( \
x, y, numpy.flipud(fwhm), \
gridsize=gridsize, \
vmin=fwhmmin, \
vmax=fwhmmax
)
pylab.colorbar()
mfwhm = fwhm.mean()
dfwhm = fwhm.std()/numpy.sqrt(fwhm.size)
pylab.title("fwhm = %.2lf(+/-)%.2lf arcsec" % (mfwhm, dfwhm) )
return mfwhm, dfwhm
def drawHexbin(x_ls, y_ls, C_ls, fig_fname=None, gridsize=100, title=None, xlabel=None, ylabel=None,\
colorBarLabel=None, reduce_C_function=None, dpi=300, mincnt=None, marginals=False, xscale='linear',\
scale='linear'):
"""
2011.12.22
xscale: [ linear | log]
Use a linear or log10 scale on the horizontal axis.
scale: [ linear | log]
Use a linear or log10 scale on the vertical axis.
mincnt: None | a positive integer
If not None, only display cells with more than mincnt number of points in the cell
marginals: True|False
if marginals is True, plot the marginal density as colormapped rectagles along the bottom of the x-axis and left of the y-axis
2011-4-27
draw 2D histogram (reduce_C_function=logSum) or any 3D plot (3rd Dimension is determined by reduce_C_function).
default of reduce_C_function: numpy.median.
moved from variation/src/misc.py
2010-7-1
add argument reduce_C_function()
2010-6-28
add argument xlabel & ylabel
2010-5-11
"""
import pylab, numpy
import matplotlib.cm as cm
if reduce_C_function is None:
reduce_C_function = numpy.median
pylab.clf()
pylab.hexbin(x_ls, y_ls, C=C_ls, gridsize=gridsize, reduce_C_function=reduce_C_function, cmap=cm.jet,\
mincnt=mincnt, marginals=marginals, xscale=xscale)#, scale=scale
pylab.axis([min(x_ls), max(x_ls), min(y_ls), max(y_ls)])
if title is None:
title = "gridsize %s, %s probes." % (gridsize, len(x_ls))
pylab.title(title)
if xlabel:
pylab.xlabel(xlabel)
if ylabel:
pylab.ylabel(ylabel)
cb = pylab.colorbar()
if colorBarLabel:
cb.set_label(colorBarLabel)
if fig_fname:
pylab.savefig(fig_fname, dpi=dpi)
def plot_density(x, y, z=None, z_method=n.mean, zlimits=None, zscale=None,
gridsize=[360, 180], projectionmethod=hammer_project_toxy,
radecgrid=True, raCen=0,
xgridlim=None, ygridlim=None, newfig=True, cb_label=''):
"""Make a density plot of x/y, potentially with z of same length. Adds radec grid
in hammer_projection. gridsize specifies binning size within plot.
z_method (reduce_C_method), zlimits (min/max of z), and zscale (None/'log'/integer/sequence)
affect plotting of z value, if provided.
xgridlim, ygridlim specify limits on final plot (may be non-functioning). """
cmap = None
if newfig:
pyl.figure()
if z!=None:
if zlimits == None:
hx = pyl.hexbin(x, y, C=z, reduce_C_function=z_method, gridsize=gridsize,
bins=zscale, cmap=cmap)
else:
delta = 0.00001
hx = pyl.hexbin(x, y, C=z, reduce_C_function=z_method, gridsize=gridsize,
vmin=zlimits[0], vmax=zlimits[1]+delta, bins=zscale, cmap=cmap)
else:
if zlimits == None:
hx = pyl.hexbin(x, y, gridsize=gridsize, mincnt=0.5, cmap=cmap, bins=zscale)
else:
hx = pyl.hexbin(x, y, gridsize=gridsize, mincnt=0.5,
vmin=zlimits[0], vmax=zlimits[1], cmap=cmap, bins=zscale)
if radecgrid:
make_radec_grid(projectionmethod, raCen=raCen*deg2rad,
xgridlim=xgridlim, ygridlim=ygridlim, newfig=False)
pyl.xlabel("x")
pyl.ylabel("y")
if zlimits!=None:
dtick = (zlimits[1] - zlimits[0])/5.0
colorticks = n.arange(zlimits[0], zlimits[1]+dtick, dtick)
cb = pyl.colorbar(hx, orientation='horizontal', pad=.1, fraction=0.05, shrink=1, aspect=40,
ticks=colorticks)
else:
cb = pyl.colorbar(hx, orientation='horizontal', pad=.1, fraction=0.05, shrink=1, aspect=40)
if zscale == 'log':
cb.set_label('log$_{10}$(N)')
cb.set_label(cb_label)
return
def plot_density(x, y, z=None, z_method=n.mean, zlimits=None, zscale=None,
gridsize=[360, 180], projectionmethod=hammer_project_toxy,
radecgrid=True, raCen=0,
xgridlim=None, ygridlim=None, newfig=True, cb_label=''):
"""Make a density plot of x/y, potentially with z of same length. Adds radec grid
in hammer_projection. gridsize specifies binning size within plot.
z_method (reduce_C_method), zlimits (min/max of z), and zscale (None/'log'/integer/sequence)
affect plotting of z value, if provided.
xgridlim, ygridlim specify limits on final plot (may be non-functioning). """
cmap = None
if newfig:
pyl.figure()
if z!=None:
if zlimits == None:
hx = pyl.hexbin(x, y, C=z, reduce_C_function=z_method, gridsize=gridsize,
bins=zscale, cmap=cmap)
else:
delta = 0.00001
hx = pyl.hexbin(x, y, C=z, reduce_C_function=z_method, gridsize=gridsize,
vmin=zlimits[0], vmax=zlimits[1]+delta, bins=zscale, cmap=cmap)
else:
if zlimits == None:
hx = pyl.hexbin(x, y, gridsize=gridsize, mincnt=0.5, cmap=cmap, bins=zscale)
else:
hx = pyl.hexbin(x, y, gridsize=gridsize, mincnt=0.5,
vmin=zlimits[0], vmax=zlimits[1], cmap=cmap, bins=zscale)
if radecgrid:
make_radec_grid(projectionmethod, raCen=raCen*deg2rad,
xgridlim=xgridlim, ygridlim=ygridlim, newfig=False)
pyl.xlabel("x")
pyl.ylabel("y")
if zlimits!=None:
dtick = (zlimits[1] - zlimits[0])/5.0
colorticks = n.arange(zlimits[0], zlimits[1]+dtick, dtick)
cb = pyl.colorbar(hx, orientation='horizontal', pad=.1, fraction=0.05, shrink=1, aspect=40,
ticks=colorticks)
else:
cb = pyl.colorbar(hx, orientation='horizontal', pad=.1, fraction=0.05, shrink=1, aspect=40)
if zscale == 'log':
cb.set_label('log$_{10}$(N)')
cb.set_label(cb_label)
return
def plot_fit_mag(self):
"""
Plots the fit of sersic magnitudes to mag_auto ones
"""
# Extracts variables of interest
mag = self.cat['mag_auto'][self.mask]
I = np.log10(self.cat['sersicfit'][self.mask, 0])
R = np.log10(self.cat['sersicfit'][self.mask, 1])
n = np.log10(self.cat['sersicfit'][self.mask, 2])
q = np.log10(self.cat['sersicfit'][self.mask, 3])
plt.figure()
a, b = self.theta_mag
plt.hexbin(mag, (a + b * mag) - self._mag_sersic(I, R, n, q),
bins='log',
cmap='Blues',
extent=(17, 25.2, -2, 2))
plt.colorbar(label='Log(N)')
plt.axhline(0, color='red')
plt.xlabel('mag_auto')
plt.ylabel('mag_auto - mag_sersic, after affine correction')
def run(n=3000):
if n == "short":
n = 50
with model:
trace = sample(n, step, start)
pl.figure()
pl.hexbin(trace['x'], trace['y'])
# lets plot the samples vs. the actual distribution
xn = 1500
yn = 1000
xs = np.linspace(-3, .25, xn)[np.newaxis, :]
ys = np.linspace(-1.5, 1.5, yn)[:, np.newaxis]
like = (xs + ys ** 2) ** 2 * N
post = np.exp(-.5 * (xs ** 2 + ys ** 2 + like))
post = post
pl.figure()
extent = np.min(xs), np.max(xs), np.min(ys), np.max(ys)
pl.imshow(post, extent=extent)
def plot_bins(img, x_list, y_list):
pl.imshow(img, cmap=pl.get_cmap('Pastel1'))
pl.plot() # sub-plot area 2 out of 2
image = pl.hexbin(x_list, y_list, C=None, gridsize=config.curr_grid_size,
bins=None, mincnt=config.curr_clicks_threshold, edgecolors='none') # hexbinning
# pl.scatter(X,Y,lw=0.5,c='k',edgecolor='w') # overlaying the sample points
pl.axis('image') # necessary for correct aspect ratio
fig = pl.imshow(img)
if config.remove_axes:
fig.axes.get_xaxis().set_visible(False)
fig.axes.get_yaxis().set_visible(False)
pl.axis('off')
if config.show_plot:
pl.show()
return image
def spline_subtract_counts(file1, file2, outfile, outplot=None, soffset=0.0):
# given two gr files, do a spline fit to find the best mapping of file2 to file1, and then subtract from file1
# the fitted value at each point
# write the difference to outfile
# soffset is added to all of the fitted values prior to subtraction
# This is basically designed for finding differences between log scaled data sets (e.g., ipod vs inp log ratio files)
print "entering spline_subtract_counts with %s %s %s %s %s" % (
file1, file2, outfile, outplot, soffset)
locs1, vals1 = read_grfile(file1)
locs2, vals2 = read_grfile(file2)
sortord = numpy.argsort(vals2)
v1_forfit = vals1[sortord]
v2_forfit = vals2[sortord]
# apply a correction to prevent negative rnapol values to correspond to positive expected ipod occupancies
v1_forfit[v2_forfit < 0.0] = 0.0
myspl = growthcurves.fit_bspline(v2_forfit, v1_forfit, numknots=5)
xmin = numpy.min(vals2)
xmax = numpy.max(vals2)
xvals_test_plot = numpy.linspace(xmin, xmax)
print "done with fit"
predvals = growthcurves.eval_bspline(vals2, myspl)
predvals_forplot = growthcurves.eval_bspline(xvals_test_plot, myspl)
print xvals_test_plot
pylab.figure()
pylab.bone()
pylab.hexbin(vals2,
vals1,
gridsize=50,
xscale='linear',
yscale='linear',
bins='log')
#pylab.plot(vals2,vals1,'bo')
pylab.plot(xvals_test_plot, predvals_forplot, 'g-')
pylab.plot(xvals_test_plot, 1 + predvals_forplot, 'b-')
pylab.plot(xvals_test_plot, predvals_forplot - 1, 'r-')
pylab.plot(xvals_test_plot, soffset + predvals_forplot, 'y-')
pylab.xlabel(file2)
pylab.ylabel(file1)
if (outplot is not None):
pylab.savefig(outplot + "_2dhist.png")
#pylab.figure()
#pylab.hist(numpy.log2( v1_forloess / v2_forloess), bins=25)
#if (outplot is not None):
# pylab.savefig(outplot + "_1dhist.png")
print "A"
pylab.figure()
tmpvals = vals1 - (predvals + soffset)
print "B"
pylab.hist(tmpvals, bins=25)
if (outplot is not None):
pylab.savefig(outplot + "_1dhist_subvals.png")
print "C"
print vals1.shape
print predvals.shape
predvals.shape = vals1.shape
newvals = vals1 - (predvals + soffset)
write_grfile(locs1, newvals, outfile)
'y' : 0}
chain = find_MAP(model, chain)
hmc_cov = approx_cov(model, chain)
step_method = hmc_step(model, model.vars, hmc_cov)
ndraw = 3e3
history = NpHistory(model.vars, ndraw)
state, t = sample(ndraw, step_method, chain, history)
print "took :", t
pl.figure()
pl.hexbin(history['x'], history['y'])
# lets plot the samples vs. the actual distribution
logp = model_logp(model)
pts = list(product(np.linspace(-2, 0, 1000), np.linspace(-1,1, 1000)))
values = np.array([logp({'x' : np.array([vx]), 'y' : np.array([vy])}) for vx,vy in pts])
pl.figure()
p = np.array(pts)
xs, ys = p[:,0], p[:,1]
pl.hexbin(xs, ys, exp(values))
pl.show()
sample_filter = (t.redshift < 0.17) & (t.redshift > 0.04)
sample_filter &= (t.m_r < 17.77)
# Completeness in redshift
bounds = [0.04, 0.17, -23.5, -18.5]
redshift = t.redshift[sample_filter]
Mr = t.r[sample_filter]
set_eps_output_1()
pylab.figure()
pylab.axis(bounds)
pylab.xlabel('Redshift')
pylab.ylabel('$M_r$')
pylab.hexbin(redshift, Mr, extent=bounds, gridsize=50, cmap=cm.Oranges)
pylab.colorbar()
if debug:
pylab.show()
else:
pylab.savefig('../doc/figuras/completeness-volume.' + outformat, format=outformat)
# Histograms
cols = ['FUV', 'NUV', 'nii_ha', 'halpha_ew',
'at_flux', 'at_mass', 'am_flux', 'am_mass', 'mcor_gal', 'AV']
filter = {'FUV': t.FUV <> -999,
import pylab as plt
import numpy as np
x = np.linspace(0, 10, 1000)
y = np.linspace(0, 10, 1000)
plt.clf()
plt.hexbin(x,y, gridsize=20, cmap=plt.cm.jet)
plt.colorbar()
raw_input('press ENTER to exit')
def main():
grid_size = 400
min_count = 1
parser = argparse.ArgumentParser()
# d is for directory
parser.add_argument('-d',action='store',dest = 'd',type = str, required = True)
# number of PC sections we want to use (going back from the last)
parser.add_argument('-n',action='store',dest = 'n',type = int, required = False)
inargs = parser.parse_args()
d = inargs.d
num_pc = inargs.n
print('type num_pc: ' + str(type(num_pc)))
os.chdir(d)
all_dir = os.listdir(".")
list_dir = []
num_runs = 0
for i,j in enumerate(all_dir):
if "poindat" in j:
list_dir.append(j)
num_runs+=1
print('num_runs is: '+str(num_runs))
# get the system info
info_f = open('info.txt','r')
l = info_f.readlines()
for i,j in enumerate(l):
if 'cell' in j:
num_cell = float(j.split()[-1])
print('num_cell: ' + str(num_cell))
if 'qq' in j:
qq = float(j.split()[-1])
print('qq: ' + str(qq))
if 'dt' in j:
dt = float(j.split()[-1])
print('dt: ' + str(dt))
if 'beta' in j:
beta = float(j.split()[-1])
print('beta: ' + str(beta))
if 'A:' in j:
A = float(j.split()[-1])
print('A: ' + str(A))
if 'cycles' in j:
cycles = int(j.split()[-1])
print('cycles: ' + str(cycles))
if 'particle number' in j:
N = int(j.split()[-1])
print('N: ' + str(N))
build = pl.zeros(N)+1.0
# this array keeps track of the variable values for each particle
var_arr = pl.array([])
all_data = pl.array([])
x = pl.array([])
for i,j in enumerate(list_dir):
print('cur_file should be: ' + str(j))
cur_file = open(j,"r")
# get the varible for the plot
var = float(cur_file.readline().split()[-1])
data = np.genfromtxt(cur_file)
#print(pl.shape(data))
# if we want more poincare section then count back from the last checking to see if we are
# at the t%2pl.pi=0 point and add data accordingly
if num_pc != None:
# indexing variable
counting = 0
# keep track of how many poincare sections we have saved
passed_pc = 0
while passed_pc <= num_pc:
if (counting*dt)%(2.0*pl.pi)<=dt:
print('passed_pc: '+str(passed_pc))
x = pl.append(x,data[-counting,N:(2*N)]%(num_cell*2.0*pl.pi))
#all_data = pl.append(all_data,data)
var_arr = pl.append(var_arr,build*var)
passed_pc += 1
counting +=1
else:
x = pl.append(x,data[-1,N:(2*N)]%(num_cell*2.0*pl.pi))
#all_data = pl.append(all_data,data)
var_arr = pl.append(var_arr,build*var)
cur_file.close()
# forms all_data into the following: [var run, time, degrees of freedom]
#all_data = all_data.reshape(num_runs,-1,2*N)
# modulus by the length of the system
#all_data[:,:,N:2*N] = all_data[:,:,N:2*N]%(num_cell*2.0*pl.pi)
pl.hexbin(var_arr,x,gridsize=grid_size,bins="log",mincnt=min_count,edgecolor="none")
#pl.hexbin(var_arr,x,gridsize=grid_size,bins="log",mincnt=2,edgecolor="none")
#cmap=pl.cm.YlOrRd_r
#pl.hexbin(A_arr,x,gridsize=400,cmap=pl.cm.Greys,bins="log",mincnt=2,edgecolor="none")
#pl.hexbin(A_arr,x,bins="log")
pl.colorbar()
pl.xlabel("$A$",fontsize=30)
pl.ylabel("$x$",fontsize=30)
pl.tight_layout()
pl.savefig("paper_bif_hist.png",dpi=300)
os.system("open paper_bif_hist.png")
stardat['y'],
z=stardat['color'],
z_method=n.std,
radecgrid=True)
pyl.title('$\sigma(g-i)$')
if savefig:
pyl.savefig('Starcolorstd.png', format='png')
pyl.figure()
ack1, ack2, ack3 = pyl.hist(stardat['color'], bins=100)
pyl.xlabel('$(g-i)$')
pyl.ylabel('Number of stars')
pyl.title('Color Distribution of Calibration Stars')
pyl.savefig('Starcolorhist.png', format='png')
visits = cp.read_visitfile()
#histogram up the visits, set visitlim to [0,90th%ile]
#do a pre-bin to crop range and 90th %tile
x, y = cpu.hammer_project_toxy(visits['ra']*deg2rad,\
visits['dec']*deg2rad)
gridsize = cpu.calc_gridsize(visits, binsize='fov', rad_fov=1.8)
ack = pyl.hexbin(x, y, gridsize=gridsize, mincnt=0.5)
counts = ack.get_array()
hist, bins = n.histogram(counts, n.arange(max(counts) + 1))
nc = n.cumsum(hist, dtype=float) / n.sum(hist)
visit_max = bins[n.max(n.where(nc < 0.99))]
pyl.figure()
cp.plot_visits(visits, visitlim=[1, visit_max])
if savefig:
pyl.savefig('Visitdensity.png', format='png')
def hex_scatter(x, y, min_cnt=2, std=False, levels=2, level_at_edge=False, smoothing=3,
hkwargs={}, skwargs={}, ckwargs={}):
"""Plot a hexbin object with outliers plotted as points.
Parameters
----------
x : array_like
the x coordinates for the input data
y : array_like
the y coordinates for the input data
Keywords
--------
min_cnt: integer (optional)
Minimum number of hex bin counts to draw contour. Bins
containing less than or equal to this will be scatter points.
Default: min_cnt = 2
std : bool (optional)
Draw contours of the fraction of points contained within each
level. The levels are set using the levels keword (see below).
Default: std = False
levels: integer or array (optional)
Only available if std = True, and indicates either the number of
contour levels to draw (integer), or an array of levels to draw.
If integer, levels are drawn at [1, 2, 3, 4, 5][:levels] simga,
i.e. levels=2 will draw contours at 1 and 2 sigma. If array,
the values indicate the fraction of points contained within each
contour. Default: levels = 2
level_at_edge: bool (optional)
Only available if std = True, and indicates if the the scatter
points should start at the last contour level given.
Dfault: cont_at_edge = False
smoothing: integer (optional)
Only available if std = True, and is used to subdivide (i.e.
smooth) the contouring grid. Default: smoothing = 3, values from
1 to 4 are recommended.
See subdiv on: http://matplotlib.org/1.3.0/api/tri_api.html?highlight=uniformtrirefiner#matplotlib.tri.UniformTriRefiner
Passed Keywords
---------------
hkwargs:
a dictionary of keywords passed to hexbin
skwargs:
a dictionary of keywords passed to plot (for scatter points)
ckwargs:
a dictionary of keywords passed to tricontour (if std is set)
Returns
-------
f: pylab.hexbin object
Object returned by hexbin
P: pylab.plot object
Object returned by plot
T: pylab.tricontour object
if std = True, object returned by tricontour
if std = False, None is returned
"""
# check if you are adding curves on top of the hex bin, if so reverse the
# default colorbar. (lower contours have higher levels)
if std:
colorbar = hkwargs.pop('cmap', mpl.cm.winter_r)
else:
colorbar = hkwargs.pop('cmap', mpl.cm.winter)
# set the default scales on x and y axis to linear
hkwargs.setdefault('xscale', 'linear')
hkwargs.setdefault('yscale', 'linear')
# hexbin the data to a mincnt of 0 (needed for sig_cont to look good),
# visible = False so it does not draw yet
f = pl.hexbin(x, y, mincnt=0, visible=False, cmap=colorbar, **hkwargs)
# no draw commands are sent until visible = True
c = f.get_array() # get the number of points in each bin
idx = (c <= min_cnt) # mask for all bins with less than the min_cnt
if std: # check if you are adding curves on top of the hex bin
# if you just indicate the number of contours you want
if isinstance(levels,int):
# dfault to [1, 2, 3, 4, 5][:levels] sigma contorus
ckwargs['levels'] = [0.682689492, 0.954499736, 0.997300204,
0.99993666, 0.999999426697][0:levels]
else:
ckwargs['levels'] = levels
# convert the hexbin counts to fraction of the total number of points
c1 = convert_to_stdev(c)
cmin_std = max(ckwargs['levels'])
# mask for all bins with less than the min_cnt
idx = (c<=min_cnt)
# mask for all bins less than cmin_std
if level_at_edge:
idx = idx | (c1 > cmin_std)
f.set_array(c1[~idx]) # keep values above cutoff
vmin=hkwargs.get('vmin',0.0)
vmax=hkwargs.get('vmax',1.0)
f.set_clim(vmin=vmin, vmax=vmax) # set the limits on the colorbar
else: # if just plotting the hexbins
f.set_array(c[~idx]) # keep values above cutoff
H = f.get_paths() # get the hexagon Path objects
xout = pl.array([]) # array to hold x scatter points
yout = pl.array([]) # array to hold y scatter points
# either this is a very boring hexbin or you have the newer version of
# matplotlib and binning on a linear scale (log scale still uses the old
# hexbin convention)
if (len(H) == 1) \
and (hkwargs['xscale'] == 'linear') \
and (hkwargs['yscale'] == 'linear'):
ext = H[0].get_extents() # the extents for this one hex
h = f.get_offsets() # the centers for each hex
if std: # check if you are adding curves on top of the hexbin
# plot contours
T = triangle_contour(h.T[0], h.T[1], c1, smoothing, ckwargs)
out = h[idx] # only the hexagons with less than min_cnt
# keep only the hexagons with values larger than min_cnt
f.set_offsets(h[~idx])
idx = idx & (c > 0)
# loop through all the hexagons with less than the min_cnt
for o in out:
# move data to current bin's center
xnow = x - o[0]
ynow = y - o[1]
# get the index of the points in extent of the hexagon
jdx = (xnow >= ext.xmin) & (xnow <= ext.xmax) & \
(ynow >= ext.ymin) & (ynow <= ext.ymax)
# loop through each point in extent of the hexagon
for p in zip(xnow[jdx], ynow[jdx]):
# if the point is actual in the hexagon add it to the list of
# outliers
if H[0].contains_point(p):
xout = np.append(xout, p[0] + o[0])
yout = np.append(yout, p[1] + o[1])
# you have the old version of matplotlib or non linear scaling on an axis
else:
if std: # check if you are adding curves on top of the hex bin
# get the centers of each hexagon
xs = np.array([(0.5 * (i.get_extents().xmax - i.get_extents().xmin) \
+ i.get_extents().xmin) for i in H])
ys = np.array([(0.5 * (i.get_extents().ymax - i.get_extents().ymin) \
+ i.get_extents().ymin) for i in H])
# plot contours
T = triangle_contour(xs, ys, c1, smoothing, ckwargs)
out = pl.array(H)[idx] # hexagons to be cut
for o in out: # loop over Paths
H.remove(o) # remove hexagons below cutoff
ext = o.get_extents() # get the x,y extent of the hexagon
# mask for data points only in the hexagon extent
jdx = (x >= ext.xmin) & (x <= ext.xmax) & \
(y >= ext.ymin) & (y <= ext.ymax)
for p in zip(x[jdx], y[jdx]): # loop over these points
if o.contains_point(p): # check if point is in the hexagon
xout = np.append(xout, p[0]) # if so append the x value
yout = np.append(yout, p[1]) # if so append the y value
f.set_visible(True) # draw remaining hexagons
# set default type and size for outlier points
skwargs.setdefault('marker', '.')
skwargs.setdefault('ms', 2)
skwargs.setdefault('ls', 'None')
P = pl.plot(xout, yout, **skwargs) # plot the outlier points
if std:
return f, P, T
else:
return f, P, None
def plot_pval_emmax_correlations(
filter=1.0, file_prefix='/storage/r2_results/250K_r2_min015'):
pickled_file = file_prefix + '_corr_info.pickled'
x_pvals = []
y_pvals = []
if os.path.isfile(pickled_file):
print 'Loading pickled data..'
f = open(pickled_file, 'rb')
d = cPickle.load(f)
f.close()
print 'Done'
for t in d:
x_pvals.append(d[t]['x'][3])
y_pvals.append(d[t]['y'][3])
else:
res_dict = _load_r2_results_()
#find pairs...
d = {}
num_res = len(res_dict['x_chr'])
print 'Plowing through %i results..' % num_res
for i in xrange(num_res):
if (i + 1) % (num_res / 100) == 0:
sys.stdout.write('.')
sys.stdout.flush()
if sp.rand() > filter: continue
x_chr = res_dict['x_chr'][i]
y_chr = res_dict['y_chr'][i]
x_pos = res_dict['x_pos'][i]
y_pos = res_dict['y_pos'][i]
#headers = ['x_chr', 'x_pos', 'y_chr', 'y_pos', 'r2', 'pval', 'f_stat', 'emmax_pval', 'beta', 'emmax_r2']
r2 = res_dict['r2'][i]
pval = res_dict['pval'][i]
f_stat = res_dict['f_stat'][i]
emmax_pval = res_dict['emmax_pval'][i]
if emmax_pval > 0.1:
continue
beta = res_dict['beta'][i]
emmax_r2 = res_dict['emmax_r2'][i]
y_t = (y_chr, y_pos)
x_t = (x_chr, x_pos)
if y_t < x_t:
t = (x_chr, x_pos, y_chr, y_pos)
flipped = True
else:
t = (y_chr, y_pos, x_chr, x_pos)
flipped = False
if not t in d: #Slow as hell!!
d[t] = {}
if flipped:
d[t]['x'] = [r2, pval, f_stat, emmax_pval, beta, emmax_r2]
else:
d[t]['y'] = [r2, pval, f_stat, emmax_pval, beta, emmax_r2]
l = d.keys()[:]
for t in l:
if 'x' in d[t] and 'y' in d[t]:
x_emmax_pval = d[t]['x'][3]
y_emmax_pval = d[t]['y'][3]
if x_emmax_pval < 1 and y_emmax_pval < 1:
x_pvals.append(x_emmax_pval)
y_pvals.append(y_emmax_pval)
else:
del d[t]
else:
del d[t]
f = open(pickled_file, 'wb')
cPickle.dump(d, f, 2)
f.close()
sp.corrcoef(x_pvals, y_pvals)[0, 1]
pylab.plot(x_pvals, y_pvals, '.')
pylab.xlabel('p-value')
pylab.ylabel('p-value')
pval_corr = sp.corrcoef(x_pvals, y_pvals)[0, 1]
pylab.title('Pval. corr.: %0.2f' % pval_corr)
pylab.savefig(env['results_dir'] + 'pval_corr_plot.png')
pylab.clf()
pylab.hexbin(x_pvals, y_pvals, gridsize=1000)
pylab.xlabel('p-value')
pylab.ylabel('p-value')
pylab.title('Pval. corr.: %0.2f' % pval_corr)
pylab.colorbar()
pylab.savefig(env['results_dir'] + 'pval_corr_2d_hist.png')
x_log_pvals = map(lambda x: -sp.log10(x), x_pvals)
y_log_pvals = map(lambda x: -sp.log10(x), y_pvals)
pylab.plot(x_log_pvals, y_log_pvals, '.')
pylab.xlabel('p-value')
pylab.ylabel('p-value')
log_pval_corr = sp.corrcoef(x_pvals, y_pvals)[0, 1]
pylab.title('Neg. log. pval. corr.: %0.2f' % log_pval_corr)
pylab.savefig(env['results_dir'] + 'log_pval_corr_plot.png')
pylab.clf()
pylab.hexbin(x_log_pvals, y_log_pvals, gridsize=1000)
pylab.xlabel('p-value')
pylab.ylabel('p-value')
pylab.title('Pval. corr.: %0.2f' % pval_corr)
pylab.colorbar()
pylab.savefig(env['results_dir'] + 'log_pval_corr_2d_hist.png')
def planner(results):
"""
Generate charts and tables for central case scenarios
"""
home = '/home/nealbob'
folder = '/Dropbox/Model/results/chapter7/'
out = '/Dropbox/Thesis/IMG/chapter7/'
img_ext = '.pdf'
table_out = '/Dropbox/Thesis/STATS/chapter7/'
#with open(home + folder + 'central_result.pkl', 'rb') as f:
# results = pickle.load(f)
# f.close()
stats_envoff, timeseries_envoff, stats, timeseries = results
###### Summary results #####
cols = ['Mean', 'SD', '2.5th', '25th', '75th', '97.5th']
rows = ['Consumptive', 'Optimal']
series = ['SW', 'Profit', 'B', 'S', 'W', 'E', 'P']
scale = {'SW' : 1000000, 'Profit' : 1000000, 'S' : 1000, 'W' : 1000, 'E' : 1000, 'B' : 1000000, 'P' : 1}
m = 2
for x in series:
data0 = []
record = {}
for col in cols:
record[col] = stats_envoff[x]['Annual'][col][m] / scale[x]
data0.append(record)
record = {}
for col in cols:
record[col] = stats[x]['Annual'][col][m] / scale[x]
data0.append(record)
data = pandas.DataFrame(data0)
data.index = rows
with open(home + table_out + ' ' + x + '.txt', 'w') as f:
f.write(data.to_latex(float_format='{:,.2f}'.format, columns=cols))
f.close()
###### Environmental flows #####
cols = ['Mean', 'SD', '2.5th', '25th', '75th', '97.5th']
rows = ['Summer', 'Winter', 'Annual']
series = ['Q_env', 'Q']
scale = {'Q_env' : 1000, 'Q' : 1000}
m = 2
for x in series:
data0 = []
for row in rows:
record = {}
for col in cols:
record[col] = stats[x][row][col][m] / scale[x]
data0.append(record)
data = pandas.DataFrame(data0)
data.index = rows
with open(home + table_out + ' ' + x + '.txt', 'w') as f:
f.write(data.to_latex(float_format='{:,.2f}'.format, columns=cols))
f.close()
###### River flows #########
cols = ['Mean', 'SD', '2.5th', '25th', '75th', '97.5th']
rows = ['Summer', 'Winter', 'Annual']
series = ['F1', 'F1_tilde', 'F3', 'F3_tilde']
scale = 1000
m = 2
for x in series:
data0 = []
for row in rows:
record = {}
for col in cols:
record[col] = stats[x][row][col][m] / scale
data0.append(record)
data = pandas.DataFrame(data0)
data.index = rows
with open(home + table_out + ' ' + x + '.txt', 'w') as f:
f.write(data.to_latex(float_format='{:,.2f}'.format, columns=cols))
f.close()
for x in series:
data0 = []
for row in rows:
record = {}
for col in cols:
record[col] = stats_envoff[x][row][col][m] / scale
data0.append(record)
data = pandas.DataFrame(data0)
data.index = rows
with open(home + table_out + ' ' + x + '_envoff.txt', 'w') as f:
f.write(data.to_latex(float_format='{:,.2f}'.format, columns=cols))
f.close()
# Flow duration curves
data = {'Natural' : timeseries['F1_tilde'][:, 0]/1000,
'Consumptive' : timeseries_envoff['F1'][:, 0]/1000,
'Optimal' : timeseries['F1'][:, 0]/1000 }
duration_curve(data, OUTFILE=home + out + 'up_sum' + img_ext)
data = {'Natural' : timeseries['F1_tilde'][:, 1]/1000,
'Consumptive' : timeseries_envoff['F1'][:, 1]/1000,
'Optimal' : timeseries['F1'][:, 1]/1000 }
duration_curve(data, OUTFILE=home + out + 'up_win' + img_ext)
data = {'Natural' : timeseries['F3_tilde'][:, 0]/1000,
'Consumptive' : timeseries_envoff['F3'][:, 0]/1000,
'Optimal' : timeseries['F3'][:, 0]/1000 }
duration_curve(data, OUTFILE=home + out + 'down_sum' + img_ext)
data = {'Natural' : timeseries['F3_tilde'][:, 1] /1000,
'Consumptive' : timeseries_envoff['F3'][:, 1] /1000,
'Optimal' : timeseries['F3'][:, 1]/1000 }
duration_curve(data, OUTFILE=home + out + 'down_win' + img_ext)
from econlearn import TilecodeRegressor as TR
tr = TR(1, [11], 20)
I = timeseries['I'][1:100000,1] / 1000
idx = I < 1400
I = I[idx]
Q = timeseries['W'][1:100000,1] / 1000
Q = Q[idx]
S = timeseries['S'][1:100000,1] / 1000
S = S[idx]
S2 = timeseries['S'][1:100000,0] / 1000
S2 = S2[idx]
Popt = timeseries['P'][:,0]
Pcons = timeseries_envoff['P'][:,0]
tr.fit(I[1:30000], Q[1:30000])
tr.tile.plot(['x'], showdata=True)
pylab.xlabel('Inflow, $I_t$ (GL)')
pylab.ylabel('Release, $W_t$ (GL)')
pylab.xlim(0, 1400)
#setFigLinesBW_list(fig)
#pylab.legend(bbox_to_anchor=(0., 1.02, 1., .102), loc=3, ncol=4, mode="expand", borderaxespad=0.)
pylab.savefig(home + out + 'env_dem_planner.pdf', bbox_inches='tight')
pylab.show()
tr.fit(S[1:30000], Q[1:30000])
tr.tile.plot(['x'], showdata=True)
pylab.xlabel('Storage, $S_t$ (GL)')
pylab.ylabel('Release, $W_t$ (GL)')
#setFigLinesBW_list(fig)
#pylab.legend(bbox_to_anchor=(0., 1.02, 1., .102), loc=3, ncol=4, mode="expand", borderaxespad=0.)
pylab.savefig(home + out + 'env_dem_plannerS.pdf', bbox_inches='tight')
pylab.show()
pylab.hexbin(S2, I, C=Q)
pylab.xlabel('Summer storage, $S_t$ (GL)')
pylab.ylabel('Winter inflow, $I_t$ (GL)')
cb = pylab.colorbar()
cb.set_label('Winter release, $W_t$ (GL)')
#pylab.ylim(0, 1000)
pylab.savefig(home + out + 'env_dem_plannerZ.pdf', bbox_inches='tight')
pylab.show()
xopt = pylab.hist(Popt, bins=120)
xcons = pylab.hist(Pcons, bins=120)
pylab.show()
fig = pylab.figure()
pylab.plot(xopt[1][1::], xopt[0]/500000, label = 'Optimal')
pylab.plot(xcons[1][1::], xcons[0]/500000, label = 'Consumptive')
setFigLinesBW_list(fig)
pylab.legend(bbox_to_anchor=(0., 1.02, 1., .102), loc=3, ncol=2, mode="expand", borderaxespad=0.)
pylab.xlabel('Summer shadow price, $ per ML')
pylab.ylabel('Frequency')
pylab.xlim(0, 500)
pylab.savefig(home + out + 'plan_price.pdf', bbox_inches='tight')
pylab.show()
with Model() as model:
x = Normal('x', 0, 1)
y = Normal('y', 0, 1)
N = 200
d = Normal('d', x + y ** 2, 1., observed=np.zeros(N))
start = model.test_point
h = np.ones(2) * np.diag(find_hessian(start))[0]
step = HamiltonianMC(model.vars, h, path_length=4.)
trace = sample(3e3, step, start)
pl.figure()
pl.hexbin(trace['x'], trace['y'])
# lets plot the samples vs. the actual distribution
from theano import function
xn = 1500
yn = 1000
xs = np.linspace(-3, .25, xn)[np.newaxis, :]
ys = np.linspace(-1.5, 1.5, yn)[:, np.newaxis]
like = (xs + ys ** 2) ** 2 * N
post = np.exp(-.5 * (xs ** 2 + ys ** 2 + like))
post = post
pl.figure()
def test_correlation(sample_num=4000, mac_filter=15, debug_filter=0.05):
dtype = 'single' #To increase matrix multiplication speed... using 32 bits.
sd = dp.parse_numerical_snp_data(env['data_dir'] + '250K_t72.csv.binary',
filter=debug_filter)
sd.filter_mac_snps(mac_filter)
snps = sd.getSnps()[:]
kinship_matrix_file = env['data_dir'] + 'snp_corr_kinship_cm72.pickled'
if not os.path.isfile(kinship_matrix_file):
K = sd.get_snp_cov_matrix()
lm.save_kinship_to_file(kinship_matrix_file, K, sd.accessions)
else:
K = lm.load_kinship_from_file(kinship_matrix_file, dtype='single')
h_inverse_matrix_file = env[
'data_dir'] + 'snp_corr_kinship_h_inv_cm72.pickled'
if not os.path.isfile(h_inverse_matrix_file):
H_sqrt = lm.cholesky(K)
H_sqrt_inv = (H_sqrt).I
with file(h_inverse_matrix_file, 'wb') as f:
cPickle.dump(H_sqrt_inv.tolist(), f)
else:
with file(h_inverse_matrix_file) as f:
H_sqrt_inv = sp.mat(cPickle.load(f))
t_snps_file = env['data_dir'] + 'snps_trans_kinship_cm72.pickled'
if debug_filter < 1 or not os.path.isfile(t_snps_file):
if os.path.isfile(kinship_matrix_file):
n_snps_mat = sd.get_normalized_snps().T
t_snps_mat = n_snps_mat * H_sqrt_inv
t_snps = t_snps_mat.tolist()
with open(t_snps_file, 'wb') as f:
cPickle.dump(t_snps, f)
else:
with open(t_snps_file) as f:
t_snps = cPickle.load(f)
cp_list = sd.getChrPosList()
l = range(len(cp_list))
y_sample = random.sample(l, sample_num)
for i in reversed(sorted(y_sample)):
del l[i]
x_sample = random.sample(l, sample_num)
res_d = {}
for i, xi in enumerate(x_sample):
(x_c, x_p) = cp_list[xi]
x_snp = snps[xi]
x_snp_t = t_snps[xi]
print 'SNP %d: chromosome=%d, position=%d' % (i, x_c, x_p)
for j, yi in enumerate(y_sample):
#pdb.set_trace()
(y_c, y_p) = cp_list[yi]
y_snp = snps[yi]
y_snp_t = t_snps[yi]
(r, pearson_pval) = st.pearsonr(x_snp, y_snp)
r2 = r * r
if r2 > 0.05:
(r_t, pearson_pval_t) = st.pearsonr(x_snp_t, y_snp_t)
r2_t = r_t * r_t
if pearson_pval == 0:
pearson_pval = min_float
if pearson_pval_t == 0:
pearson_pval_t = min_float
res_d[(yi, xi)] = [r2_t, r2, pearson_pval, pearson_pval_t]
xs = []
for yi, xi in res_d:
xs.append(res_d[(yi, xi)][0])
pylab.hist(xs, bins=20, log=True)
pylab.xlabel('Transformed $r^2$')
pylab.savefig(env['results_dir'] + 'r2_t_hist.png')
pylab.clf()
# #xs = -sp.log10(xs)
# #ys = -sp.log10(ys)
# pylab.plot(xs, ys, '.', markersize=2)
# pylab.savefig(env['results_dir'] + 'norm_emma_log_pvals.png')
# pylab.clf()
xs = []
for yi, xi in res_d:
xs.append(res_d[(yi, xi)][1])
pylab.hist(xs, bins=20, log=True)
pylab.xlabel('Standard $r^2$')
pylab.savefig(env['results_dir'] + 'r2_hist.png')
pylab.clf()
xs = []
ys = []
for yi, xi in res_d:
ys.append(res_d[(yi, xi)][0])
xs.append(res_d[(yi, xi)][1])
x_max = max(xs)
y_max = max(ys)
lim = min(x_max, y_max)
pylab.plot(xs, ys, '.', markersize=2)
pylab.plot([0, lim], [0, lim], color='g', alpha=0.5)
pylab.xlabel('Standard $r^2$')
pylab.ylabel('Transformed $r^2$')
pylab.savefig(env['results_dir'] + 'r2_r2_t.png')
pylab.clf()
pylab.hexbin(xs, ys)
pylab.xlabel('Standard $r^2$')
pylab.ylabel('Transformed $r^2$')
cb = pylab.colorbar()
cb.set_label('Count')
pylab.savefig(env['results_dir'] + 'r2_r2_t_3Dhist.png')
pylab.clf()
xs = []
ys = []
for yi, xi in res_d:
ys.append(res_d[(yi, xi)][3])
xs.append(res_d[(yi, xi)][2])
xs = -sp.log10(xs)
ys = -sp.log10(ys)
x_max = max(xs)
y_max = max(ys)
lim = min(x_max, y_max)
pylab.plot(xs, ys, '.', markersize=2)
pylab.plot([0, lim], [0, lim], color='g', alpha=0.5)
pylab.xlabel('Standard $r^2$ -log(p-value)')
pylab.ylabel('Transformed $r^2$ -log(p-value)')
pylab.savefig(env['results_dir'] + 'pval_pval_t.png')
pylab.clf()
pylab.hexbin(xs, ys)
pylab.xlabel('Standard $r^2$ -log(p-value)')
pylab.ylabel('Transformed $r^2$ -log(p-value)')
cb = pylab.colorbar()
cb.set_label('Count')
pylab.savefig(env['results_dir'] + 'pval_pval_t_3Dhist.png')
def starPlots(truestarfile="stardata.dat",
bestfitstarfile="test_bestfit_Star.dat"):
#assume both the files are sorted by star id.
stardat = cp.read_true_stars(
truestarfile
) #keys are ('id', 'dbid', 'ra', 'dec', 'magtrue', 'color')
starcal = ui.readDatafile(bestfitstarfile,
keys=('id', 'magcal', 'rms', 'iqr_sigma',
'nobs'))
HP = True
if np.size(starcal['id']) == 0:
starcal = ui.readDatafile(bestfitstarfile, keys=('id', 'magcal'))
HP = False
fitted_stars = np.in1d(stardat['id'], starcal['id'])
for key in list(stardat.keys()):
stardat[key] = stardat[key][fitted_stars]
for key in list(starcal.keys()):
stardat[key] = starcal[key]
stardat['magdiff'] = stardat['magcal'] - stardat['magtrue']
stardat = cp.adjust_stars_calvstrue(stardat)
starResults = {}
HPResults = {}
starResults['N stars Fit'] = np.size(stardat['magdiff'])
outlier_clip = np.where(np.abs(stardat['magdiff']) < .05)
rms = stardat['magdiff'][outlier_clip].std()
print('true-bestfit RMS = %f, true-bestfit (clipped) = %f' %
(stardat['magdiff'].std(), rms))
lim = rms * 2.5
if lim < 0.001:
lim = round(lim * 10000) / 10000.0
elif lim < 0.01:
lim = round(lim * 1000) / 1000.0
print("Using magnitude limits for plots of %f, %f" % (-lim, lim))
maglim = [-lim, lim]
starResults['Residuals RMS (mmag)'] = stardat['magdiff'].std() * 1e3
starResults['IQR_sigma (mmag)'] = so.robustSigma(stardat['magdiff']) * 1e3
#star residual map
stardat['x'], stardat['y'] = cpu.hammer_project_toxy(
stardat['ra'] * deg2rad, stardat['dec'] * deg2rad)
gridsize = cpu.calc_gridsize(stardat, binsize='patch')
etitle = " RMS = %4.1f mmag" % (rms * 1000)
cpu.plot_density(stardat['x'],
stardat['y'],
stardat['magdiff'] * 1000,
z_method=np.mean,
zlimits=np.array(maglim) * 1000,
gridsize=gridsize,
radecgrid=True,
newfig=True,
cb_label='mmag')
figtitle = "Stars dMag(true-bestfit)"
pyl.title(figtitle, fontsize='medium')
pyl.savefig('Sdmag.png', type='png')
maglim2 = [0, maglim[1] / 2.]
cp.plot_stars_dmagcaltrue(stardat,
maglim=maglim2,
etitle=" RMS " + etitle,
z_method=np.std)
pyl.savefig('Sdmag_rms.png', format='png')
histrange = [maglim[0] * 2.5, maglim[1] * 2.5]
nbins = 100
cp.histogram_stars_dmagcaltrue(stardat,
nbins=nbins,
range=histrange,
etitle=etitle)
pyl.savefig('Sdamg_hist.png', format='png')
#color distribution of stars
pyl.figure()
pyl.hist(stardat['color'], bins=100)
pyl.xlabel(r'$g-i$')
pyl.ylabel('# of stars')
pyl.title('Color Distribution')
pyl.savefig('Scolordist.png', type='png')
#accuracy v color
pyl.figure()
#pyl.scatter(stardat['color'], stardat['magdiff']*1000., c=stardat['magtrue'], alpha=0.1,edgecolors=None)
pyl.hexbin(stardat['color'], stardat['magdiff'] * 1000., bins='log')
cb = pyl.colorbar()
cb.ax.set_ylabel(r'$\log{\rm{N}}$')
pyl.xlabel(r'$g-i$')
pyl.ylabel('Bestfit-True (mmag)')
pyl.savefig('Saccuracyvcolor.png', type='png')
#accuracy v mag
pyl.figure()
#pyl.scatter(stardat['magtrue'], stardat['magdiff']*1000., c=stardat['color'], alpha=0.1,edgecolors=None)
pyl.hexbin(stardat['magtrue'], stardat['magdiff'] * 1000., bins='log')
cb = pyl.colorbar()
cb.ax.set_ylabel(r'$\log{\rm{N}}$')
pyl.xlabel('True Mag')
pyl.ylabel('Bestfit-True (mmag)')
pyl.savefig('Saccuracyvmag.png', type='png')
if HP:
zlim = [
np.min(stardat['nobs']),
(np.median(stardat['nobs']) - np.min(stardat['nobs'])) +
np.median(stardat['nobs'])
]
cpu.plot_density(stardat['x'],
stardat['y'],
stardat['nobs'],
z_method=np.mean,
zlimits=zlim,
gridsize=gridsize,
radecgrid=True,
newfig=True,
cb_label='N observations')
pyl.title('Visit Density')
pyl.savefig('Snobs.png', format='png')
pyl.figure()
#I should just figure out from the data what nsides is!
hpid = hp.ang2pix(16, (stardat['dec'] + 90.) * deg2rad,
stardat['ra'] * deg2rad)
cpu.plot_density(stardat['x'],
stardat['y'],
np.mod(hpid, 250),
z_method=np.median,
gridsize=gridsize,
radecgrid=True,
newfig=True,
cb_label='HEALpix ID mod 250')
pyl.title("HEALpix Map")
pyl.savefig('HPid.png')
HPResults['number of HEALpix'] = np.size(np.unique(hpid))
pyl.figure()
num, b, p = pyl.hist(stardat['nobs'],
bins=100,
range=zlim,
histtype='bar')
pyl.xlabel("Number of Observations per Star")
pyl.title('Median = %d' % np.median(stardat['nobs']))
pyl.savefig('Snobs_hist.png', type='png')
starResults['median repeat obs'] = np.median(stardat['nobs'])
pyl.figure()
x = np.sort(stardat['iqr_sigma']) * 1e3
y = np.arange(np.size(x), dtype='float')
y = y / np.max(y)
per50 = np.max(np.where(y <= .5))
per90 = np.max(np.where(y <= .9))
pyl.plot(x,
y,
'k--',
label='IQR, 50th,90th %2.1f, %2.1f' % (x[per50], x[per90]))
pyl.plot([0, x[per50]], [y[per50], y[per50]], 'b--')
pyl.plot([x[per50], x[per50]], [0, y[per50]], 'b--')
pyl.plot([0, x[per90]], [y[per90], y[per90]], 'b--')
pyl.plot([x[per90], x[per90]], [0, y[per90]], 'b--')
#pyl.title('Cumulative Distribution of Stellar Repeatability')
#pyl.xlabel('Repeatability IQR RMS (mmag)')
#pyl.ylabel('Cumulative Fraction')
#pyl.savefig('Srepeat_IQR_cumulative.png',type='png')
pyl.legend()
pyl.savefig('Srepeat_cumulative.png', type='png')
#repeatability from IQR
rs = so.robustSigma(stardat['iqr_sigma'])
med = np.median(stardat['iqr_sigma'])
maglim = np.around(np.array([med - 3 * rs, med + 3. * rs]) * 1000,
decimals=2) / 1000
cpu.plot_density(stardat['x'],
stardat['y'],
stardat['iqr_sigma'] * 1000,
z_method=np.mean,
zlimits=maglim * 1000,
gridsize=gridsize,
radecgrid=True,
newfig=True,
cb_label='mmag')
pyl.title('Repeatability (IQR)')
pyl.savefig('Srepeat_IQR.png', type='png')
pyl.figure()
num, b, p = pyl.hist(stardat['iqr_sigma'] * 1000,
bins=100,
range=maglim * 1000,
histtype='bar')
pyl.xlabel('RMS per star (mmag)')
pyl.title('Repeatability IQR, Median = %4.2f mmag' %
np.median(stardat['iqr_sigma'] * 1000))
pyl.savefig('Srepeat_IQR_hist.png', type='png')
starResults['Median Repeatability (IQR) (mmag)'] = np.median(med) * 1e3
pyl.figure()
good = np.where(stardat['magtrue'] < 18)
num, b, p = pyl.hist(stardat['iqr_sigma'][good] * 1000,
bins=100,
range=maglim * 1000,
histtype='bar')
pyl.xlabel('RMS per star, m < 18 (mmag)')
pyl.title('Repeatability IQR, Median = %4.2f mmag' %
np.median(stardat['iqr_sigma'][good] * 1000))
pyl.savefig('Srepeat_IQR_bright_hist.png', type='png')
starResults['Median Repeatability Bright (IQR) (mmag)'] = np.median(
stardat['iqr_sigma'][good]) * 1e3
#repeatability v color
pyl.figure()
#pyl.plot(stardat['color'], stardat['rms']*1000., 'ko', alpha=.1)
pyl.hexbin(stardat['color'], stardat['iqr_sigma'] * 1000., bins='log')
cb = pyl.colorbar()
cb.set_label(r'$\log{\rm{N}}$')
pyl.ylim([0, 40])
pyl.xlabel(r'$g-i$')
pyl.ylabel(r'Repeatability $\sigma_{\rm{IQR}}$ (mmag)')
pyl.savefig('Srepeatvcolor.png', type='png')
#repeatability v mag
pyl.figure()
#pyl.plot(stardat['magtrue'], stardat['rms']*1000., 'ko', alpha=.1)
pyl.hexbin(stardat['magtrue'],
stardat['iqr_sigma'] * 1000.,
bins='log')
cb = pyl.colorbar()
cb.set_label(r'$\log{\rm{N}}$')
pyl.ylim([0, 40])
pyl.xlabel('True Mag')
pyl.ylabel(r'Repeatability $\sigma_{\rm{IQR}}$ (mmag)')
pyl.savefig('Srepeatvmag.png', type='png')
#repeatability
rs = so.robustSigma(stardat['rms'])
med = np.median(stardat['rms'])
maglim = np.around(np.array([med - 3 * rs, med + 3. * rs]) * 1000,
decimals=2) / 1000
cpu.plot_density(stardat['x'],
stardat['y'],
stardat['rms'] * 1000,
z_method=np.mean,
zlimits=maglim * 1000,
gridsize=gridsize,
radecgrid=True,
newfig=True,
cb_label='mmag')
pyl.title('Repeatability')
pyl.savefig('Srepeat.png', type='png')
pyl.figure()
num, b, p = pyl.hist(stardat['rms'] * 1000,
bins=100,
range=maglim * 1000,
histtype='bar')
pyl.xlabel('RMS per star (mmag)')
pyl.title('Repeatability, Median = %4.2f mmag' %
np.median(stardat['rms'] * 1000))
pyl.savefig('Srepeat_hist.png', type='png')
starResults['Median Repeatability (mmag)'] = np.median(med) * 1e3
pyl.figure()
x = np.sort(stardat['rms']) * 1e3
y = np.arange(np.size(x), dtype='float')
y = y / np.max(y)
per50 = np.max(np.where(y <= .5))
per90 = np.max(np.where(y <= .9))
pyl.plot(x, y, 'k', label='RMS, %2.1f, %2.1f' % (x[per50], x[per90]))
pyl.plot([0, x[per50]], [y[per50], y[per50]], 'k')
pyl.plot([x[per50], x[per50]], [0, y[per50]], 'k')
pyl.plot([0, x[per90]], [y[per90], y[per90]], 'k')
pyl.plot([x[per90], x[per90]], [0, y[per90]], 'k')
pyl.title('Cumulative Distribution of Stellar Repeatability')
pyl.xlabel('Repeatability RMS (mmag)')
pyl.ylabel('Cumulative Fraction')
pyl.savefig('Srepeat_cumulative.png', type='png')
#need to make a spatial uniformity plot since we can now have varying densities of stars.
nside = 64
stardat['hpid'] = hp.ang2pix(nside, (stardat['dec'] + 90.) * deg2rad,
stardat['ra'] * deg2rad)
uhpid = np.unique(stardat['hpid'])
hp_mean = uhpid * 0.
hp_std = uhpid * 0.
hp_dec, hp_ra = hp.pix2ang(nside, uhpid)
#hp_ra = hp_ra
hp_dec = hp_dec - 90. * deg2rad
hp_x, hp_y = cpu.hammer_project_toxy(hp_ra, hp_dec)
stardat = dictsort(stardat, 'hpid')
left = np.searchsorted(stardat['hpid'], uhpid)
right = np.searchsorted(stardat['hpid'], uhpid, side='right')
for i in np.arange(left.size):
hp_mean[i] = np.mean(stardat['magdiff'][left[i]:right[i]])
hp_std[i] = np.std(stardat['magdiff'][left[i]:right[i]])
cpu.plot_density(hp_x,
hp_y,
hp_mean * 1000,
z_method=np.mean,
zlimits=None,
gridsize=gridsize,
radecgrid=True,
newfig=True,
cb_label='mmag')
HPResults['Spatial Uniformity RMS (mmag)'] = np.std(hp_mean) * 1e3
HPResults['Spatial Uniformity IQR RMS (mmag)'] = so.robustSigma(
hp_mean) * 1e3
pyl.title('mean(True-Bestfit), binned by HEALpix')
pyl.savefig('HPresid.png', type='png')
cpu.plot_density(hp_x,
hp_y,
hp_std * 1000,
z_method=np.mean,
zlimits=None,
gridsize=gridsize,
radecgrid=True,
newfig=True,
cb_label='mmag')
pyl.title('std(True-Bestfit), binned by HEALpix')
pyl.savefig('HPstd.png', type='png')
pyl.figure()
num, b, p = pyl.hist(hp_mean * 1000,
bins=100,
range=None,
histtype='bar')
pyl.xlabel("HEALpix(True-Bestfit) (mmag)")
pyl.title('Spatial Uniformity, RMS=%4.2f' %
HPResults['Spatial Uniformity RMS (mmag)'])
pyl.savefig('HPmean_hist.png', type='png')
pyl.figure()
num, b, p = pyl.hist(hp_std * 1000,
bins=100,
range=None,
histtype='bar')
pyl.xlabel("std(True-Bestfit) per HEALpix (mmag)")
pyl.title('Variation within HEALpix, median=%4.2f mmag' %
np.median(hp_std * 1000))
pyl.savefig('HPstd_hist.png', type='png')
return starResults, HPResults
pylab.show()
else:
pylab.savefig('../doc/figuras/histo_wha_color.' + outformat, format=outformat)
# NUV-r versus WHa
set_eps_output_1()
pylab.figure()
bounds = [0.5, 6.0, 2.5, 6.5]
pylab.axis(bounds)
filter = (WHa > 0.5) & (WHa < 6.0) & (logN2Ha > -0.4)
h, ex, ey = np.histogram2d(WHa[filter], NUV_r[filter], bins=25,
range=[[bounds[0], bounds[1]],[bounds[2], bounds[3]]])
#pylab.scatter(WHa[filter], t.NUV[filter] - t.r[filter], c='orange', marker='o',
# edgecolor='None', s=1)
pylab.hexbin(WHa[filter], NUV_r[filter], extent=bounds,# bins='log',
gridsize=30, cmap=cm.OrRd)
pylab.contour(h.T, extent=bounds, colors='black', linewidths=0.5)
pylab.axvline(3.0, color='k', linestyle='--')
pylab.xlabel('$W_{\\mathrm{H}\\alpha}$ [\\AA]')
pylab.ylabel('$\\mathrm{NUV} - r$')
if debug:
pylab.show()
else:
pylab.savefig('../doc/figuras/wha_nuv.' + outformat, format=outformat)
# WHaN plot
set_eps_output_1()
pylab.figure()
bounds = [-1.0,0.6,-1.0,2.5]
def heatmap(prob_map):
x, y = generate_x_y(prob_map)
figure()
hexbin(x, y, C=prob_map.ravel())
if args.test:
import sys
print("^_^ imports work, minimal test successful")
sys.exit(0)
files_affected_log = []
files_affected = []
with open(args.datafiles[0] if args.datafiles else
"files-affected-by-time-with-issue-only-bugs.dat") as f:
for i, n in enumerate(f):
n = n.split()[-1]
files_affected_log.append((i, math.log(1 + int(n), 10)))
files_affected.append((i, int(n)))
pl.hexbin(*zip(*files_affected_log),
norm=matplotlib.colors.LogNorm(),
gridsize=50)
cb = pl.colorbar()
cb.set_label('number of matching issues')
pl.ylabel(args.ylabel)
pl.xlabel("issues sorted by time, oldest first")
locs, labels = pl.yticks()
explabels = [10**(loc) - 1 for loc in locs]
pl.yticks(locs, [int(i) if i >= 0.01 else "" for i in explabels])
pl.ylim(0, max(j for i, j in files_affected_log))
pl.title(args.title)
running_median = []
running_95 = []
radius = 500
nissues = len(files_affected)
for i, n in files_affected:
def sens(sample=20):
home = '/home/nealbob'
folder = '/Dropbox/Model/results/chapter7/chapter7/'
out = '/Dropbox/Thesis/IMG/chapter7/'
img_ext = '.pdf'
table_out = '/Dropbox/Thesis/STATS/chapter7/'
rows = ['CS', 'SWA', 'OA', 'CS-HL']
results = {run_no: {row : 0 for row in rows} for run_no in range(1,sample)}
samprange = []
for run_no in range(1, sample):
try:
for row in rows:
with open(home + folder + str(run_no) + '_' + row + '_result.pkl', 'rb') as f:
results[run_no][row] = pickle.load(f)
f.close()
m = len(results[run_no]['CS'][0]['S']['Annual']['Mean']) - 1
SW = results[run_no][row][0]['SW']['Annual']['Mean'][m]
if math.isnan(SW) or math.isinf(SW):
raise Exception("Found a nan")
samprange.append(run_no)
except:
print row
print 'Run no: ' + str(run_no) + ' failed.'
n = len(samprange)
print str(n) + ' good runs of ' + str(sample - 1) + ' total'
###### Summary tables #####
series = ['SW', 'Profit', 'B', 'Budget', 'S']
title = {'SW' : 'Social welfare relative to CS', 'Profit' : 'Profit relative to CS', 'B' : 'Environmental benefits relative to CS', 'S' : 'Storage relative to CS', 'Budget' : 'Environmental trade relative to CS'}
scale = {'SW' : 1000000, 'Profit' : 1000000, 'S' : 1000, 'W' : 1000, 'E' : 1000, 'B' : 1000000, 'Z' : 1000, 'Q_low' : 1000, 'Q_high' : 1000, 'Q_env' : 1000, 'A_low' : 1000, 'A_high' : 1000, 'A_env' : 1000, 'S_low' : 1000, 'S_high' : 1000, 'S_env' : 1000, 'U_low' : 1000000, 'U_high' : 1000000, 'Budget' : 1000000}
m = len(results[1]['CS'][0]['S']['Annual']['Mean']) - 1
X = {}
XI = {}
for x in series:
data0 = []
data1 = []
data2 = []
for row in rows:
temp = np.zeros(n)
record = {}
record1 = {}
i = 0
for run_no in samprange:
temp[i] = results[run_no][row][0][x]['Annual']['Mean'][m] / scale[x]
i += 1
record[run_no] = results[run_no][row][0][x]['Annual']['Mean'][m] / scale[x]
record1['Mean'] = np.mean(temp)
record1['Min'] = np.min(temp)
record1['Q1'] = np.percentile(temp, 25)
record1['Q3'] = np.percentile(temp, 75)
record1['Max'] = np.max(temp)
X[row] = temp
data0.append(record)
data1.append(record1)
data = pandas.DataFrame(data0)
data.index = rows
data1 = pandas.DataFrame(data1)
data1.index = rows #['Mean', 'Min', 'Q1', 'Q3', 'Max']
for row in rows:
record2 = {}
temp1 = np.zeros(n)
for i in range(n):
temp1[i] = X[row][i] / X['CS'][i]
XI[row] = temp1
record2['Mean'] = np.mean(temp1)
record2['Min'] = np.min(temp1)
record2['Q1'] = np.percentile(temp1, 25)
record2['Q3'] = np.percentile(temp1, 75)
record2['Max'] = np.max(temp1)
data2.append(record2)
data2 = pandas.DataFrame(data2)
data2.index = rows #['Mean', 'Min', 'Q1', 'Q3', 'Max']
with open(home + table_out + 'sens_full' + x + '.txt', 'w') as f:
f.write(data.to_latex(float_format='{:,.2f}'.format, columns=samprange))
f.close()
with open(home + table_out + 'sens_sum' + x + '.txt', 'w') as f:
f.write(data1.to_latex(float_format='{:,.2f}'.format, columns=['Mean', 'Min', 'Q1', 'Q3', 'Max']))
f.close()
with open(home + table_out + 'sens_table' + x + '.txt', 'w') as f:
f.write(data2.to_latex(float_format='{:,.2f}'.format, columns=['Mean', 'Min', 'Q1', 'Q3', 'Max']))
f.close()
minx = np.percentile([min(XI[i]) for i in XI], 1)
maxx = np.percentile([max(XI[i]) for i in XI],99)
chart_ch7(XI, 0.985 * minx, 1.015 * maxx, title[x], out, str(x) + '_sens')
##################################################################################### Regression
Y = np.zeros([n, 4])
j = 0
for row in rows:
i = 0
for run_no in samprange:
Y[i, j] = results[run_no][row][0]['SW']['Annual']['Mean'][m] / results[run_no]['CS'][0]['SW']['Annual']['Mean'][m]
i += 1
j += 1
paras = []
for run_no in range(1, sample):
with open(home + folder + str(run_no) + '_para.pkl', 'rb') as f:
paras.append(pickle.load(f))
f.close()
pname1 = ['delta0', 'I_K', 'SD_I', 't_cost', 'N_high', 'rho_I', 'alpha', 'rho_eps', 'sig_eta', 'LL']
numpara1 = len(pname1)
pname2 = ['omega_mu', 'omega_sig', 'omegadelta', 'delta_a', 'delta_Ea', 'delta_Eb', 'delta_R', 'b_1', 'b_value', 'e_sig']
numpara2 = len(pname2)
para_labels = pname1 + pname2 + ['lambda', 'lambdaHL', 'lambdae']
numpara = numpara1 + numpara2 + 3
X = np.zeros([n, numpara])
para_names = ['$\delta0$', '$E[I]/K$', '$c_v$', '$\tau$', '$n_{high}$', '$\rho_I$', '$\alpha$', '$\rho_e$', '$\sigma_{\eta}$', '${\aA_{low} \over E[I]/K}$', '$\mu_\omega$', '$\sigma_\omega$', '$\omega_\delta$', '$\delta_a$', '$\delta_{Ea}$', '$\delta_{Eb}$', '$\delta_R$', '$b_1$', '$b_{\$} \over \bar I$', '$\sigma_{e0}$', '$\Lambda_{high} - \hat \Lambda_{high}$', '$\Lambda_{high}^{CS-HL} - \hat \Lambda_{high}^{CS-HL}$', '$\lambda_0 - \hat \lambda_0$' ]
for j in range(numpara1):
for i in range(n):
if pname1[j] == 'LL':
X[i, j] = paras[samprange[i]-1].para_list[pname1[j]] / paras[samprange[i]-1].para_list['I_K']
else:
X[i, j] = paras[samprange[i]-1].para_list[pname1[j]]
for j in range(numpara1, numpara2+numpara1):
for i in range(n):
if pname2[j - numpara1] == 'b_value':
X[i, j] = paras[samprange[i]-1].ch7[pname2[j - numpara1]] / (paras[samprange[i]-1].para_list['I_K']*1000000)
else:
X[i, j] = paras[samprange[i]-1].ch7[pname2[j - numpara1]]
CS_c = -0.153007555
CS_b = 0.00930613
CSHL_c = -0.0891846
CSHL_b = 0.0047009
for i in range(n):
if i > 20:
y = paras[samprange[i]-1].y
else:
y = CS_c + CS_b * paras[samprange[i]-1].para_list['N_high']
X[i, numpara2 + numpara1] = paras[samprange[i]-1].Lambda_high - y
for i in range(n):
if i > 20:
yhl = paras[samprange[i]-1].yhl
else:
yhl = CSHL_c + CSHL_b * paras[samprange[i]-1].para_list['N_high']
X[i, numpara2 + numpara1 + 1] = paras[samprange[i]-1].Lambda_high_HL - yhl
yelist = [0.4443, 0.1585, 0.1989, 0.2708, 0.3926, 0.0697, 0.1290, 0.1661, 0.2687, 0.0868, 0.1239, 0.3598, 0.3543, 0.2883, 0.2367, 0.2139, 0.2485, 0.2641, 0.5730, 0.1745]
lambdae = np.zeros(n)
for i in range(n):
if i >= 20:
ye = paras[samprange[i]-1].E_lambda_hat
else:
ye = yelist[samprange[i]-1]
X[i, numpara2 + numpara1 + 2] = paras[samprange[i]-1].ch7['inflow_share'] - ye
lambdae[i] = paras[samprange[i]-1].ch7['inflow_share']
index = lambdae < 0.5
pylab.hexbin(lambdae[index], X[index,1], C=Y[index, 2], gridsize=15)
pylab.xlabel('Environmental share, $\lambda_0$')
pylab.ylabel('Mean Inflow to Capacity, $E[I_t]/K$')
cb = pylab.colorbar()
cb.set_label('OA welfare relative to CS')
#pylab.ylim(0, 1000)
pylab.savefig(home + out + 'OAversusCS.pdf', bbox_inches='tight')
pylab.show()
pylab.hexbin(X[:, numpara -1], X[:, 1], C=Y[:, 3], gridsize=15)
pylab.xlabel('Environmental share, $\lambda_0 - \hat \lambda_0$')
pylab.ylabel('Mean Inflow to Capacity, $E[I_t]/K$')
cb = pylab.colorbar()
cb.set_label('CS-HL welfare relative to CS')
#pylab.ylim(0, 1000)
pylab.savefig(home + out + 'CSHLversusCS.pdf', bbox_inches='tight')
pylab.show()
tree = Tree(n_estimators=500, n_jobs=4)
tree.fit(X, Y)
rank = tree.feature_importances_ * 100
data0 = []
inn = 0
for p in para_names:
record = {}
record['Importance'] = rank[inn]
data0.append(record)
inn = inn + 1
tab = pandas.DataFrame(data0)
tab.index = para_names
tab = tab.sort(columns=['Importance'], ascending=False)
tab_text = tab.to_latex(float_format='{:,.2f}'.format, escape=False)
print tab_text
with open(home + table_out + 'importance.txt', 'w') as f:
f.write(tab_text)
f.close()
for i in range(numpara):
Xtemp = np.zeros([200, numpara])
for j in range(numpara):
Xtemp[:, j] = np.ones(200) * np.mean(X[:, j])
Xtemp[:, i] = np.linspace(np.min(X[:, i]), np.max(X[:, i]), 200)
Ytemp = tree.predict(Xtemp)
data = [[Xtemp[:, i], Ytemp]]
data0 = []
for k in range(200):
record = {}
record['SWA'] = Ytemp[k, 1]
record['OA'] = Ytemp[k, 2]
record['CS-HL'] = Ytemp[k, 3]
data0.append(record)
data = pandas.DataFrame(data0)
data.index = Xtemp[:, i]
chart_data = {'OUTFILE': home + out + 'SW_' + para_labels[i] + img_ext,
'XLABEL': '',
'YLABEL': '',
'YMIN': 0.85,
'YMAX': 1.03}
print para_labels[i]
build_chart(chart_data, data, chart_type='date', ylim=True, save=True)
##################################################################################### Classifier
srnum = {'CS' : 0, 'SWA' : 1, 'OA' : 2, 'CS-HL' : 3}
Y = np.zeros(n)
for i in range(n):
SW = 0
SWmax = -1
for row in rows:
SW = results[samprange[i]][row][0]['SW']['Annual']['Mean'][m]
if SW > SWmax:
SWmax = SW
Y[i] = srnum[row]
for row in rows:
idx = np.where(Y == srnum[row])
print row + ': ' + str(len(Y[idx]))
treec = Tree_classifier(n_estimators=500, n_jobs=4) #min_samples_split=3, min_samples_leaf=2)
treec.fit(X, Y)
rank = treec.feature_importances_ * 100
data0 = []
inn = 0
for p in para_names:
record = {}
record['Importance'] = rank[inn]
record['CS'] = np.mean(X[np.where(Y == 0), inn])
record['SWA'] = np.mean(X[np.where(Y == 1), inn])
record['OA'] = np.mean(X[np.where(Y == 2), inn])
record['CS-HL'] = np.mean(X[np.where(Y == 3), inn])
data0.append(record)
inn = inn + 1
tab = pandas.DataFrame(data0)
tab.index = para_names
tab = tab.sort(columns=['Importance'], ascending=False)
tab_text = tab.to_latex(float_format='{:,.2f}'.format, escape=False)
with open(home + table_out + 'classifier_table.txt', 'w') as f:
f.write(tab.to_latex(float_format='{:,.2f}'.format, escape=False, columns=['Importance', 'CS', 'SWA', 'OA', 'CS-HL']))
f.close()
pylab.ioff()
fig_width_pt = 350
inches_per_pt = 1.0 / 72.27
golden_mean = 1.2360679774997898 / 2.0
fig_width = fig_width_pt * inches_per_pt
fig_height = fig_width * golden_mean
fig_size = [fig_width, fig_height]
params = {'backend': 'ps',
'axes.labelsize': 10,
'text.fontsize': 10,
'legend.fontsize': 10,
'xtick.labelsize': 8,
'ytick.labelsize': 8,
'text.usetex': True,
'figure.figsize': fig_size}
pylab.rcParams.update(params)
plot_colors = 'rybg'
cmap = pylab.cm.RdYlBu
yi = numpara-1
minyi = -0.1
maxyi = 0.1
(xx, yy,) = np.meshgrid(np.arange(min(X[:, 1]), max(X[:, 1]), 0.02), np.arange(min(X[:, yi]), max(X[:, yi]), 0.01))
nnn = xx.ravel().shape[0]
Xlist = [np.mean(X[:,i])*np.ones(nnn) for i in range(numpara)]
Xlist[1] = xx.ravel()
Xlist[yi] = yy.ravel()
XX = np.array(Xlist).T
Z = treec.predict(XX).reshape(xx.shape)
fig = pylab.contourf(xx, yy, Z, [0, 0.9999, 1.9999, 2.9999, 3.9999], colors=('red', 'yellow', 'blue', 'green'), alpha=0.5, antialiased=False, extend='both')
for (i, c,) in zip(xrange(4), plot_colors):
idx0 = np.where(Y == i)
pylab.scatter(X[idx0, 1], X[idx0, yi], c=c, cmap=cmap, label=rows[i], s = 12, lw=0.5 )
pylab.legend(bbox_to_anchor=(0.0, 1.02, 1.0, 0.102), loc=3, ncol=4, mode='expand', borderaxespad=0.0)
pylab.xlabel('Mean inflow over capacity')
pylab.ylabel('Environmental inflow share')
pylab.ylim(minyi, maxyi)
OUT = home + out + 'class_fig.pdf'
pylab.savefig(OUT, bbox_inches='tight')
pylab.show()
def starPlots(truestarfile = "stardata.dat", bestfitstarfile = "test_bestfit_Star.dat"):
#assume both the files are sorted by star id.
stardat = cp.read_true_stars(truestarfile) #keys are ('id', 'dbid', 'ra', 'dec', 'magtrue', 'color')
starcal = ui.readDatafile(bestfitstarfile, keys=('id','magcal','rms','iqr_sigma','nobs'))
HP = True
if np.size(starcal['id']) == 0:
starcal = ui.readDatafile(bestfitstarfile, keys=('id','magcal'))
HP = False
fitted_stars = np.in1d(stardat['id'], starcal['id'])
for key in stardat.keys():
stardat[key] = stardat[key][fitted_stars]
for key in starcal.keys():
stardat[key] = starcal[key]
stardat['magdiff'] = stardat['magcal'] - stardat['magtrue']
stardat = cp.adjust_stars_calvstrue(stardat)
starResults = {}
HPResults = {}
starResults['N stars Fit'] = np.size(stardat['magdiff'])
outlier_clip=np.where(np.abs(stardat['magdiff']) < .05)
rms = stardat['magdiff'][outlier_clip].std()
print 'true-bestfit RMS = %f, true-bestfit (clipped) = %f'%(stardat['magdiff'].std(), rms)
lim = rms*2.5
if lim < 0.001:
lim = round(lim * 10000) / 10000.0
elif lim < 0.01:
lim = round(lim * 1000) / 1000.0
print "Using magnitude limits for plots of %f, %f" %(-lim, lim)
maglim= [-lim, lim]
starResults['Residuals RMS (mmag)'] = stardat['magdiff'].std()*1e3
starResults['IQR_sigma (mmag)'] = so.robustSigma(stardat['magdiff'])*1e3
#star residual map
stardat['x'], stardat['y'] = cpu.hammer_project_toxy(stardat['ra']*deg2rad,
stardat['dec']*deg2rad)
gridsize = cpu.calc_gridsize(stardat, binsize='patch')
etitle = " RMS = %4.1f mmag"%(rms*1000)
cpu.plot_density(stardat['x'], stardat['y'], stardat['magdiff']*1000, z_method=np.mean,
zlimits=np.array(maglim)*1000, gridsize=gridsize, radecgrid=True, newfig=True, cb_label='mmag')
figtitle = "Stars dMag(true-bestfit)"
pyl.title(figtitle, fontsize='medium')
pyl.savefig('Sdmag.png',type='png')
maglim2 = [0, maglim[1]/2.]
cp.plot_stars_dmagcaltrue(stardat, maglim=maglim2, etitle=" RMS "+etitle, z_method=np.std)
pyl.savefig('Sdmag_rms.png', format='png')
histrange = [maglim[0]*2.5, maglim[1]*2.5]
nbins = 100
cp.histogram_stars_dmagcaltrue(stardat, nbins=nbins, range=histrange, etitle=etitle)
pyl.savefig('Sdamg_hist.png', format='png')
#color distribution of stars
pyl.figure()
pyl.hist(stardat['color'], bins=100)
pyl.xlabel(r'$g-i$')
pyl.ylabel('# of stars')
pyl.title('Color Distribution')
pyl.savefig('Scolordist.png',type='png')
#accuracy v color
pyl.figure()
#pyl.scatter(stardat['color'], stardat['magdiff']*1000., c=stardat['magtrue'], alpha=0.1,edgecolors=None)
pyl.hexbin(stardat['color'], stardat['magdiff']*1000., bins='log')
cb=pyl.colorbar()
cb.ax.set_ylabel(r'$\log{\rm{N}}$')
pyl.xlabel(r'$g-i$')
pyl.ylabel('Bestfit-True (mmag)')
pyl.savefig('Saccuracyvcolor.png',type='png')
#accuracy v mag
pyl.figure()
#pyl.scatter(stardat['magtrue'], stardat['magdiff']*1000., c=stardat['color'], alpha=0.1,edgecolors=None)
pyl.hexbin(stardat['magtrue'], stardat['magdiff']*1000., bins='log')
cb=pyl.colorbar()
cb.ax.set_ylabel(r'$\log{\rm{N}}$')
pyl.xlabel('True Mag')
pyl.ylabel('Bestfit-True (mmag)')
pyl.savefig('Saccuracyvmag.png',type='png')
if HP:
zlim = [np.min(stardat['nobs']), (np.median(stardat['nobs'])-np.min(stardat['nobs']))+np.median(stardat['nobs'])]
cpu.plot_density(stardat['x'], stardat['y'],stardat['nobs'], z_method=np.mean,
zlimits=zlim, gridsize=gridsize, radecgrid=True, newfig=True, cb_label='N observations')
pyl.title('Visit Density')
pyl.savefig('Snobs.png', format='png')
pyl.figure()
#I should just figure out from the data what nsides is!
hpid = hp.ang2pix(16, (stardat['dec']+90.)*deg2rad, stardat['ra']*deg2rad)
cpu.plot_density(stardat['x'], stardat['y'],np.mod(hpid,250), z_method=np.median,
gridsize=gridsize, radecgrid=True, newfig=True, cb_label='HEALpix ID mod 250')
pyl.title("HEALpix Map")
pyl.savefig('HPid.png')
HPResults['number of HEALpix'] = np.size(np.unique(hpid))
pyl.figure()
num, b, p = pyl.hist(stardat['nobs'], bins=100, range=zlim, histtype='bar')
pyl.xlabel("Number of Observations per Star")
pyl.title('Median = %d'%np.median(stardat['nobs']))
pyl.savefig('Snobs_hist.png',type='png')
starResults['median repeat obs'] = np.median(stardat['nobs'])
pyl.figure()
x = np.sort(stardat['iqr_sigma'])*1e3
y = np.arange(np.size(x), dtype='float')
y = y/np.max(y)
per50 = np.max(np.where(y <= .5))
per90 = np.max(np.where(y <= .9))
pyl.plot(x,y, 'k--', label='IQR, 50th,90th %2.1f, %2.1f'%(x[per50],x[per90]))
pyl.plot([0,x[per50]],[y[per50],y[per50]],'b--')
pyl.plot([x[per50],x[per50]],[0,y[per50]],'b--')
pyl.plot([0,x[per90]],[y[per90],y[per90]],'b--')
pyl.plot([x[per90],x[per90]],[0,y[per90]],'b--')
#pyl.title('Cumulative Distribution of Stellar Repeatability')
#pyl.xlabel('Repeatability IQR RMS (mmag)')
#pyl.ylabel('Cumulative Fraction')
#pyl.savefig('Srepeat_IQR_cumulative.png',type='png')
pyl.legend()
pyl.savefig('Srepeat_cumulative.png',type='png')
#repeatability from IQR
rs = so.robustSigma(stardat['iqr_sigma'])
med = np.median(stardat['iqr_sigma'])
maglim = np.around(np.array([med-3*rs, med+3.*rs])*1000, decimals=2)/1000
cpu.plot_density(stardat['x'], stardat['y'], stardat['iqr_sigma']*1000, z_method=np.mean,
zlimits=maglim*1000, gridsize=gridsize, radecgrid=True, newfig=True, cb_label='mmag')
pyl.title('Repeatability (IQR)')
pyl.savefig('Srepeat_IQR.png',type='png')
pyl.figure()
num, b, p = pyl.hist(stardat['iqr_sigma']*1000, bins=100, range=maglim*1000, histtype='bar')
pyl.xlabel('RMS per star (mmag)')
pyl.title('Repeatability IQR, Median = %4.2f mmag'%np.median(stardat['iqr_sigma']*1000))
pyl.savefig('Srepeat_IQR_hist.png',type='png')
starResults['Median Repeatability (IQR) (mmag)'] = np.median(med)*1e3
pyl.figure()
good = np.where(stardat['magtrue'] < 18)
num, b, p = pyl.hist(stardat['iqr_sigma'][good]*1000, bins=100, range=maglim*1000, histtype='bar')
pyl.xlabel('RMS per star, m < 18 (mmag)')
pyl.title('Repeatability IQR, Median = %4.2f mmag'%np.median(stardat['iqr_sigma'][good]*1000))
pyl.savefig('Srepeat_IQR_bright_hist.png',type='png')
starResults['Median Repeatability Bright (IQR) (mmag)'] = np.median(stardat['iqr_sigma'][good])*1e3
#repeatability v color
pyl.figure()
#pyl.plot(stardat['color'], stardat['rms']*1000., 'ko', alpha=.1)
pyl.hexbin(stardat['color'], stardat['iqr_sigma']*1000., bins='log')
cb = pyl.colorbar()
cb.set_label(r'$\log{\rm{N}}$')
pyl.ylim([0,40])
pyl.xlabel(r'$g-i$')
pyl.ylabel(r'Repeatability $\sigma_{\rm{IQR}}$ (mmag)')
pyl.savefig('Srepeatvcolor.png',type='png')
#repeatability v mag
pyl.figure()
#pyl.plot(stardat['magtrue'], stardat['rms']*1000., 'ko', alpha=.1)
pyl.hexbin(stardat['magtrue'], stardat['iqr_sigma']*1000.,bins='log')
cb = pyl.colorbar()
cb.set_label(r'$\log{\rm{N}}$')
pyl.ylim([0,40])
pyl.xlabel('True Mag')
pyl.ylabel(r'Repeatability $\sigma_{\rm{IQR}}$ (mmag)')
pyl.savefig('Srepeatvmag.png',type='png')
#repeatability
rs = so.robustSigma(stardat['rms'])
med = np.median(stardat['rms'])
maglim = np.around(np.array([med-3*rs, med+3.*rs])*1000, decimals=2)/1000
cpu.plot_density(stardat['x'], stardat['y'], stardat['rms']*1000, z_method=np.mean,
zlimits=maglim*1000, gridsize=gridsize, radecgrid=True, newfig=True, cb_label='mmag')
pyl.title('Repeatability')
pyl.savefig('Srepeat.png',type='png')
pyl.figure()
num, b, p = pyl.hist(stardat['rms']*1000, bins=100, range=maglim*1000, histtype='bar')
pyl.xlabel('RMS per star (mmag)')
pyl.title('Repeatability, Median = %4.2f mmag'%np.median(stardat['rms']*1000))
pyl.savefig('Srepeat_hist.png',type='png')
starResults['Median Repeatability (mmag)'] = np.median(med)*1e3
pyl.figure()
x = np.sort(stardat['rms'])*1e3
y = np.arange(np.size(x), dtype='float')
y = y/np.max(y)
per50 = np.max(np.where(y <= .5))
per90 = np.max(np.where(y <= .9))
pyl.plot(x,y, 'k', label='RMS, %2.1f, %2.1f'%(x[per50],x[per90]))
pyl.plot([0,x[per50]],[y[per50],y[per50]],'k')
pyl.plot([x[per50],x[per50]],[0,y[per50]],'k')
pyl.plot([0,x[per90]],[y[per90],y[per90]],'k')
pyl.plot([x[per90],x[per90]],[0,y[per90]],'k')
pyl.title('Cumulative Distribution of Stellar Repeatability')
pyl.xlabel('Repeatability RMS (mmag)')
pyl.ylabel('Cumulative Fraction')
pyl.savefig('Srepeat_cumulative.png',type='png')
#need to make a spatial uniformity plot since we can now have varying densities of stars.
nside = 64
stardat['hpid'] = hp.ang2pix(nside, (stardat['dec']+90.)*deg2rad, stardat['ra']*deg2rad)
uhpid = np.unique(stardat['hpid'])
hp_mean = uhpid*0.
hp_std = uhpid*0.
hp_dec, hp_ra = hp.pix2ang(nside,uhpid)
#hp_ra = hp_ra
hp_dec = hp_dec-90.*deg2rad
hp_x, hp_y = cpu.hammer_project_toxy(hp_ra, hp_dec)
stardat = dictsort(stardat,'hpid')
left = np.searchsorted(stardat['hpid'], uhpid)
right = np.searchsorted(stardat['hpid'], uhpid,side='right')
for i in np.arange(left.size):
hp_mean[i] = np.mean(stardat['magdiff'][left[i]:right[i]])
hp_std[i] = np.std(stardat['magdiff'][left[i]:right[i]])
cpu.plot_density(hp_x, hp_y, hp_mean*1000, z_method=np.mean,
zlimits=None, gridsize=gridsize, radecgrid=True, newfig=True, cb_label='mmag')
HPResults['Spatial Uniformity RMS (mmag)'] = np.std(hp_mean)*1e3
HPResults['Spatial Uniformity IQR RMS (mmag)'] = so.robustSigma(hp_mean)*1e3
pyl.title('mean(True-Bestfit), binned by HEALpix')
pyl.savefig('HPresid.png',type='png')
cpu.plot_density(hp_x, hp_y, hp_std*1000, z_method=np.mean,
zlimits=None, gridsize=gridsize, radecgrid=True, newfig=True, cb_label='mmag')
pyl.title('std(True-Bestfit), binned by HEALpix')
pyl.savefig('HPstd.png',type='png')
pyl.figure()
num, b, p = pyl.hist( hp_mean*1000, bins=100, range=None, histtype='bar')
pyl.xlabel("HEALpix(True-Bestfit) (mmag)")
pyl.title('Spatial Uniformity, RMS=%4.2f'%HPResults['Spatial Uniformity RMS (mmag)'])
pyl.savefig('HPmean_hist.png',type='png')
pyl.figure()
num, b, p = pyl.hist( hp_std*1000, bins=100, range=None, histtype='bar')
pyl.xlabel("std(True-Bestfit) per HEALpix (mmag)")
pyl.title('Variation within HEALpix, median=%4.2f mmag'%np.median(hp_std*1000))
pyl.savefig('HPstd_hist.png',type='png')
return starResults,HPResults
def plot_fit_kde(self):
"""
Compares the kde estimation of the joint distribution of axis ratios
and sersic indices
"""
# Extracts variables of interest
mag = self.cat['mag_auto'][self.mask]
I = np.log10(self.cat['sersicfit'][self.mask, 0])
R = np.log10(self.cat['sersicfit'][self.mask, 1])
n = np.log10(self.cat['sersicfit'][self.mask, 2])
q = np.log10(self.cat['sersicfit'][self.mask, 3])
plt.figure(figsize=(15, 5))
plt.subplot(131)
plt.hexbin(n,
q,
gridsize=50,
cmap='jet',
bins='log',
extent=(-1, 0.8, -1.2, 0))
plt.xlabel("$\log_{10}(n)$")
plt.ylabel("$\log_{10}(q)$")
plt.title('without magnitude cuts')
plt.subplot(132)
nobjs = len(n)
s = self.shape_kde.resample(nobjs)
plt.hexbin(np.clip(s[0, :], -1, 0.778),
np.clip(s[1, :], -1.2, 0),
gridsize=50,
cmap='jet',
bins='log',
extent=(-1, 0.8, -1.2, 0))
plt.xlabel("$\log_{10}(n)$")
plt.ylabel("$\log_{10}(q)$")
plt.title('Sampled from KDE')
plt.subplot(133)
ind = (mag > self.mag_range[0]) * (mag < self.mag_range[1])
plt.hexbin(n[ind],
q[ind],
gridsize=50,
cmap='jet',
bins='log',
extent=(-1, 0.8, -1.2, 0))
plt.xlabel("$\log_{10}(n)$")
plt.ylabel("$\log_{10}(q)$")
plt.title('within fitting magnitude range')
plt.figure(figsize=(15, 5))
plt.subplot(121)
plt.hist(q[ind],
50,
label='within fitting region',
normed=True,
alpha=0.5)
plt.hist(np.clip(s[1, :], -1.2, 0),
50,
label='KDE samples',
normed=True,
alpha=0.5)
plt.hist(q, 50, label='without cuts', normed=True, alpha=0.5)
plt.legend(loc=2)
plt.xlabel("$\log_{10}(q)$")
plt.subplot(122)
plt.hist(n[ind],
50,
label='within fitting region',
normed=True,
alpha=0.5)
plt.hist(np.clip(s[0, :], -1, 0.778),
50,
label='KDE samples',
normed=True,
alpha=0.5)
plt.hist(n, 50, label='without cuts', normed=True, alpha=0.5)
plt.legend(loc=2)
plt.xlabel("$\log_{10}(n)$")
def opt_rates(astrom, dates, orbits):
dt = (dates[1] - dates[0]) * (86400.0)
dt_scale = 384.0 / dt
#print dt, dt_scale
sRA, sDEC = [], []
#print datev[1:] - datev[:-1]
x, y, c = [], [], []
x2, y2, c2 = [], [], []
FILE = open('loc.dat', 'w')
FILE.write('#$%s\n' % (dates[0] + 2400000.5))
for i in range(0, len(astrom)):
RA, DEC = float(astrom[i][0][0]), float(astrom[i][0][1])
#print RA, DEC
FILE.write('%s %s\n' % (RA, DEC))
color = float(orbits[i].split()[-1])
x.append(RA * math.pi / 180.0)
y.append(DEC * math.pi / 180.0)
c.append(color)
FILE.close()
for i in range(0, len(astrom)):
RA, DEC = float(astrom[i][-1][0]), float(astrom[i][-1][1])
#print RA, DEC
color = float(orbits[i].split()[-1])
x2.append(RA * math.pi / 180.0)
y2.append(DEC * math.pi / 180.0)
c2.append(color)
mx1 = numpy.median(x)
mx2 = numpy.median(x2)
mex, mey = numpy.mean(x), numpy.mean(y)
my1 = numpy.median(y)
my2 = numpy.median(y2)
#print 'Offset between median and mean:'
#print (mx1- mex)*math.cos( mx1 ) * 180.0 * 3600.0 / math.pi
#print (my1- mey) *180.0 * 3600.0 / math.pi
#print '------------'
dx1 = numpy.asarray(x - mx1) * math.cos(my1)
dx2 = numpy.asarray(x2 - mx2) * math.cos(my2)
dy1 = numpy.asarray(y - my1)
dy2 = numpy.asarray(y2 - my2)
c = []
for i in range(0, len(astrom)):
#RA0, DEC0 = float(astrom[i][0][0]), float(astrom[i][0][1])
shiftRA, shiftDEC = [], []
#for j in range(1, len(datev)):
# if abs( datev[j] - datev[j-1] ) > 2.0E-2:
# ### skip if longer than an orbit between
# continue
if 0 == 0:
RA, DEC = float(astrom[i][1][0]), float(astrom[i][1][1])
#print RA, DEC
RA0, DEC0 = float(astrom[i][0][0]), float(astrom[i][0][1])
#print RA0, DEC0, RA, DEC
dRA, dDEC = math.cos(DEC0 * math.pi / 180.0) * (RA - RA0), (
DEC - DEC0) ### delta-degrees?
#shiftRA.append( dRA )
#shiftDEC.append( dDEC )
#print len(shiftRA), len(datev)
sRA.append(dRA)
sDEC.append(dDEC)
print datev[0] - datev[1]
sRA, sDEC = numpy.asarray(sRA) * 3600.0, numpy.asarray(sDEC) * 3600.0
print math.sqrt((max(sRA) - min(sRA))**2 + (max(sDEC) - min(sDEC))**2)
#dv = []
#for i in range(0, len(sRA)):
# deltaRA = ( sRA - sRA[i] )**2
# deltaDEC = ( sDEC - sDEC[i] )**2
# delta = numpy.percentile( numpy.sqrt( numpy.mean( deltaRA + deltaDEC, axis=1 )), 95 )
# dv.append( delta )
# c.append( delta )
epsilon = 0.04
dx = math.sqrt(3) * epsilon
dy = 1.5 * epsilon
xg1 = numpy.arange(min(sRA), max(sRA), dx)
xg2 = numpy.arange(min(sRA) + 0.5 * dx, max(sRA) + 0.5 * dx, dx)
yg1 = numpy.arange(min(sDEC), max(sDEC), 2.0 * dy)
yg2 = numpy.arange(min(sDEC) + dy, max(sDEC) + dy, 2.0 * dy)
X1, Y1 = numpy.meshgrid(xg1, yg1)
X2, Y2 = numpy.meshgrid(xg2, yg2)
X = numpy.append(X1.ravel(), X2.ravel())
Y = numpy.append(Y1.ravel(), Y2.ravel())
print len(X)
x0 = [0.01, 0.01, 0.1]
x1 = fmin(grid, x0, args=(
sRA,
sDEC,
))
count, Xo, Yo = grid(x1, sRA, sDEC, epsilon=0.04, full=True)
pylab.ion()
fig = pylab.figure(figsize=(8, 8))
ax = fig.add_subplot(111, aspect='equal')
pylab.hexbin(sRA,
sDEC,
bins='log',
cmap=pylab.cm.binary,
gridsize=(100, 100),
extent=(numpy.median(sRA) - 0.6, numpy.median(sRA) + 0.6,
numpy.median(sDEC) - 0.6, numpy.median(sDEC) + 0.6))
ax.scatter(Xo, Yo, color='r', s=2)
for i in range(0, len(Xo)):
ax.add_artist(
Circle(xy=(Xo[i], Yo[i]),
radius=epsilon,
edgecolor='r',
facecolor='none'))
ax.set_xlabel(r'$\Delta\alpha$ (arcsec)')
ax.set_ylabel(r'$\Delta\delta$ (arcsec)')
ax.set_xlim(numpy.median(sRA) - 0.6, numpy.median(sRA) + 0.6)
ax.set_ylim(numpy.median(sDEC) - 0.6, numpy.median(sDEC) + 0.6)
#r = numpy.sqrt( ( (dx1-dx2)*180.0 *3600.0/ (math.pi))**2 + ( (dy1-dy2)*180.0 *3600.0/ (math.pi))**2 )
#r2 = numpy.sqrt( ( (dx1)*180.0 *3600.0/ (math.pi))**2 + ( (dy1)*180.0*3600.0/ (math.pi))**2 )
#pylab.scatter( r, r2, c=c )
#pylab.scatter( [r[ numpy.argmin( r ) ]], [r2[ numpy.argmin( r ) ]], s=100 )
pylab.draw()
pylab.draw()
#print numpy.argmin( r )
#print dx1[ numpy.argmin( r ) ]*180.0 *3600.0/ (math.pi), dy1[ numpy.argmin( r ) ]*180.0 *3600.0/ (math.pi)
pylab.savefig('HST_rategrid.png')
pause = raw_input('...')
#pylab.clf()
#pylab.hist( dt_scale * numpy.asarray( dv ) * 3600.0 / 0.04, bins=30 )
#pylab.draw()
#pylab.draw()
#v = numpy.asarray(orbits)[ numpy.argsort( dv ) ][0:50]
#d, sma = [],[]
#for k in v:
# k = k.strip().split()
# d.append( float( k[-1] ) )
# sma.append( float(k[0] ) )
#print min(d), max(d)
#print min(sma), max(sma)
#pause = raw_input('...')
return 0
def hex_difference(xy1,xy2,show_all=False,hkwargs={},color_bar=True,fignum=1):
"""A function that plots the difference between two hexbin plots.
Parameters
----------
xy1 : A tuple of (x,y) corrdianates for the first hexbin. A tuple of (x,y,C)
can also be passed in where C is the value for each (x,y) point.
xy2 : A tuple of (x,y) corrdianates for the second hexbin. A tuple of (x,y,C)
can also be passed in where C is the value for each (x,y) point.
NOTE : the 'C' functinality is untested and may not work as expected.
Keywords
--------
show_all : bool (optional)
If True all intermediate hexbin plots are returned.
Default: show_all=False
color_bar : bool (optional)
If True a colorbar is placed on the plot(s)
Default: colorbar=True
fignum : int (optional)
The number to give the resulting figure(s). If
show_all=True, the intermediate plots will be
fignum+1 and fignum+2 while the difference will
be fignum.
default: fignum=1
Passed Keywords
---------------
hkwargs:
a dictionary of keywords passed to hexbin
(see http://matplotlib.org/api/pyplot_api.html#matplotlib.pyplot.hexbin
for additional keywords that can be set)
Returns
-------
d : pylab.hexbin object
Object returned by the difference hexbin
h1 : pylab.hexbin object
Object returned by hexbin of first data set
NOTE: only returned if show_all=True
h2 : pylab.hexbin object
Object returned by hexbin of second data set
NOTE: only returned if show_all=True
c : matplotlib.colorbar.Colorbar instance
NOTE: only returned if color_bar=True
Usage
-----
import numpy as np
n=100000
x1=np.random.standard_normal(n) #random x points
y1=2+3*x1+4*np.random.standard_normal(n) #random y points
x2=np.random.standard_normal(n) #random x points
y2=2-3*x2+4*np.random.standard_normal(n) #random y points
hex_difference((x1,y1),(x2,y2),show_all=True,color_bar=True,hkwargs={'gridsize':100,'extent':[-4.5,4.5,-25,25],'vmin':-180,'vmax':180})
pl.show()
"""
if show_all: #if shoing all hexbins then draw them as you go (you can't change the drawing axis object after creation)
pl.figure(fignum+1)
hex1=pl.hexbin(*xy1,**hkwargs)
if color_bar:
pl.colorbar()
pl.figure(fignum+2)
hex2=pl.hexbin(*xy2,**hkwargs)
if color_bar:
pl.colorbar()
else: #make but don't draw the 2 hexbins
hex1=pl.hexbin(*xy1,visible=False,**hkwargs) #make the hexbins, visible it False to avoid drawing them to a plot
hex2=pl.hexbin(*xy2,visible=False,**hkwargs)
pl.figure(fignum)
hex_dif=pl.hexbin(*xy1,visible=False,**hkwargs) #this will have the counts overwritten (so don't draw yet)
c1=hex1.get_array() #the counts for hex1
c2=hex2.get_array() #the counts for hex2
c_dif=c1-c2 #difference between plots
gdx=~((c1==0)&(c2==0)) #the bins to draw (removes where both hists had no counts)
#NOTE: if the 'C' values are set checking against 0 is NOT a good idea...
hex_dif.set_array(c_dif[gdx]) #set the defferences into the hex_dif object
h=hex_dif.get_paths() #get the hexagon Path object(s)
if len(h)>1: #you have an old version of matplotlib, use this bit of code
rem_me=pl.array(h)[~gdx] #bins to remove
for r in rem_me:
h.remove(r) #remove blank bins
else: #either you have a boaring hexbin or a newer version of matplotlib
h=hex_dif.get_offsets()
hex_dif.set_offsets(h[gdx])
hex_dif.set_visible(True) #this draws the new hex_dif
ret=[hex_dif]
if show_all:
ret.append(hex1)
ret.append(hex2)
if color_bar:
ains=inset_axes(pl.gca(),width='80%',height='5%',loc=9)
#TODO: externalize colorbar keywords
c=pl.colorbar(hex_dif,cax=ains,orientation='horizontal')
ret.append(c)
return tuple(ret)
def hex_contour(x, y, min_cnt=2, std=False, levels=2, smoothing=3,
hkwargs={}, skwargs={}, ckwargs={}):
"""Plot density contours with outliers plotted as scatter points.
Parameters
----------
x : array_like
the x coordinates for the input data
y : array_like
the y coordinates for the input data
Keywords
--------
min_cnt: integer (optional)
Minimum number of hex bin counts to draw contour. Bins
containing less than or equal to this will be scatter points.
Default: min_cnt = 2
std : bool (optional)
Draw contours of the fraction of points contained within each
level. The levels are set using the levels keword (see below).
Default: std = False
levels: integer or array (optional)
Indicates either the number of contour levels to draw (integer),
or an array of levels to draw. If integer and std = True, levels
are drawn at [1, 2, 3, 4, 5][:levels] simga, otherwise the
levels will be evenly spaced in density from min_cnt upto the
maximum bin count. If array and std = True, the values indicate
the fraction of points contained within each contour, otherwise
they indicate the number of points in each bin.
Default: levels = 2
smoothing: integer (optional)
Is used to subdivide (i.e. smooth) the contouring grid.
Default: smoothing = 3, values from 1 to 4 are recommended.
See subdiv on: http://matplotlib.org/1.3.0/api/tri_api.html?highlight=uniformtrirefiner#matplotlib.tri.UniformTriRefiner
Passed Keywords
---------------
hkwargs:
a dictionary of keywords passed to hexbin
skwargs:
a dictionary of keywords passed to plot (for scatter points)
ckwargs:
a dictionary of keywords passed to tricontour (if std is set)
Returns
-------
f: pylab.hexbin object
Object returned by hexbin
P: pylab.plot object
Object returned by plot
T: pylab.tricontour object
Object returned by tricontour
"""
# set the default scales on x and y axis to linear
hkwargs.setdefault('xscale', 'linear')
hkwargs.setdefault('yscale', 'linear')
# hexbin the data to a mincnt of 0 (needed for sig_cont to look good),
# visible = False so it does not draw
f = pl.hexbin(x, y, mincnt=1, visible=False, **hkwargs)
# NOTE: may have to change the mincnt if the C keyword is set...
c = f.get_array() # get the number of points in each bin
H = f.get_paths() # get the hexagon Path objects
# if binning is linear hexbin uses the new convention
if (hkwargs['xscale'] == 'linear') and (hkwargs['yscale'] == 'linear'):
h = f.get_offsets() # the centers for each hex
xs = h.T[0]
ys = h.T[1]
# if bining is non-linear it uses the old convention
else:
# get the centers of each hexagon
xs = np.array([(0.5 * (i.get_extents().xmax - i.get_extents().xmin) + \
i.get_extents().xmin) for i in H])
ys = np.array([(0.5 * (i.get_extents().ymax - i.get_extents().ymin) + \
i.get_extents().ymin) for i in H])
if std: # check if you are making standard deviation curves
# convert the hexbin counts to fraction of the total number of points
c1 = convert_to_stdev(c)
# if you just indicate the number of contours you want
if isinstance(levels, int):
ckwargs['levels'] = [0.682689492, 0.954499736, 0.997300204,
0.99993666, 0.999999426697][:levels]
else:
ckwargs['levels'] = levels
# set where the last contour goes and where outlier points begin
cmin_std = max(ckwargs['levels'])
# mask for all bins with less than the min_cnt or less than cmin_std
idx = ((c1 > cmin_std) | (c <= min_cnt)) & (c > 0)
# make Triangulation object using the centers of each of the hexbins
# plot contours
T = triangle_contour(xs, ys, c1, smoothing, ckwargs)
else: # if you are making normal contours
if isinstance(levels, int):
# default to contours from min_cnt to just below the maximum value
# in even steps.
ckwargs['levels'] = np.linspace(min_cnt, c.max(), levels + 1)[:-1]
else:
ckwargs['levels'] = levels
idx = (c <= min_cnt) & (c > 0) # mask for all bins with less than the min_cnt
# plot contours
T = triangle_contour(xs, ys, c, smoothing, ckwargs)
xout = pl.array([]) # array to hold x scatter points
yout = pl.array([]) # array to hold y scatter points
# if binning is linear hexbin uses the new convention
if (hkwargs['xscale'] == 'linear') and (hkwargs['yscale'] == 'linear'):
out = h[idx] # only the hexagons with less than min_cnt
H = H[0] # there is only hexagon one path now centered at 0,0
ext = H.get_extents() # the extents of this one hexagon
for o in out:
# move data to current bin's center
xnow = x - o[0]
ynow = y - o[1]
# get the index of the points in extent of the hexagon
jdx = (xnow >= ext.xmin) & (xnow <= ext.xmax) & \
(ynow >= ext.ymin) & (ynow <= ext.ymax)
# loop through each point in extent of the hexagon
for p in zip(xnow[jdx], ynow[jdx]):
# if the point is actual in the hexagon add it to the list of
# outliers
if H.contains_point(p):
xout = np.append(xout, p[0] + o[0])
yout = np.append(yout, p[1] + o[1])
# if binning is non-linear it uses the old convention
else:
out = pl.array(H)[idx] # hexagons to be cut
for o in out: # loop over Paths
H.remove(o) # remove hexagons below cutoff
ext = o.get_extents() # get the x,y extent of the hexagon
# mask for data points only in the hexagon extent
jdx = (x >= ext.xmin) & (x <= ext.xmax) & \
(y >= ext.ymin) & (y <= ext.ymax)
for p in zip(x[jdx], y[jdx]): # loop over these points
if o.contains_point(p): # check if point is in the hexagon
xout = np.append(xout, p[0]) # if so append the x value
yout = np.append(yout, p[1]) # if so append the y value
# set default type and size for outlier points
skwargs.setdefault('marker', '.')
skwargs.setdefault('ms', 2)
skwargs.setdefault('ls', 'None')
P = pl.plot(xout, yout, **skwargs) # plot the outlier points
return T, P
filesave='cc_bundy.png'
#make the cut for blue galaxies
colorgr=g-r
colorri=r-i
idgri=numpy.where((colorgr<2) & (colorgr>-1) & (colorri<2) & (colorri>-1) & (TYPE==3))
print idgri[0].shape
colorgr=colorgr[idgri]
colorri=colorri[idgri]
x0=[0,2]
y0=[1.1,1.1]
y1=[0.45,0.45]
x2=[0.85,0.85]
y2=[0,2]
pylab.hexbin(colorri,colorgr,gridsize=300)
pylab.plot(x0,y0,'r')
pylab.plot(x0,y1,'r')
pylab.plot(x2,y2,'r')
pylab.xlim(0,1)
pylab.ylim(0,2)
pylab.xlabel('r-i')
pylab.ylabel('g-r')
pylab.savefig(filesave)
u_err=table.field('ERR_U')
g_err=table.field('ERR_G')
r_err=table.field('ERR_R')
i_err=table.field('ERR_I')
z_err=table.field('ERR_Z')
for n_components in n_components_range:
t0 = time()
rp = SparseRandomProjection(n_components=n_components)
projected_data = rp.fit_transform(data)
print("Projected %d samples from %d to %d in %0.3fs"
% (n_samples, n_features, n_components, time() - t0))
if hasattr(rp, 'components_'):
n_bytes = rp.components_.data.nbytes
n_bytes += rp.components_.indices.nbytes
print("Random matrix with size: %0.3fMB" % (n_bytes / 1e6))
projected_dists = euclidean_distances(
projected_data, squared=True).ravel()[nonzero]
pl.figure()
pl.hexbin(dists, projected_dists, gridsize=100)
pl.xlabel("Pairwise squared distances in original space")
pl.ylabel("Pairwise squared distances in projected space")
pl.title("Pairwise distances distribution for n_components=%d" %
n_components)
cb = pl.colorbar()
cb.set_label('Sample pairs counts')
rates = projected_dists / dists
print("Mean distances rate: %0.2f (%0.2f)"
% (np.mean(rates), np.std(rates)))
pl.figure()
pl.hist(rates, bins=50, normed=True, range=(0., 2.))
pl.xlabel("Squared distances rate: projected / original")
pl.ylabel("Distribution of samples pairs")
samples_y = []
normalization = truncnorm.pdf(0, -1.0, 1.0) ** 2.
for sample in xrange(nsamples):
while True:
if uniform:
x = random.uniform(-1.0, 1.0)
y = random.uniform(-1.0, 1.0)
else:
x = gauss_cut()
y = gauss_cut()
p = math.exp(-0.5 * (x ** 2 + y ** 2) - alpha * (x ** 4 + y ** 4))
if not uniform:
p = p/(truncnorm.pdf(x, -1.0, 1.0) * truncnorm.pdf(y, -1.0, 1.0) / normalization)
if random.uniform(0.0, 1.0) < p:
break
samples_x.append(x)
samples_y.append(y)
pylab.hexbin(samples_x, samples_y, gridsize=50, bins=1000)
pylab.axis([-1.0, 1.0, -1.0, 1.0])
cb = pylab.colorbar()
pylab.xlabel('x')
pylab.ylabel('y')
pylab.title('A1_1')
if uniform:
pylab.savefig('plot_A1_1_uniform.png')
else:
pylab.savefig('plot_A1_1.png')
pylab.show()
sample &= (t.m_r < 17.77)
NUV_r = t.NUV[sample] - t.r[sample]
g_r = t.g[sample] - t.r[sample]
bounds = [.5,7,.2,.9]
h, ex, ey = np.histogram2d(NUV_r, g_r, bins=30,
range=[[bounds[0], bounds[1]],[bounds[2], bounds[3]]])
set_eps_output_1()
pylab.figure()
pylab.axis(bounds)
#pylab.title('Densidade')
pylab.xlabel('$\\mathrm{NUV} - r$')
pylab.ylabel('$g - r$')
pylab.hexbin(NUV_r, g_r, extent=bounds, bins='log', gridsize=50, cmap=cm.OrRd)
cb = pylab.colorbar()
cb.set_label('$\\log{N}$')
pylab.contour(h.T, extent=bounds, colors='black', linewidths=0.5)
if debug:
pylab.show()
#exit()
else:
pylab.savefig('../doc/figuras/uvcolor-color-density.' + outformat, format=outformat)
# mcor_gal : massa em estrelas
# at_flux: idade ponderada em fluxo
# at_mass: idade em massa
# am_flux: metalicidade em fluxo
# am_mass: metalicidade em massa
# AV: extincao
def main():
if len(sys.argv) < 2:
sys.exit('Usage: ' + sys.argv[0] + ' [Accounting files]')
else:
joblist = sys.argv[1:]
jobs = jobstats.alljobs(joblist)
qts_ppn = list()
qts_procs = list()
corehrs_ppn = list()
corehrs_procs = list()
for job in jobs:
if job.cores > 1 and job.tiq > 0 and not np.isnan(job.tiq):
if job.ppn == 0:
qts_procs.append(np.log10(job.tiq/3600.0))
corehrs_procs.append(np.log10(job.cores*job.walltimereq/3600.0))
else:
qts_ppn.append(np.log10(job.tiq/3600.0))
corehrs_ppn.append(np.log10(job.cores*job.walltimereq/3600.0))
subx_procs = list()
suby = list()
for i,j in zip(corehrs_procs,qts_procs):
if j > -1.0 and i > -1.0:
subx_procs.append(i)
suby.append(j)
z_procs = np.polyfit(subx_procs,suby,1)
p_procs = np.poly1d(z_procs)
print(z_procs)
subx_ppn = list()
suby = list()
for i,j in zip(corehrs_ppn, qts_ppn):
if j > -1.0 and i > -1.0:
subx_ppn.append(i)
suby.append(j)
z_ppn = np.polyfit(subx_ppn,suby,1)
p_ppn = np.poly1d(z_ppn)
print(z_ppn)
plt.cla()
plt.hexbin(corehrs_procs, qts_procs, bins='log')
#plt.plot([min(corehrs_procs), max(corehrs_procs)], [min(qts_procs), max(qts_procs)], 'k--')
plt.plot(subx_ppn, p_ppn(subx_ppn), 'y', linewidth=4)
plt.plot(subx_procs, p_procs(subx_procs), 'w', linewidth=4)
plt.xlabel('log(Core hours requested)')
plt.ylabel('log(Queue Time (hours))')
plt.savefig('procs.png')
plt.cla()
plt.hexbin(corehrs_ppn, qts_ppn, bins='log')
# plt.plot([min(qts_ppn), max(qts_ppn)], [min(qts_ppn), max(qts_ppn)], 'k--')
plt.plot(subx_procs, p_procs(subx_procs), 'w', linewidth=4)
plt.plot(subx_ppn, p_ppn(subx_ppn),'y', linewidth=4)
plt.xlabel('log(Core hours requested)')
plt.ylabel('log(Queue Time (hours))')
plt.savefig('ppn.png')
if ( rhat <= 1.05 ):
npass += 1
rhat_prev = rhat
else:
if npass > 0:
npass = 0
if npass > 3: break
if iteration>MAX_ITERATIONS:
self._tweet ( "Maximum number of iterations exceeded. Bailing out...")
break
accept = numpy.append ( accept, numpy.zeros (self.n_generations+1), 0)
Z_diagnostic = numpy.append(Z_diagnostic, numpy.zeros ((Npar, self.num_population, self.n_generations+1)), 2)
self._tweet ("Finished simulation!")
self._tweet ( "Returned %d samples"%int(m0+iteration*numpy.floor(self.n_burnin/self.n_thin)))
return ( Z, accept/self.num_population )
# return (Z[:,:int(m0+iteration*numpy.floor(self.n_burnin/self.n_thin))], accept/self.num_population)
if __name__=="__main__":
DEMC = DEMC_sampler ( 4, n_generations=400, n_burnin=1, n_thin=1, logger="test_me.log")
parameter_list=[['x1', 'scipy.stats.norm(-100, 20)'], ['x2', 'scipy.stats.norm(-100, 20)']]
parameters = ['x1','x2']
DEMC.prior_distributions ( parameter_list, parameters )
Z = DEMC.ProposeStartingMatrix ( 150 )
(Z_out, accept_rate) = DEMC.demc_zs ( Z )
import pylab
pylab.figure();pylab.plot ( Z_out[0,:], Z_out[1,:], 'k,')
pylab.figure();pylab.hist(Z_out[0,:],bins=10);pylab.hist(Z_out[1,:],bins=10);
pylab.figure();pylab.hexbin(Z_out[0,:], Z_out[1,:], bins='log')
pylab.show()
with Model() as model:
x = Normal('x', 0, 1)
y = Normal('y', 0, 1)
N = 200
d = Normal('d', x + y**2, 1., observed=np.zeros(N))
start = model.test_point
h = np.ones(2) * np.diag(find_hessian(start))[0]
step = HamiltonianMC(model.vars, h, path_length=4.)
trace = sample(3e3, step, start)
pl.figure()
pl.hexbin(trace['x'], trace['y'])
# lets plot the samples vs. the actual distribution
from theano import function
xn = 1500
yn = 1000
xs = np.linspace(-3, .25, xn)[np.newaxis, :]
ys = np.linspace(-1.5, 1.5, yn)[:, np.newaxis]
like = (xs + ys**2)**2 * N
post = np.exp(-.5 * (xs**2 + ys**2 + like))
post = post
pl.figure()
extent = np.min(xs), np.max(xs), np.min(ys), np.max(ys)
while True:
x = random.gauss(0.0, 1.0)
if abs(x) <= 1.0:
return x
alpha = 0.5
nsteps = 1000000
samples_x = []
samples_y = []
x, y = 0.0, 0.0
for step in range(nsteps):
xnew = gauss_cut()
ynew = gauss_cut()
exp_new = -0.5 * (xnew**2 + ynew**2) - alpha * (xnew**4 + ynew**4)
exp_old = -0.5 * (x**2 + y**2) - alpha * (x**4 + y**4)
if random.uniform(0.0, 1.0) < math.exp(exp_new - exp_old):
x = xnew
y = ynew
samples_x.append(x)
samples_y.append(y)
pylab.hexbin(samples_x, samples_y, gridsize=50, bins=1000)
pylab.axis([-1.0, 1.0, -1.0, 1.0])
cb = pylab.colorbar()
pylab.xlabel('x')
pylab.ylabel('y')
pylab.title('A3_1')
pylab.savefig('plot_A3_1.png')
pylab.show()
def rp_johnson(X, problem):
'''
eps_range = np.linspace(0.1, 0.99, 5)
n_samples_range = np.logspace(2, 6, 5)
colors = pl.cm.Blues(np.linspace(0.3, 1.0, len(n_samples_range)))
pl.figure()
for n_samples, color in zip(n_samples_range, colors):
min_n_components = johnson_lindenstrauss_min_dim(n_samples, eps=eps_range)
pl.semilogy(eps_range, min_n_components, color=color)
pl.legend(["n_samples = %d" % n for n in n_samples_range], loc="upper right")
pl.xlabel("Distortion eps")
pl.ylabel("Minimum number of dimensions")
pl.title("Johnson-Lindenstrauss bounds:\nn_components vs eps")
pl.show()
'''
X = X[:5000]
n_samples, n_features = X.shape
print("Embedding %d samples with dim %d using various random projections" %
(n_samples, n_features))
n_components_range = np.array([2, 5, 10, 20, 60, 80, 100, 120])
dists = euclidean_distances(X, squared=True).ravel()
# select only non-identical samples pairs
nonzero = dists != 0
dists = dists[nonzero]
for n_components in n_components_range:
t0 = time()
rp = SparseRandomProjection(n_components=n_components)
projected_data = rp.fit_transform(X)
print("Projected %d samples from %d to %d in %0.3fs" %
(n_samples, n_features, n_components, time() - t0))
if hasattr(rp, 'components_'):
n_bytes = rp.components_.data.nbytes
n_bytes += rp.components_.indices.nbytes
print("Random matrix with size: %0.3fMB" % (n_bytes / 1e6))
projected_dists = euclidean_distances(projected_data,
squared=True).ravel()[nonzero]
pl.figure()
pl.hexbin(dists, projected_dists, gridsize=100, cmap=pl.cm.PuBu)
pl.xlabel("Pairwise squared distances in original space")
pl.ylabel("Pairwise squared distances in projected space")
pl.title("Pairwise distances distribution for n_components=%d" %
n_components)
cb = pl.colorbar()
cb.set_label('Sample pairs counts')
rates = projected_dists / dists
print("Mean distances rate: %0.2f (%0.2f)" %
(np.mean(rates), np.std(rates)))
pl.figure()
pl.hist(rates, bins=50, normed=True, range=(0., 2.), edgecolor='k')
pl.xlabel("Squared distances rate: projected / original")
pl.ylabel("Distribution of samples pairs")
pl.title("Histogram of pairwise distance rates for n_components=%d" %
n_components)
pl.show()
def BD_DensityPlot_Pro(X,Y, numHexes=51, numXHistBins=51, numYHistBins=51, show_density=1, show_xcurrent=0, show_xhist=0, show_yhist=1, xhist_scale=0.3, yhist_scale=0.3, histogram_linecode='r-', bunch_charge=0, xoffset=None, yoffset=None, density_type='log', threshold_population=0, color_scale=pl.cm.Greys): #### Most of these variables adjust the arguments used for Matplotlib's hexbin function: ## http://matplotlib.org/api/pyplot_api.html#matplotlib.pyplot.hexbin #X, the X values of the data set. MANDATORY #Y, the Y values of the data set to be plotted. MANDATORY #numHexes=51 This is the number of bins on each side for which to perform the hexbinning. #numXHistBins=51 These values set the number of bins for the histogram. Default is the same as numHexes, but there are situations where a different scale is justified. #numYHistBins=51 #show_density=1 # This parameter dictates whether to show the density plot. By default, this is the only, as "bdplotter" is short for "beam density plotter". There are some cases, such as current profiles, where this should be turned off. #show_xcurrent=0 ### Shows the current projection on the X-axis. Scaling is TBD. This should not be used as an overlay to a density hexbin plot. #show_xhist=0 #show_yhist=0 ### These parameters determine whether a histogram projection is shown on the x and y axis, respectively. #xhist_scale=0.3 #yhist_scale=0.3 ##### These two parameters set the normalization of the histogram heights, as a fraction of the total heigth and width of the plot. #histogram_linecode='r-' #bunch_charge=0 #xoffset=0 #yoffset=0 #density_type='log'. Log plots the density as a function log_10(i+1). If you want a linear scale, you DO NOT enter 'linear'. Rather, use an integer that sets the number of gradients in color (divided from max to min). This can also be used a sequence of values that specify the range for each color. #threshold_population=0. This value is subtracted from the bin population. #color_scale=pl.cm.Greys. This uses standard matplotlib color scales. brg and jet are two of the more popular ones, but examples of all standard color maps can be found at: http://www.physics.ox.ac.uk/Users/msshin/science/code/matplotlib_cm/ # X and Y coordinates are mandatory. x=X-X.mean() y=Y-Y.mean() max_x=x.max() max_y=y.max() min_x=x.min() min_y=y.min() dx=max_x-min_x dy=max_y-min_y ### # CRP Note, 11-22-2013:Original code had an extra buffer, but that makes plots look poor when non-Greys color scales (or other color maps for which cm(pop=0) is not white. xinter=(max_x-min_x) xminplot=min_x#-.05*xinter xmaxplot=max_x#+.05*xinter yinter=(max_y-min_y) yminplot=min_y#-.05*yinter ymaxplot=max_y#+.05*yinter # Offset value for where to place the histogram projection. When no value is entered, it is zeroed at the edge. if xoffset==None: xoffset=xminplot # Offset value for where to place the histogram projection. When no value is entered, it is zeroed at the edge. if yoffset==None: yoffset=yminplot if show_density != 0: pl.hexbin(x,y, cmap=color_scale, bins=density_type,gridsize=numHexes, mincnt=threshold_population) pl.axis([xminplot, xmaxplot, yminplot, ymaxplot]) # Includes the projection of the horizontal distribution on the x axis as a temporal current, using the specified bunch charge. Do not use this is bunch_charge=0. if show_xcurrent != 0: currenthist,edgeshist = np.histogram(x,normed=0, bins=numXHistBins) numparts=x.size numbins=currenthist.size binlength=dx/numbins binpop=currenthist maxpop=currenthist.max() maxChargePop=maxpop*bunch_charge/numparts binCharge=currenthist*bunch_charge/numparts totalCharge=sum(binCharge) peakCurrent=2.998e8*maxChargePop/binlength TotalParts=currenthist.sum() normTest=currenthist.sum() binCurrent=binCharge*2.998e8/binlength binCurrentZeros=np.zeros(len(binCurrent)+2) binCurrentZeros[0]=0 binCurrentZeros[1:(len(binCurrent)+1)]=binCurrent binCurrentZeros[(len(binCurrent)+1)]=0 edgesZeros=np.zeros(len(edgeshist[1:])+2) edgesZeros[0]=min(edgeshist[1:])-np.std(edgeshist[1:])*.03 edgesZeros[1:(len(edgeshist[1:])+1)]=edgeshist[1:] edgesZeros[(len(edgeshist[1:])+1)]=max(edgeshist[1:])-np.std(edgeshist[1:])*.03 plt.step( edgesZeros,binCurrentZeros+yoffset,histogram_linecode) # Includes the projection of the horizontal distribution on the x axis. if show_xhist != 0: xhist,edgeshist = np.histogram(x,normed=0, bins=numXHistBins) numparts=x.size numbins=xhist.size binlength=dx/numbins*1.0 binpop=xhist*1.0 maxpop=xhist.max() binPopZeros=np.zeros(len(binpop)+2) binPopZeros[0]=0 binPopZeros[1:(len(binpop)+1)]=binpop binPopZeros[(len(binpop)+1)]=0 edgesZeros=np.zeros(len(edgeshist[1:])+2) edgesZeros[0]=min(edgeshist[1:])-np.std(edgeshist[1:]) edgesZeros[1:(len(edgeshist[1:])+1)]=edgeshist[1:] edgesZeros[(len(edgeshist[1:])+1)]=max(edgeshist[1:])-np.std(edgeshist[1:]) plt.step( edgesZeros,(binPopZeros/maxpop)*yinter*xhist_scale+yoffset,histogram_linecode) # Includes the projection of the vertical distribution on the y axis. if show_yhist != 0: yhist, yedgeshist = np.histogram(y,normed=0, bins=numYHistBins) numparts=y.size numbins=yhist.size binlength=dy/numbins*1.0 binpop=yhist*1.0 maxpop=yhist.max() binPopZeros=np.zeros(len(binpop)+2) binPopZeros[0]=0 binPopZeros[1:(len(binpop)+1)]=binpop binPopZeros[(len(binpop)+1)]=0 edgesZeros=np.zeros(len(yedgeshist[1:])+2) edgesZeros[0]=min(yedgeshist[1:])-np.std(yedgeshist[1:]) edgesZeros[1:(len(yedgeshist[1:])+1)]=yedgeshist[1:] edgesZeros[(len(yedgeshist[1:])+1)]=max(yedgeshist[1:])-np.std(yedgeshist[1:]) plt.step((binPopZeros/maxpop)*xinter*yhist_scale+xoffset,edgesZeros,histogram_linecode)
data = data[data['Price'] > 0.0]
figsize = (10, 7)
dpi = 100
pl.figure(figsize=figsize, dpi=dpi)
pl.semilogy(data['Price'], data['Approx Net Revenue'], 'bo', alpha=0.3, lw=0.2)
pl.xlabel("Price")
pl.ylabel("Estimated Net Revenue (log)")
mu.plot_out()
pl.figure(figsize=figsize, dpi=dpi)
pl.hexbin(data['Price'],
np.log(data['Approx Net Revenue']),
gridsize=32,
mincnt=1,
cmap=pl.cm.winter)
pl.colorbar()
pl.xlabel("Price")
pl.ylabel("Estimated Net Revenue (log)")
mu.plot_out()
pl.figure(figsize=figsize, dpi=dpi)
pl.loglog(data['Players'], data['Approx Net Revenue'], 'bo', alpha=0.3, lw=0.2)
pl.xlabel("Players (log)")
pl.ylabel("Estimated Net Revenue (log)")
mu.plot_out()
pl.figure(figsize=figsize, dpi=dpi)
pl.hexbin(data['DaysSinceRelease'],
def hex_difference(xy1,
xy2,
show_all=False,
hkwargs={},
color_bar=True,
fignum=1):
"""A function that plots the difference between two hexbin plots.
Parameters
----------
xy1 : A tuple of (x,y) corrdianates for the first hexbin. A tuple of (x,y,C)
can also be passed in where C is the value for each (x,y) point.
xy2 : A tuple of (x,y) corrdianates for the second hexbin. A tuple of (x,y,C)
can also be passed in where C is the value for each (x,y) point.
NOTE : the 'C' functinality is untested and may not work as expected.
Keywords
--------
show_all : bool (optional)
If True all intermediate hexbin plots are returned.
Default: show_all=False
color_bar : bool (optional)
If True a colorbar is placed on the plot(s)
Default: colorbar=True
fignum : int (optional)
The number to give the resulting figure(s). If
show_all=True, the intermediate plots will be
fignum+1 and fignum+2 while the difference will
be fignum.
default: fignum=1
Passed Keywords
---------------
hkwargs:
a dictionary of keywords passed to hexbin
(see http://matplotlib.org/api/pyplot_api.html#matplotlib.pyplot.hexbin
for additional keywords that can be set)
Returns
-------
d : pylab.hexbin object
Object returned by the difference hexbin
h1 : pylab.hexbin object
Object returned by hexbin of first data set
NOTE: only returned if show_all=True
h2 : pylab.hexbin object
Object returned by hexbin of second data set
NOTE: only returned if show_all=True
c : matplotlib.colorbar.Colorbar instance
NOTE: only returned if color_bar=True
Usage
-----
import numpy as np
n=100000
x1=np.random.standard_normal(n) #random x points
y1=2+3*x1+4*np.random.standard_normal(n) #random y points
x2=np.random.standard_normal(n) #random x points
y2=2-3*x2+4*np.random.standard_normal(n) #random y points
hex_difference((x1,y1),(x2,y2),show_all=True,color_bar=True,hkwargs={'gridsize':100,'extent':[-4.5,4.5,-25,25],'vmin':-180,'vmax':180})
pl.show()
"""
if show_all: #if shoing all hexbins then draw them as you go (you can't change the drawing axis object after creation)
pl.figure(fignum + 1)
hex1 = pl.hexbin(*xy1, **hkwargs)
if color_bar:
pl.colorbar()
pl.figure(fignum + 2)
hex2 = pl.hexbin(*xy2, **hkwargs)
if color_bar:
pl.colorbar()
else: #make but don't draw the 2 hexbins
hex1 = pl.hexbin(
*xy1, visible=False, **hkwargs
) #make the hexbins, visible it False to avoid drawing them to a plot
hex2 = pl.hexbin(*xy2, visible=False, **hkwargs)
pl.figure(fignum)
hex_dif = pl.hexbin(
*xy1, visible=False,
**hkwargs) #this will have the counts overwritten (so don't draw yet)
c1 = hex1.get_array() #the counts for hex1
c2 = hex2.get_array() #the counts for hex2
c_dif = c1 - c2 #difference between plots
gdx = ~(
(c1 == 0) &
(c2 == 0)) #the bins to draw (removes where both hists had no counts)
#NOTE: if the 'C' values are set checking against 0 is NOT a good idea...
hex_dif.set_array(c_dif[gdx]) #set the defferences into the hex_dif object
h = hex_dif.get_paths() #get the hexagon Path object(s)
if len(h
) > 1: #you have an old version of matplotlib, use this bit of code
rem_me = pl.array(h)[~gdx] #bins to remove
for r in rem_me:
h.remove(r) #remove blank bins
else: #either you have a boaring hexbin or a newer version of matplotlib
h = hex_dif.get_offsets()
hex_dif.set_offsets(h[gdx])
hex_dif.set_visible(True) #this draws the new hex_dif
ret = [hex_dif]
if show_all:
ret.append(hex1)
ret.append(hex2)
if color_bar:
ains = inset_axes(pl.gca(), width='80%', height='5%', loc=9)
#TODO: externalize colorbar keywords
c = pl.colorbar(hex_dif, cax=ains, orientation='horizontal')
ret.append(c)
return tuple(ret)
def opt_rates( astrom, dates, orbits ):
dt = (dates[1] - dates[0])*( 86400.0 )
dt_scale = 384.0 / dt
#print dt, dt_scale
sRA, sDEC = [],[]
#print datev[1:] - datev[:-1]
x,y,c = [],[],[]
x2,y2,c2 = [],[],[]
FILE = open('loc.dat', 'w')
FILE.write( '#$%s\n'%( dates[0] +2400000.5) )
for i in range(0, len(astrom)):
RA, DEC = float(astrom[i][0][0]), float(astrom[i][0][1])
#print RA, DEC
FILE.write('%s %s\n'%( RA, DEC ) )
color = float( orbits[i].split()[-1] )
x.append( RA * math.pi / 180.0 )
y.append( DEC * math.pi / 180.0 )
c.append( color )
FILE.close()
for i in range(0, len(astrom)):
RA, DEC = float(astrom[i][-1][0]), float(astrom[i][-1][1])
#print RA, DEC
color = float( orbits[i].split()[-1] )
x2.append( RA * math.pi / 180.0 )
y2.append( DEC * math.pi / 180.0 )
c2.append( color )
mx1 = numpy.median( x )
mx2 = numpy.median( x2 )
mex, mey = numpy.mean( x ), numpy.mean( y )
my1 = numpy.median( y )
my2 = numpy.median( y2 )
#print 'Offset between median and mean:'
#print (mx1- mex)*math.cos( mx1 ) * 180.0 * 3600.0 / math.pi
#print (my1- mey) *180.0 * 3600.0 / math.pi
#print '------------'
dx1 = numpy.asarray( x - mx1 )*math.cos( my1 )
dx2 = numpy.asarray( x2 - mx2 )*math.cos( my2 )
dy1 = numpy.asarray( y - my1 )
dy2 = numpy.asarray( y2 - my2 )
c = []
for i in range(0, len(astrom)):
#RA0, DEC0 = float(astrom[i][0][0]), float(astrom[i][0][1])
shiftRA, shiftDEC = [],[]
#for j in range(1, len(datev)):
# if abs( datev[j] - datev[j-1] ) > 2.0E-2:
# ### skip if longer than an orbit between
# continue
if 0 == 0:
RA, DEC = float(astrom[i][1][0]), float(astrom[i][1][1])
#print RA, DEC
RA0, DEC0 = float(astrom[i][0][0]), float(astrom[i][0][1])
#print RA0, DEC0, RA, DEC
dRA, dDEC = math.cos( DEC0 * math.pi / 180.0 ) * (RA - RA0), (DEC - DEC0) ### delta-degrees?
#shiftRA.append( dRA )
#shiftDEC.append( dDEC )
#print len(shiftRA), len(datev)
sRA.append( dRA )
sDEC.append( dDEC )
print datev[0] - datev[1]
sRA, sDEC = numpy.asarray( sRA )*3600.0, numpy.asarray( sDEC )*3600.0
print math.sqrt( (max(sRA) - min(sRA))**2 + (max(sDEC) - min(sDEC))**2 )
#dv = []
#for i in range(0, len(sRA)):
# deltaRA = ( sRA - sRA[i] )**2
# deltaDEC = ( sDEC - sDEC[i] )**2
# delta = numpy.percentile( numpy.sqrt( numpy.mean( deltaRA + deltaDEC, axis=1 )), 95 )
# dv.append( delta )
# c.append( delta )
epsilon = 0.04
dx = math.sqrt(3) * epsilon
dy = 1.5 * epsilon
xg1 = numpy.arange(min(sRA), max(sRA), dx )
xg2 = numpy.arange(min(sRA)+0.5*dx, max(sRA)+0.5*dx, dx )
yg1 = numpy.arange(min(sDEC), max(sDEC), 2.0*dy)
yg2 = numpy.arange(min(sDEC) + dy, max(sDEC) + dy, 2.0*dy)
X1, Y1 = numpy.meshgrid( xg1, yg1 )
X2, Y2 = numpy.meshgrid( xg2, yg2 )
X = numpy.append( X1.ravel(), X2.ravel() )
Y = numpy.append( Y1.ravel(), Y2.ravel() )
print len(X)
x0 = [0.01, 0.01, 0.1]
x1 = fmin( grid, x0, args=(sRA, sDEC,))
count, Xo, Yo = grid( x1, sRA, sDEC, epsilon=0.04, full=True )
pylab.ion()
fig = pylab.figure(figsize=(8,8))
ax = fig.add_subplot(111, aspect='equal')
pylab.hexbin( sRA, sDEC, bins='log', cmap=pylab.cm.binary,
gridsize=(100,100), extent=(numpy.median(sRA)-0.6, numpy.median(sRA)+0.6,
numpy.median(sDEC)-0.6, numpy.median(sDEC)+0.6) )
ax.scatter( Xo, Yo, color='r', s=2 )
for i in range(0, len(Xo)):
ax.add_artist( Circle( xy=(Xo[i],Yo[i]), radius=epsilon, edgecolor='r', facecolor='none') )
ax.set_xlabel(r'$\Delta\alpha$ (arcsec)')
ax.set_ylabel(r'$\Delta\delta$ (arcsec)')
ax.set_xlim(numpy.median(sRA)-0.6, numpy.median(sRA)+0.6)
ax.set_ylim(numpy.median(sDEC)-0.6, numpy.median(sDEC)+0.6)
#r = numpy.sqrt( ( (dx1-dx2)*180.0 *3600.0/ (math.pi))**2 + ( (dy1-dy2)*180.0 *3600.0/ (math.pi))**2 )
#r2 = numpy.sqrt( ( (dx1)*180.0 *3600.0/ (math.pi))**2 + ( (dy1)*180.0*3600.0/ (math.pi))**2 )
#pylab.scatter( r, r2, c=c )
#pylab.scatter( [r[ numpy.argmin( r ) ]], [r2[ numpy.argmin( r ) ]], s=100 )
pylab.draw()
pylab.draw()
#print numpy.argmin( r )
#print dx1[ numpy.argmin( r ) ]*180.0 *3600.0/ (math.pi), dy1[ numpy.argmin( r ) ]*180.0 *3600.0/ (math.pi)
pylab.savefig('HST_rategrid.png')
pause = raw_input('...')
#pylab.clf()
#pylab.hist( dt_scale * numpy.asarray( dv ) * 3600.0 / 0.04, bins=30 )
#pylab.draw()
#pylab.draw()
#v = numpy.asarray(orbits)[ numpy.argsort( dv ) ][0:50]
#d, sma = [],[]
#for k in v:
# k = k.strip().split()
# d.append( float( k[-1] ) )
# sma.append( float(k[0] ) )
#print min(d), max(d)
#print min(sma), max(sma)
#pause = raw_input('...')
return 0
get_ifar(timeslide_snr,bkg_snrs,timeslide_snr_ifar)
get_ifar(zerolag_snr,bkg_snrs,zerolag_snr_ifar)
# note, infs are not plotted!
pylab.figure(1)
pylab.loglog(injection_likelihood_ifar,injection_snr_ifar,'rx',label='injections')
pylab.hold(1)
pylab.loglog(timeslide_likelihood_ifar,timeslide_snr_ifar,'k.',label='timeslides')
pylab.hold(0)
pylab.xlabel('IFAR from MVSC Likelihood')
pylab.ylabel('IFAR from combined effective SNR')
pylab.savefig('IFAR_IFAR_scatterplot.png')
ax = pylab.figure(0).gca()
pylab.figure(2)
pylab.hexbin(injection_likelihood_ifar,injection_snr_ifar,gridsize=50,xscale='log',yscale='log',bins='log',cmap=pylab.cm.jet,alpha=1)
pylab.gca = ax
pylab.colorbar()
pylab.xlabel('IFAR from MVSC Likelihood')
pylab.ylabel('IFAR from combined effective SNR')
pylab.title('log10 density of injections')
pylab.savefig('IFAR_densityfig_injection.png')
pylab.figure(3)
pylab.hexbin(timeslide_likelihood_ifar,timeslide_snr_ifar,gridsize=50,xscale='log',yscale='log',bins='log',cmap=pylab.cm.jet,alpha=1)
pylab.gca = ax
pylab.colorbar()
pylab.xlabel('IFAR from MVSC Likelihood')
pylab.ylabel('IFAR from combined effective SNR')
pylab.title('log10 density of timeslides')
pylab.savefig('IFAR_densityfig_timeslide.png')
def plotDeltaMagMap(matchedCats,
icmag='MAG_AUTO',
coaddmag='MAG_AUTO-new',
coaddRA='ALPHA_J2000',
coaddDelta='DELTA_J2000'):
xpos = []
ypos = []
deltamags = []
coaddmags = []
icmags = []
for iccat, coaddcat, icheader, icPos, icWCS in matchedCats:
goodmatch = np.logical_and(
np.logical_and(
np.sqrt((icWCS[:, 0] - coaddcat[coaddRA])**2 +
(icWCS[:, 1] - coaddcat[coaddDelta])**2) < 4.e-05,
np.logical_and(iccat['FLAGS'] == 0, coaddcat['Flag'] == 0)),
np.logical_and(iccat['FLUX_MAX'] < 25000,
iccat['FLUX_AUTO'] / iccat['FLUXERR_AUTO'] > 5))
# np.logical_and(np.logical_and(coaddcat['FLUX_RADIUS'] < 5,
coaddmags.append((coaddcat[coaddmag])[goodmatch])
icmags.append((iccat[icmag])[goodmatch])
deltaMag = iccat[icmag] - coaddcat[coaddmag]
xpos.append(icPos[:, 0][goodmatch])
ypos.append(icPos[:, 1][goodmatch])
deltamags.append(deltaMag[goodmatch])
xpos = np.hstack(xpos)
ypos = np.hstack(ypos)
coaddmags = np.hstack(coaddmags)
icmags = np.hstack(icmags)
deltamags = np.hstack(deltamags)
goodmatch = np.ones_like(coaddmags) == 1.
# goodmatch = np.logical_and(coaddmags > -10, coaddmags < -7)
# goodmatch = np.logical_and(coaddmags > -5, coaddmags < 0)
# goodmatch = coaddmags < -9
residuals = deltamags - np.median(deltamags)
print np.min(residuals[goodmatch]), np.max(residuals[goodmatch])
fig = pylab.figure()
# temp_resids = residuals[goodmatch].copy()
# temp_resids[temp_resids < -0.05] = -0.051
# temp_resids[temp_resids > 0.05] = 0.051
pylab.scatter(xpos[goodmatch], ypos[goodmatch], c=residuals[goodmatch])
pylab.colorbar()
fig2 = pylab.figure()
ax = fig2.add_axes([0.12, 0.12, 0.95 - 0.12, 0.95 - 0.12])
ax.hist(residuals, bins=130)
fig3 = pylab.figure()
dist = np.sqrt((xpos)**2 + (ypos)**2)
pylab.hexbin(np.abs(ypos[goodmatch]), residuals[goodmatch], gridsize=50)
pylab.axhline(0, c='k', linewidth=1.5)
pylab.xlabel('Ypos')
fig4 = pylab.figure()
pylab.hexbin(np.abs(xpos[goodmatch]), residuals[goodmatch], gridsize=50)
pylab.axhline(0, c='k', linewidth=1.5)
pylab.xlabel('Xpos')
fig5 = pylab.figure()
pylab.hexbin(coaddmags[coaddmags < 30],
((icmags - coaddmags) / coaddmags)[coaddmags < 30],
gridsize=(80, 280),
bins='log')
return (fig, fig2, fig3, fig4, fig5), (xpos, ypos, residuals)
cpu.plot_density(stardat['x'],stardat['y'],z=stardat['color'],z_method=n.std,radecgrid=True)
pyl.title('$\sigma(g-i)$')
if savefig:
pyl.savefig('Starcolorstd.png', format='png')
pyl.figure()
ack1,ack2,ack3 = pyl.hist(stardat['color'], bins=100)
pyl.xlabel('$(g-i)$')
pyl.ylabel('Number of stars')
pyl.title('Color Distribution of Calibration Stars')
pyl.savefig('Starcolorhist.png', format='png')
visits=cp.read_visitfile()
#histogram up the visits, set visitlim to [0,90th%ile]
#do a pre-bin to crop range and 90th %tile
x, y = cpu.hammer_project_toxy(visits['ra']*deg2rad,\
visits['dec']*deg2rad)
gridsize = cpu.calc_gridsize(visits,binsize='fov', rad_fov=1.8)
ack=pyl.hexbin(x,y,gridsize=gridsize, mincnt=0.5)
counts=ack.get_array()
hist, bins=n.histogram(counts, n.arange(max(counts)+1))
nc=n.cumsum(hist, dtype=float)/n.sum(hist)
visit_max=bins[n.max(n.where(nc < 0.99))]
pyl.figure()
cp.plot_visits(visits, visitlim=[1,visit_max])
if savefig:
pyl.savefig('Visitdensity.png',format='png')
def plotDistortionCor(matchedCats,
fluxcor,
icmag='MAG_AUTO',
coaddmag='MAG_AUTO-new',
coaddRA='ALPHA_J2000',
coaddDelta='DELTA_J2000'):
xpos = []
ypos = []
coaddmags = []
deltamags = []
accumulatedfilter = []
for iccat, coaddcat, icheader, icPos, icWCS in matchedCats:
goodmatch = np.logical_and(
np.logical_and(
np.sqrt((icWCS[:, 0] - coaddcat[coaddRA])**2 +
(icWCS[:, 1] - coaddcat[coaddDelta])**2) < 4.e-05,
np.logical_and(iccat['FLAGS'] == 0, coaddcat['Flag'] == 0)),
np.logical_and(iccat['FLUX_MAX'] < 25000,
iccat['FLUX_AUTO'] / iccat['FLUXERR_AUTO'] > 5))
# np.logical_and(np.logical_and(coaddcat['FLUX_RADIUS'] < 5,
deltaMag = iccat[icmag] - coaddcat[coaddmag]
coaddmags.append(coaddcat[coaddmag][goodmatch])
xpos.append(icPos[:, 0][goodmatch])
ypos.append(icPos[:, 1][goodmatch])
deltamags.append(deltaMag[goodmatch])
accumulatedfilter.append(goodmatch)
xpos = np.hstack(xpos)
ypos = np.hstack(ypos)
coaddmags = np.hstack(coaddmags)
deltamags = np.hstack(deltamags)
deltamags = deltamags - np.median(deltamags)
accumulatedfilter = np.hstack(accumulatedfilter)
print len(accumulatedfilter), len(fluxcor), len(
fluxcor[accumulatedfilter]), len(deltamags)
correction = -2.5 * np.log10(fluxcor)[accumulatedfilter]
magcorresid = deltamags - correction
goodmatch = np.logical_and((fluxcor != -9999)[accumulatedfilter],
coaddmags < 30)
# goodmatch = (fluxcor != -9999.)[accumulatedfilter]
fig = pylab.figure()
pylab.scatter(xpos[goodmatch], ypos[goodmatch], c=magcorresid[goodmatch])
pylab.colorbar()
fig2 = pylab.figure()
pylab.hexbin(np.abs(xpos[goodmatch]),
magcorresid[goodmatch],
gridsize=50,
extent=[0, 4000, -0.1, 0.1])
pylab.axhline(0, c='k', linewidth=1.5)
pylab.xlabel('Xpos')
fig3 = pylab.figure()
pylab.hexbin(np.abs(ypos[goodmatch]),
magcorresid[goodmatch],
gridsize=50,
extent=[0, 4000, -0.1, 0.1])
pylab.axhline(0, c='k', linewidth=1.5)
pylab.xlabel('Ypos')
fig4 = pylab.figure()
pylab.hist(magcorresid, bins=200)
fig5 = pylab.figure()
pylab.scatter(xpos[goodmatch], ypos[goodmatch], c=correction[goodmatch])
pylab.colorbar()
return (fig, fig2, fig3, fig4), (xpos, ypos, magcorresid)
def get_lin_sub_mat(mat_a, mat_b, plotfile=None, minperc=98.0):
# apply a linear regression to subtract predictions based on mat_b from mat_a
# we apply a constrained linear regression which must pass through the origin, acting only on points with mat_b> minval, and
# then use a slope that keeps 95% of those high-ChIP samples below the line
# minperc is the minimum percentile of the chip data (mat_b) to use
minval = scipy.stats.scoreatpercentile(mat_b, minperc)
print "Minval is %f" % minval
out_mat = numpy.zeros_like(mat_a)
if plotfile is not None:
pylab.figure()
pylab.rcParams.update({'font.size': 22})
pylab.hexbin(mat_b, mat_a, bins='log', cmap=pylab.get_cmap("Blues"))
pylab.xlabel('Log$_2$ ChIP vs. input')
pylab.ylabel('Log$_2$ IPOD vs. input')
pylab.savefig(plotfile + "_noline.png")
#pylab.plot(col_b[numpy.argsort(col_b)], pred_for_plot, 'r-', linewidth=2.5)
for col_i in range(mat_a.shape[1]):
col_a = mat_a[:, col_i]
col_b = mat_b[:, col_i]
fitslope = do_lin_fit(col_b, col_a, minval)
print "Using slope of %f" % fitslope
predvals = do_lin_interpolation_array(col_b, fitslope, 0)
predvals = numpy.fmax(predvals, 0.0)
newvals = col_a - predvals
print col_b.shape
print predvals.shape
print "***"
if (plotfile is not None) and (col_i == 0):
pylab.plot(col_b[numpy.argsort(col_b)],
predvals[numpy.argsort(col_b)],
'r--',
linewidth=3)
pylab.savefig(plotfile + "_main.png")
pylab.figure()
pylab.hexbin(col_a,
newvals,
bins='log',
cmap=pylab.get_cmap("Blues"))
pylab.colorbar()
pylab.xlabel('Log$_2$ IPOD vs. input')
pylab.ylabel('ChIP-subtracted IPOD-HR')
pylab.savefig(plotfile + "_sub.png")
pylab.figure()
pylab.hexbin(col_b,
newvals,
bins='log',
cmap=pylab.get_cmap("Blues"))
pylab.colorbar()
pylab.xlabel('Log$_2$ ChIP vs. input')
pylab.ylabel('ChIP-subtracted IPOD-HR')
pylab.savefig(plotfile + "_chipsub_lin.png")
#if len(newvals.shape) == 1:
# newvals = newvals.reshape(-1,1)
out_mat[:, col_i] = newvals
return out_mat
def main():
min_count = 1
parser = argparse.ArgumentParser()
# d is for directory
parser.add_argument('-d',action='store',dest = 'd',type = str, required = False, default = './')
# number of PC sections we want to use (going back from the last)
parser.add_argument('-n',action='store',dest = 'n',type = int, required = False)
# project onto which axis?
parser.add_argument('-p',action='store',dest = 'p',type = str, required = False, default = 'x')
# grid size
parser.add_argument('-g',action='store',dest = 'g',type = int, required = False, default = 300)
# This is going to be a unique option to collect AAAALLLLLL the data of several individual
# bifurcation runs (everything the same except random initial conditions) into one so that the
# diagram does not look so messy. The argument being passed is a directory --> no default
# option.
parser.add_argument('--together',action='store',dest='together',type = bool, required = False, default=False)
inargs = parser.parse_args()
d = inargs.d
projection = inargs.p
grid_size = inargs.g
together = inargs.together
num_pc = inargs.n
print('type num_pc: ' + str(type(num_pc)))
os.chdir(d)
# ancl --> anal class
ancl = of.anal_run()
ancl.together = together
ancl.get_info()
ancl.set_list_dir()
# get the system info
# note: if Dim = 1 y_num_cell -> ' No y ' and order -> 'polygamma'
print('Dim is: ' +str(ancl.Dim))
print('x_num_cell: ' + str(ancl.x_num_cell))
# depending on the projection type we will determine "ax_num" variable. This number is an easy
# way to controle whicn axis we are projecting onto by the way we slce the data. form is this:
# data[time,ax_num*N:(ax_num + 1)*N]
# for ax_num = 0 ---> vx data
# for ax_num = 1 ---> vy data
# for ax_num = 2 ---> x data
# for ax_num = 3 ---> y data
# lets also define the string for the axis lable depending on the projection
if projection == 'x':
ax_str = r'$x$'
slice_num = ancl.Dim
if projection == 'y':
if ancl.Dim == 1:
print('No y projectino in 1D')
quit()
slice_num = ancl.Dim+1
ax_str = r'$y$'
if projection == 'vx':
ax_str = r'$\dot{x}$'
slice_num = 0
if projection == 'vy':
if Dim == 1:
print('No vy projectino in 1D')
quit()
slice_num = ancl.Dim-1
ax_str = r'$\dot{y}$'
build = pl.zeros(ancl.N)+1.0
# this array keeps track of the variable values for each particle
var_arr = pl.array([])
all_data = pl.array([])
# the data to be ploted. projection type will determine which data gets stored in this array
to_plot = pl.array([])
for i,j in enumerate(ancl.list_dir):
print('cur_file should be: ' + str(j))
cur_file = open(j,"r")
# get the varible for the plot
var = float(cur_file.readline().split()[-1])
data = np.genfromtxt(cur_file)
print('len(data): '+str(len(data)))
print('shape(data): ' + str(pl.shape(data)))
# modulus data by system length
data[:,ancl.Dim*ancl.N:(ancl.Dim+1)*ancl.N] = data[:,ancl.Dim*ancl.N:(ancl.Dim+1)*ancl.N]%(ancl.x_num_cell*2.0*pl.pi)
if ancl.Dim ==2:
print('in Dim==2 section')
data[:,(ancl.Dim+1)*ancl.N:(ancl.Dim+2)*ancl.N] = data[:,(ancl.Dim+1)*ancl.N:(ancl.Dim+2)*ancl.N]%(ancl.y_num_cell*2.0*pl.pi)
#print('shape(data) after % x/y_num_cell: ' + str(pl.shape(data)))
# if we want more poincare section then count back from the last checking to see if we are
# at the t%2pl.pi=0 point and add data accordingly
if num_pc != None:
# indexing variable
counting = 0
# keep track of how many poincare sections we have saved
passed_pc = 0
while passed_pc <= num_pc:
if (counting*ancl.dt)%(2.0*pl.pi)<=ancl.dt:
print('passed_pc: '+str(passed_pc))
to_plot = pl.append(to_plot,data[-counting,slice_num*ancl.N:(slice_num+1)*ancl.N])
print('making to_plot')
#print(to_plot)
var_arr = pl.append(var_arr,build*var)
passed_pc += 1
counting +=1
else:
print('in else')
#print('to be added to to_plot: '+ str(data[-1,slice_num*N:(slice_num+1)*N]))
to_plot = pl.append(to_plot,data[-1,slice_num*ancl.N:(slice_num+1)*ancl.N])
print('making to_plot')
#print(to_plot)
#all_data = pl.append(all_data,data)
var_arr = pl.append(var_arr,build*var)
cur_file.close()
print('len(to_plot): '+str(len(to_plot)))
print(to_plot)
print('len(var_arr): ' + str(len(var_arr)))
print(var_arr)
pl.hexbin(var_arr,to_plot,gridsize=grid_size,bins='log',mincnt=min_count,edgecolor='none')
#cmap=pl.cm.YlOrRd_r
#pl.hexbin(A_arr,x,gridsize=400,cmap=pl.cm.Greys,bins='log',mincnt=2,edgecolor='none')
#pl.hexbin(A_arr,x,bins='log')
pl.colorbar()
pl.xlabel(ancl.sweep_str,fontsize=30)
pl.ylabel(ax_str,fontsize=30)
#pl.tight_layout()
# No more over writing images
number = 0
save_str = 'paper_'+projection+'_bif_hist'+str(number)+'.png'
while save_str in os.listdir('.'):
number +=1
save_str = 'paper_'+projection+'_bif_hist'+str(number)+'.png'
pl.savefig(save_str,dpi=200)
os.system('open paper_'+projection+'_bif_hist'+str(number)+'.png')
subprocess.call('say Finished the bifurcation histogram',shell = True)
pylab.xlim(x_min - 1, x_max + 1)
pylab.ylim(y_min - 0.01, y_max + .01)
pylab.xlabel('$q$')
pylab.ylabel('$\chi_1$')
pylab.grid()
#pylab.savefig("smallContour"+options.out_file)
#pylab.savefig("Contour"+options.out_file,dpi=1000)
pylab.savefig("ContourmatchesQChi1.pdf", dpi=1000)
pylab.savefig("ContourmatchesQChi1.png", dpi=1000)
# plot the mass-mass plane
pylab.figure(pltid)
pltid += 1
pylab.axes([0.125, 0.125, 0.95 - 0.125, 0.95 - 0.2])
pylab.hexbin(q, chi1, C=matches, bins=50)
cb = pylab.colorbar()
cb.set_label("FF")
pylab.hold(True)
pylab.plot(tmpltq, tmpltch, 'kx', markersize=3.)
pylab.xlim(x_min - 1, x_max + 1)
pylab.ylim(y_min - 0.01, y_max + .01)
pylab.xlabel('$q$')
pylab.ylabel('$\chi_1$')
pylab.grid()
#pylab.savefig("smallContour"+options.out_file)
#pylab.savefig("Contour"+options.out_file,dpi=1000)
pylab.savefig("HistmatchesQChi1.pdf", dpi=1000)
pylab.savefig("HistmatchesQChi1.png", dpi=1000)
for metric in [
'abs_correlation', 'braycurtis', 'canberra', 'correlation', 'cosine',
'minkowski', 'seuclidean'
]:
for n_neighbors in [4, 5, 6]:
pl.figure()
results = isomap(D, n_neighbors, metric=metric)
x, y = results.real[:, 0], results.real[:, 1]
pl.scatter(x, y, c=np.arange(X.shape[1])[SW], cmap=pl.jet())
pl.title("%s n_neighbors %d" % (metric, n_neighbors))
#pl.savefig("%s.n_neighbors.%d.png" % (metric, n_neighbors))
pl.show()
exit()
print "Saving restored frames..."
name = "movie"
if not os.path.exists(name): os.mkdir(name)
for i in range(len(X)):
pl.figure(figsize=(2, 2))
pl.hexbin(x, y, gridsize=50, C=X[i], cmap=pl.gray())
pl.savefig("%s/frame.%04d.png" % (name, i))
pl.close()
print ".",
print "Done."
print "Creating movie..."
os.system("ffmpeg -y -f image2 -i %s/frame.%%04d.png -r 12 %s.mov" %
(name, name))
