def calc_grid_sum_xyvs(xyvs, bins=100):
"""Convert an XYVs object into a gridded and summed XYVs object.
xyvs a list of tuples (x, y, ...)
bins the number of required bins, either <int> or [<int>, <int>]
Returns a tuple ((bin_x, bin_y), xyz_data) where xyz_data is a binned
version of the input object. bin_? is bin count for the ? axis.
"""
# convert to numpy array
xyvs = scipy.array(xyvs)
# figure out number of value columns
num_columns = len(xyvs[0]) - 2
if num_columns < 1:
msg = 'calc_grid_sum_xyvs: xyvs object has no data columns'
raise RuntimeError(msg)
# get histogram with value1 column
xyv1 = scipy.take(xyvs, (0, 1, 2), axis=1)
(result, xedges, yedges) = scipy.histogram2d(xyv1[:, 0],
xyv1[:, 1],
bins=bins,
normed=False,
weights=xyv1[:, 2])
# create XYZ object
result = make_xyz(result, xedges, yedges)
# get number of bins for result
# -1 since ?edges is numer of *edges*, we want bins
bins_x = scipy.shape(xedges)[0] - 1
bins_y = scipy.shape(yedges)[0] - 1
# handle each extra value column seperately
for i in range(num_columns - 1):
xyvn = scipy.take(xyvs, (0, 1, i + 3), axis=1)
(binned_data, xedges, yedges) = scipy.histogram2d(xyvn[:, 0],
xyvn[:, 1],
bins=bins,
normed=False,
weights=xyvn[:, 2])
binned_data = make_xyz(binned_data, xedges, yedges)
v_col = scipy.take(binned_data, (2, ), axis=1)
result = scipy.hstack((result, v_col))
return ((bins_x, bins_y), result)
def better_2d_density_plot(x_data, y_data, threshold=3, bins=(100, 100)):
"""
Takes x and y coordinates of the data points and creates a 2D density plot. Everything below
threshold is represented as individual points, bins tuple is the number of bins along each axis.
Essentially is a wrapper around scipy's histogram2d. Does not show, so a pyplot.show() is
required to show the result of plotting
:param x_data: x coordinates of the data points
:param y_data: y coordinates of the data points
:param threshold: if there are less than that number of points in a bin, points are
represented individually
:param bins: number of bins along each axis
"""
xy_range = [[min(x_data), max(x_data)], [min(y_data), max(y_data)]]
distortion = (xy_range[1][1] - xy_range[1][0]) / \
(xy_range[0][1] - xy_range[0][0])
x_data = x_data * distortion
xy_range = [[min(x_data), max(x_data)], [min(y_data), max(y_data)]]
hh, loc_x, loc_y = histogram2d(x_data, y_data, range=xy_range, bins=bins)
pos_x = np.digitize(x_data, loc_x)
pos_y = np.digitize(y_data, loc_y)
ind = (pos_x > 0) & (pos_x <= bins[0]) & (pos_y > 0) & (pos_y <= bins[1])
# values of the histogram where the points are
hh_sub = hh[pos_x[ind] - 1, pos_y[ind] - 1]
x_dat1 = x_data[ind][hh_sub < threshold] # low density points
y_dat1 = y_data[ind][hh_sub < threshold]
hh[hh < threshold] = np.nan # fill the areas with low density by NaNs
plt.imshow(np.flipud(hh.T), cmap='jet',
extent=np.array(xy_range).flatten(), interpolation='none')
plt.plot(x_dat1, y_dat1, '.')
def window_func2d(x,
y,
z,
func,
xmin=None,
xmax=None,
ymin=None,
ymax=None,
nbins=[10, 10]):
"""This function does compute a user-supplied function
on the subsets of y grouped by values of x
E.g. imagine that you have x from 0 to 100 and you want
to know the mean values of y for 0<x<10, 10<x<2- etc..
In that case you need to do
xbin,funcy,nperbin=window_func(x,y,lambda x: x.mean(),0,100,nbin=10)
where xbin is the array with the centers of bins,
funcy is the func -- evaluated for y where x is within the appropriate bin
and nperbin is the number of points in the appropriate bin
empty keyword is needed if you want to retrieve the function evaluation
in empty bins too, otherwise they aren't returned at all
"""
if xmin is None:
xmin = x.min()
if xmax is None:
xmax = x.max()
if ymin is None:
ymin = y.min()
if ymax is None:
ymax = y.max()
hh, locx, locy = scipy.histogram2d(x,
y,
range=((xmin, xmax), (ymin, ymax)),
bins=nbins)
xinds = numpy.digitize(x, locx) - 1
yinds = numpy.digitize(y, locy) - 1
mask = numpy.zeros(hh.shape, bool)
retv = hh * 0.
subind = (xinds >= 0) & (yinds >= 0) & (xinds < nbins[0]) & (yinds <
nbins[1])
xinds, yinds, z1 = [_[subind] for _ in [xinds, yinds, z]]
valind = yinds * (nbins[0]) + xinds
sortind = np.argsort(valind)
valind = valind[sortind]
poss = np.where(np.diff(valind) > 0)[0]
z1 = z1[sortind]
for i in range(len(poss) + 1):
if i == 0:
left = 0
right = poss[0] + 1
elif i == len(poss):
left = poss[-1] + 1
right = len(valind)
else:
left = poss[i - 1] + 1
right = poss[i] + 1
curval = valind[left]
retv[curval % (nbins[0]), curval / (nbins[0])] = func(z1[left:right])
return retv, locx[0], locx[-1], locy[0], locy[-1], hh
def rendHist(x, y, imageBounds, pixelSize):
X = numpy.arange(imageBounds.x0, imageBounds.x1, pixelSize)
Y = numpy.arange(imageBounds.y0, imageBounds.y1, pixelSize)
im, edx, edy = scipy.histogram2d(x, y, bins=(X, Y))
return im
def density_scatter(ax, x, y, nbins=(50,50), binsize=None,
threshold=5, c='b', s=1, cmap='jet'):
'''
Draw a combination density map / scatterplot to the given axes.
Adapted from answer to stackoverflow question #10439961
'''
# Make histogram of data
bounds = [[np.min(x)-1.e-10, np.max(x)+1.e-10],
[np.min(y)-1.e-10, np.max(y)+1.e-10]]
if binsize != None:
nbins = []
if len(binsize) != 2:
raise Exception('binsize must have size 2. Size is %d.' % len(binsize))
for i in range(2):
nbins.append((bounds[i][1] - bounds[i][0])/float(binsize[i]))
h, loc_x, loc_y = scipy.histogram2d(x, y, range=bounds, bins=nbins)
pos_x, pos_y = np.digitize(x, loc_x), np.digitize(y, loc_y)
# Mask histogram points below threshold
idx = (h[pos_x - 1, pos_y - 1] < threshold)
h[h < threshold] = np.nan
# Density plot
img = ax.imshow(np.log(h.T), origin='lower', cmap=cmap,
extent=np.array(bounds).flatten(),
interpolation='nearest', aspect='auto')
# Scatterplot
ax.scatter(x[idx], y[idx], c=c, s=s, edgecolors='none')
return img
def rendHist(x,y, imageBounds, pixelSize):
X = numpy.arange(imageBounds.x0,imageBounds.x1, pixelSize)
Y = numpy.arange(imageBounds.y0,imageBounds.y1, pixelSize)
im, edx, edy = scipy.histogram2d(x,y, bins=(X,Y))
return im
def make2dhist(fill=True, bins=30, ncolors=64):
print "************"
for i, name in zip(range(len(p)), p):
print i, name
print
i = int(raw_input("parX?"))
if i < 0: return False
j = int(raw_input("parY?"))
if j < 0: return False
if fill:
plotcont = pylab.contourf
else:
plotcont = pylab.contour
pylab.clf()
if i == j:
pylab.hist(M[i, :], bins=bins**2)
else:
h2, xb, yb = S.histogram2d(M[i, :], M[j, :], bins=bins)
xb = 0.5 * (xb[:-1] + xb[1:])
yb = 0.5 * (yb[:-1] + yb[1:])
plotcont(xb, yb, h2, ncolors)
pylab.colorbar()
# calculate mode:
imax = h2.argmax()
ii, jj = imax % h2.shape[0], imax / h2.shape[0]
print "mode:", xb[ii], yb[jj]
pylab.show()
return True
def cont(x,y,xlim=None,ylim=None,levels=[0.95,0.683],alpha=0.7,color='blue',nbins=256,nsmooth=4,Fill=True,**kwargs):
levels.sort()
levels.reverse()
cols=getcols(color)
dx=np.max(x)-np.min(x)
dy=np.max(y)-np.min(y)
if xlim is None: xlim=[np.min(x)-dx/3,np.max(x)+dx/3]
if ylim is None: ylim=[np.min(y)-dy/3,np.max(y)+dy/3]
range=[xlim,ylim]
a,xmap,ymap=scipy.histogram2d(x,y,bins=256,range=range)
a=np.transpose(a)
xmap=xmap[:-1]
ymap=ymap[:-1]
dx=xmap[1]-xmap[0]
dy=ymap[1]-ymap[0]
z=scipy.ndimage.filters.gaussian_filter(a,nsmooth)
z=z/np.sum(z)/dx/dy
sz=np.sort(z.flatten())[::-1]
cumsz=integrate.cumtrapz(sz)
cumsz=cumsz/max(cumsz)
f=interpolate.interp1d(cumsz,np.arange(np.size(cumsz)))
indices=f(levels).astype('int')
vals=sz[indices].tolist()
vals.append(np.max(sz))
vals.sort()
if Fill:
for i in np.arange(np.size(levels)):
contourf(xmap, ymap, z, vals[i:i+2],colors=cols[i],alpha=alpha,**kwargs)
else:
contour(xmap, ymap, z, vals[0:1],colors=cols[1],**kwargs)
contour(xmap, ymap, z, vals[1:2],colors=cols[1],**kwargs)
a=Rectangle((np.max(xmap),np.max(ymap)),0.1,0.1,fc=cols[1])
return(a)
def better2D_desisty_plot(xdat, ydat, thresh=3, bins=(100, 100)):
xyrange = [[min(xdat), max(xdat)], [min(ydat), max(ydat)]]
distortion = (xyrange[1][1] - xyrange[1][0]) / \
(xyrange[0][1] - xyrange[0][0])
xdat = xdat * distortion
xyrange = [[min(xdat), max(xdat)], [min(ydat), max(ydat)]]
hh, locx, locy = histogram2d(xdat, ydat, range=xyrange, bins=bins)
posx = np.digitize(xdat, locx)
posy = np.digitize(ydat, locy)
ind = (posx > 0) & (posx <= bins[0]) & (posy > 0) & (posy <= bins[1])
# values of the histogram where the points are
hhsub = hh[posx[ind] - 1, posy[ind] - 1]
xdat1 = xdat[ind][hhsub < thresh] # low density points
ydat1 = ydat[ind][hhsub < thresh]
hh[hh < thresh] = np.nan # fill the areas with low density by NaNs
plt.imshow(
np.flipud(
hh.T),
cmap='jet',
extent=np.array(xyrange).flatten(),
interpolation='none')
plt.plot(xdat1, ydat1, '.')
def __plot_scatter_singular(prl, max_points=None, fileout="PR_scatter.png", title="Precision Recall Scatter Plot"):
"""
:param prl: list of tuples (precision, recall)
:param max_points: max number of tuples to plot
:param fileout: output filename
:param title: plot title
"""
prs = [i[0] for i in prl]
recs = [i[1] for i in prl]
if max_points is not None:
prs = prs[:max_points]
recs = recs[:max_points]
# histogram definition
xyrange = [[0, 1],[0, 1]] # data range
bins = [50, 50] # number of bins
thresh = 3 # density threshold
xdat, ydat = np.array(prs), np.array(recs)
# histogram the data
hh, locx, locy = scipy.histogram2d(xdat, ydat, range=xyrange, bins=bins)
# select points within the histogram
hh[hh < thresh] = np.nan # fill the areas with low density by NaNs
plt.imshow(np.flipud(hh.T), cmap='jet',
extent=np.array(xyrange).flatten(), interpolation='none', origin='upper', alpha=0.8)
plt.colorbar()
plt.title(title)
plt.ylabel("Recall", fontsize=20, labelpad=15)
plt.xlabel("Precision", fontsize=20)
plt.savefig(fileout)
def hist2d(ax, xdat, ydat, xyrange, bins, thresh=2, cmap=plt.cm.Greys, log=False, scatterother=False):
import scipy
tt = ax.get_aspect()
# histogram the data
hh, locx, locy = scipy.histogram2d(xdat, ydat, range=xyrange, bins=bins)
mhh = np.mean(hh)
shh = np.std(hh)
if log:
lhh = np.log10(hh)
else:
lhh = hh
posx = np.digitize(xdat, locx)
posy = np.digitize(ydat, locy)
#select points within the histogram
ind = (posx > 0) & (posx <= bins[0]) & (posy > 0) & (posy <= bins[1])
hhsub = hh[posx[ind] - 1, posy[ind] - 1] # values of the histogram where the points are
xdat1 = xdat[ind][hhsub < thresh] # low density points
ydat1 = ydat[ind][hhsub < thresh]
lhh[hh < thresh] = np.nan # fill the areas with low density by NaNs
ar = (0.6/0.65)*(np.diff(xyrange[0])/np.diff(xyrange[1]))[0]
c = ax.imshow(np.flipud(lhh.T),extent=np.array(xyrange).flatten(), interpolation='none', cmap=cmap, aspect=ar)
ax.set_aspect(tt)
if scatterother:
ax.plot(xdat1, ydat1, 'k,')
return c
def dens_hist(data):
#data definition
xdat, ydat = data[0],data[1]
x_range=np.array([-2,2])+np.mean(xdat)
y_range=np.array([-2,2])+np.mean(ydat)
xyrange = [x_range,y_range] # data range
bins = [100,100] # number of bins
thresh = 1 #density threshold
# histogram the data
hh, locx, locy = scipy.histogram2d(xdat, ydat, range=xyrange, bins=bins)
posx = np.digitize(xdat, locx)
posy = np.digitize(ydat, locy)
#select points within the histogram
ind = (posx > 0) & (posx <= bins[0]) & (posy > 0) & (posy <= bins[1])
hhsub = hh[posx[ind] - 1, posy[ind] - 1] # values of the histogram where the points are
xdat1 = xdat[ind][hhsub < thresh] # low density points
ydat1 = ydat[ind][hhsub < thresh]
hh[hh < thresh] = np.nan # fill the areas with low density by NaNs
plt.imshow(np.flipud(hh.T),cmap='jet',extent=np.array(xyrange).flatten(), interpolation='none', origin='upper')
plt.title("xmean i {},xstd is {} and ymean is{}, ystd is {}".format(np.mean(xdat),np.std(xdat),np.mean(ydat),np.std(ydat)))
plt.colorbar()
plt.show()
def better2D_desisty_plot(xdat, ydat, thresh=3, bins=(100, 100)):
xyrange = [[min(xdat), max(xdat)], [min(ydat), max(ydat)]]
distortion = (xyrange[1][1] - xyrange[1][0]) / \
(xyrange[0][1] - xyrange[0][0])
xdat = xdat * distortion
xyrange = [[min(xdat), max(xdat)], [min(ydat), max(ydat)]]
hh, locx, locy = histogram2d(xdat, ydat, range=xyrange, bins=bins)
posx = np.digitize(xdat, locx)
posy = np.digitize(ydat, locy)
ind = (posx > 0) & (posx <= bins[0]) & (posy > 0) & (posy <= bins[1])
# values of the histogram where the points are
hhsub = hh[posx[ind] - 1, posy[ind] - 1]
xdat1 = xdat[ind][hhsub < thresh] # low density points
ydat1 = ydat[ind][hhsub < thresh]
hh[hh < thresh] = np.nan # fill the areas with low density by NaNs
plt.imshow(
np.flipud(
hh.T),
cmap='jet',
extent=np.array(xyrange).flatten(),
interpolation='none')
plt.plot(xdat1, ydat1, '.')
def make2dhist(fill=True, bins=30, ncolors=64):
print "************"
for i, name in zip(range(len(p)), p): print i, name
print
i = int(raw_input("parX?"))
if i < 0: return False
j = int(raw_input("parY?"))
if j < 0: return False
if fill:
plotcont = pylab.contourf
else:
plotcont = pylab.contour
pylab.clf()
if i == j:
pylab.hist(M[i, :], bins=bins ** 2)
else:
h2, xb, yb = S.histogram2d(M[i, :], M[j, :], bins=bins)
xb = 0.5 * (xb[:-1] + xb[1:])
yb = 0.5 * (yb[:-1] + yb[1:])
plotcont(xb, yb, h2, ncolors)
pylab.colorbar()
# calculate mode:
imax = h2.argmax()
ii, jj = imax % h2.shape[0], imax / h2.shape[0]
print "mode:", xb[ii], yb[jj]
pylab.show()
return True
def better_2d_density_plot(x_data, y_data, threshold=3, bins=(100, 100)):
"""
Takes x and y coordinates of the data points and creates a 2D density plot. Everything below
threshold is represented as individual points, bins tuple is the number of bins along each axis.
Essentially is a wrapper around scipy's histogram2d. Does not show, so a pyplot.show() is
required to show the result of plotting
:param x_data: x coordinates of the data points
:param y_data: y coordinates of the data points
:param threshold: if there are less than that number of points in a bin, points are
represented individually
:param bins: number of bins along each axis
"""
xy_range = [[min(x_data), max(x_data)], [min(y_data), max(y_data)]]
distortion = (xy_range[1][1] - xy_range[1][0]) / \
(xy_range[0][1] - xy_range[0][0])
x_data = x_data * distortion
xy_range = [[min(x_data), max(x_data)], [min(y_data), max(y_data)]]
hh, loc_x, loc_y = histogram2d(x_data, y_data, range=xy_range, bins=bins)
pos_x = np.digitize(x_data, loc_x)
pos_y = np.digitize(y_data, loc_y)
ind = (pos_x > 0) & (pos_x <= bins[0]) & (pos_y > 0) & (pos_y <= bins[1])
# values of the histogram where the points are
hh_sub = hh[pos_x[ind] - 1, pos_y[ind] - 1]
x_dat1 = x_data[ind][hh_sub < threshold] # low density points
y_dat1 = y_data[ind][hh_sub < threshold]
hh[hh < threshold] = np.nan # fill the areas with low density by NaNs
plt.imshow(np.flipud(hh.T),
cmap='jet',
extent=np.array(xy_range).flatten(),
interpolation='none')
plt.plot(x_dat1, y_dat1, '.')
def histogram(self, x, y, **kwargs):
data, _, _ = scipy.histogram2d(x,
y,
bins=[self.nx, self.ny],
range=[[self.xmin, self.xmax],
[self.ymin, self.ymax]],
**kwargs)
return data
def visualize2SVM(maxT, sumM, name, nozero=False, k=2.2e35, useall=False, lims=[2.2e3,9e3], minval=[2,5], conc='c_w_l',loc='svmcrossvalid.p',idx2=None):
""" log plot of the data with only PEC data"""
val, c_w_l, idx = conditionVal(name=name, nozero=nozero, conc=conc)
data = scipy.io.readsav('W_Abundances_grid_puestu_adpak_fitscaling_74_0.00000_5.00000_1000_idlsave')
y = time.time()
xax = scipy.logspace(2,5,201)
yax = scipy.logspace(-5,1,101)
xtemp = xax[1:]/2.+xax[:-1]/2.
ytemp = yax[1:]/2.+yax[:-1]/2.
Y,X = scipy.meshgrid(xtemp, ytemp)
if idx2 is None:
histdata, xed, yed = scipy.histogram2d(maxT,val/c_w_l/sumM*k,bins=[xax,yax])
else:
histdata, xed, yed = scipy.histogram2d(maxT[idx2],(val/c_w_l/sumM*k)[idx2],bins=[xax,yax])
extent = [xed[0], xed[-1], yed[0], yed[-1]]
plt.pcolormesh(xax,yax,histdata.T, cmap='viridis', rasterized=True,vmin=1.,norm=LogNorm())
plt.gca().set_yscale('log')
plt.gca().set_xscale('log')
plt.gca().set_ylim(1e-4,1e1)
plt.gca().set_xlim(1e3,2e4)
cmap = plt.cm.get_cmap('viridis')
cmap.set_under('white')
idx3 = scipy.logical_and(data['en'] > lims[0], data['en'] < lims[1])
data2 = pickle.load(open(loc,'rb'))
fz = data2[1][minval[0]][minval[1]]
print(len(fz))
print(data['en'][idx3].shape)
plt.loglog(data['en'][idx3],fz/fz.max(),lw=3.5,color='darkorange',linestyle='-',label='SVM fit')
plt.xlabel(r'$T_e$ [eV]')
plt.ylabel(r'const$\cdot I_{CSXR} / c_W \sum M$')
colorbar = plt.colorbar()
colorbar.ax.set_ylabel('datapoint density (a.u.)')
leg = plt.legend(loc=4,fontsize=20,title=r'\underline{\hspace{1em} C=10$^4$, $\gamma=.1$ \hspace{1em}}')
plt.setp(leg.get_title(),fontsize=14)
plt.subplots_adjust(bottom=.12,right=1.)
def calc_grid_sum_xyvs(xyvs, bins=100):
"""Convert an XYVs object into a gridded and summed XYVs object.
xyvs a list of tuples (x, y, ...)
bins the number of required bins, either <int> or [<int>, <int>]
Returns a tuple ((bin_x, bin_y), xyz_data) where xyz_data is a binned
version of the input object. bin_? is bin count for the ? axis.
"""
# convert to numpy array
xyvs = scipy.array(xyvs)
# figure out number of value columns
num_columns = len(xyvs[0]) - 2
if num_columns < 1:
msg = 'calc_grid_sum_xyvs: xyvs object has no data columns'
raise RuntimeError(msg)
# get histogram with value1 column
xyv1 = scipy.take(xyvs, (0,1,2), axis=1)
(result, xedges, yedges) = scipy.histogram2d(xyv1[:,0], xyv1[:,1],
bins=bins, normed=False,
weights=xyv1[:,2])
# create XYZ object
result = make_xyz(result, xedges, yedges)
# get number of bins for result
# -1 since ?edges is numer of *edges*, we want bins
bins_x = scipy.shape(xedges)[0] - 1
bins_y = scipy.shape(yedges)[0] - 1
# handle each extra value column seperately
for i in range(num_columns-1):
xyvn = scipy.take(xyvs, (0,1,i+3), axis=1)
(binned_data, xedges, yedges) = scipy.histogram2d(xyvn[:,0], xyvn[:,1],
bins=bins, normed=False,
weights=xyvn[:,2])
binned_data = make_xyz(binned_data, xedges, yedges)
v_col = scipy.take(binned_data, (2,), axis=1)
result = scipy.hstack((result, v_col))
return ((bins_x, bins_y), result)
def mutual_information(x, y, bins=10):
'''
Computes the mutual information between x and y. Assumes x and y are unbinned data vectors.
Only works if x and y are the same size. This is a *very* naive estimator that is obtained
by simply directly computing a 2d histogram for x and y.
'''
pxy = histogram2d(x.flatten(), y.flatten(), bins)[0]
# fake the density=True flag that scipy.histogram has
pxy = pxy / pxy.sum()
return entropy(x, bins) + entropy(y, bins) - binned_entropy(pxy)
def xy_density(xdat,
ydat,
cmap='jet',
marker='.',
imshow_kwargs={},
bins=[100, 100],
density_thresh=0,
xyrange=None,
plot_kwargs={}):
'''
graphs the density of (x,y) points in the plane, using color (defined by cmap) except when density is below
density_thresh, in which case the points themselves are ploted
'''
# validation
assert len(xdat) == len(ydat), 'xdat and ydat must have the same length'
#histogram definition
xyrange = xyrange or [[np.min(xdat), np.max(xdat)],
[np.min(ydat), np.max(ydat)]] # data range
if isinstance(bins, int):
bins = [bins, bins]
# else:
# if isinstance(bins[0], int):
# bins[0] = range(bins[0])
# if isinstance(bins[1], int):
# bins[1] = range(bins[1])
if density_thresh < 1: # if density_thresh is a float, interpret it as a ratio of the number of bins
density_thresh = density_thresh * bins[0] * bins[1]
# histogram the data
hh, locx, locy = scipy.histogram2d(xdat, ydat, range=xyrange, bins=bins)
# return hh, locx, locy
posx = np.digitize(xdat, locx)
posy = np.digitize(ydat, locy)
#select points within the histogram
# ind = (posx > 0) & (posx <= np.max(bins[0])) & (posy > 0) & (posy <= np.max(bins[1]))
ind = (posx > 0) & (posx <= bins[0]) & (posy > 0) & (posy <= bins[1])
hhsub = hh[posx[ind] - 1,
posy[ind] - 1] # values of the histogram where the points are
xdat1 = xdat[ind][hhsub < density_thresh] # low density points
ydat1 = ydat[ind][hhsub < density_thresh]
# hh[hh < density_thresh] = np.nan # fill the areas with low density by NaNs
plt.imshow(np.flipud(hh.T),
cmap=cmap,
interpolation='none',
extent=np.array(xyrange).flatten(),
**imshow_kwargs)
plt.colorbar()
plot_kwargs = dict({'color': plt.cm.get_cmap(cmap)(0)}, **plot_kwargs)
plt.plot(xdat1, ydat1, marker, **plot_kwargs)
plt.show()
return hh
def plot_3d_population(self,figname=None):
"""
Makes density contour plots of all the cloud population.
.. note::
Set `figname` to the path of your file where to save the plot, otherwise it outputs to screen.
"""
from matplotlib import gridspec,colors
x0 = self.cartesian_galactocentric.x.value
x1 = self.cartesian_galactocentric.y.value
x2 = self.cartesian_galactocentric.z.value
planes={'x-y':[x0,x1],'x-z':[x0,x2],'y-z':[x1,x2]}
c=1
fig=plt.figure(figsize=(15,15))
gs = gridspec.GridSpec(3, 1 )#width_ratios=[1.5, 2,1.5],height_ratios=[1.5,2,1.5])
for a in list(planes.keys()):
x,y=planes[a]
a1,a2=a.split("-",2)
xyrange=[[min(x),max(x)],[min(y),max(y)]]
nybins,nxbins=50,50
bins=[nybins,nxbins]
thresh=2#density threshold
hh, locx, locy = histogram2d(x, y, range=xyrange, bins=[nybins,nxbins])
posx = np.digitize(x, locx)
posy = np.digitize(y, locy)
ind = (posx > 0) & (posx <= bins[0]) & (posy > 0) & (posy <= bins[1])
hhsub = hh[posx[ind] - 1, posy[ind] - 1] # values of the histogram where the points are
xdat1 = x[ind][hhsub < thresh] # low density points
ydat1 = y[ind][hhsub < thresh]
hh[hh < thresh] = np.nan # fill the areas with low density by NaNs
ax=plt.subplot(gs[c-1])
ax.set_xlabel(a1+' [kpc]',fontsize=20)
ax.set_ylabel(a2+' [kpc]',fontsize=20)
if a2=='z' and self.model!='Spherical':
im=ax.imshow(np.flipud(hh.T),cmap='jet',vmin=0, vmax=hhsub.max()/2, extent=[-15,15, -1,1],interpolation='gaussian', origin='upper')
ax.set_yticks((-.5,0,.5))
else:
im=ax.imshow(np.flipud(hh.T),cmap='jet',vmin=0, vmax=hhsub.max()/2, extent=np.array(xyrange).flatten(),interpolation='gaussian', origin='upper')
ax.plot(xdat1, ydat1, '.',color='darkblue')
c+=1
fig.subplots_adjust(right=0.8)
cbar_ax = fig.add_axes([0.85, 0.15, 0.05, 0.7])
fig.colorbar(im,cax=cbar_ax)
if figname is None:
plt.show()
else:
plt.savefig(figname)
plt.close()
def cont(x,y,xlim=None,ylim=None,levels=[0.9545,0.6827],alpha=0.7,color='blue',
nbins=256,
nsmooth=4,Fill=True,plotCorrCoef=True,plotScatter=False,**kwargs):
levels.sort()
levels.reverse()
cols=getcols(color)
dx=np.max(x)-np.min(x)
dy=np.max(y)-np.min(y)
if xlim is None: xlim=[np.min(x)-dx/3,np.max(x)+dx/3]
if ylim is None: ylim=[np.min(y)-dy/3,np.max(y)+dy/3]
range=[xlim,ylim]
a,xmap,ymap=scipy.histogram2d(x,y,bins=256,range=range)
a=np.transpose(a)
xmap=xmap[:-1]
ymap=ymap[:-1]
dx=xmap[1]-xmap[0]
dy=ymap[1]-ymap[0]
z=scipy.ndimage.filters.gaussian_filter(a,nsmooth)
z=z/np.sum(z)/dx/dy
sz=np.sort(z.flatten())[::-1]
cumsz=integrate.cumtrapz(sz)
cumsz=cumsz/max(cumsz)
f=interpolate.interp1d(cumsz,np.arange(np.size(cumsz)))
indices=f(levels).astype('int')
vals=sz[indices].tolist()
vals.append(np.max(sz))
vals.sort()
if Fill:
for i in np.arange(np.size(levels)):
contourf(xmap, ymap, z, vals[i:i+2],colors=cols[i],alpha=alpha,**kwargs)
else:
contour(xmap, ymap, z, vals[0:1],colors=cols[1],**kwargs)
contour(xmap, ymap, z, vals[1:2],colors=cols[1],**kwargs)
a=Rectangle((np.max(xmap),np.max(ymap)),0.1,0.1,fc=cols[1])
if plotScatter: scatter(x,y,color=color,marker=u'.')
if plotCorrCoef:
mmx,ssx = x.mean(),x.std()
mmy,ssy = y.mean(),y.std()
rcorrcoeff = np.corrcoef(x,y)[0,1]
#if abs(rcorrcoeff)>0.65:
label_cont='$\\rho$=%0.2f'%(rcorrcoeff)
xarr = array([mmx-ssx,mmx+ssx])
yarr = array([mmy-ssy,mmy+ssy])
plot(xarr,xarr*0.0+mmy,color,label=label_cont)
plot(xarr*0.0+mmx,yarr,color)
legend(loc=2,frameon=False,numpoints=1,fontsize=12) #8 15 20
return(a)
def window_func2d(x,y,z,func,xmin=None,xmax=None,ymin=None,ymax=None,nbins=[10,10]): """This function does compute a user-supplied function on the subsets of y grouped by values of x E.g. imagine that you have x from 0 to 100 and you want to know the mean values of y for 0<x<10, 10<x<2- etc.. In that case you need to do xbin,funcy,nperbin=window_func(x,y,lambda x: x.mean(),0,100,nbin=10) where xbin is the array with the centers of bins, funcy is the func -- evaluated for y where x is within the appropriate bin and nperbin is the number of points in the appropriate bin empty keyword is needed if you want to retrieve the function evaluation in empty bins too, otherwise they aren't returned at all """ if xmin is None: xmin = x.min() if xmax is None: xmax = x.max() if ymin is None: ymin = y.min() if ymax is None: ymax = y.max() hh, locx, locy = scipy.histogram2d(x, y, range=((xmin,xmax), (ymin,ymax)), bins=nbins) xinds = numpy.digitize(x, locx) - 1 yinds = numpy.digitize(y, locy) - 1 mask = numpy.zeros(hh.shape, bool) retv = hh * 0. subind = (xinds >= 0) & (yinds >= 0) & (xinds < nbins[0]) & (yinds < nbins[1]) xinds, yinds, z1 = [_[subind] for _ in [xinds,yinds,z]] valind = yinds * (nbins[0]) + xinds sortind = np.argsort(valind) valind = valind[sortind] poss = np.where(np.diff(valind) > 0)[0] z1 = z1[sortind] for i in range(len(poss) + 1): if i == 0: left = 0 right = poss[0] + 1 elif i == len(poss): left = poss[-1] + 1 right = len(valind) else: left = poss[i-1] + 1 right = poss[i] + 1 curval = valind[left] retv[curval % (nbins[0]), curval / (nbins[0])] = func(z1[left:right]) return retv, locx[0], locx[-1], locy[0], locy[-1], hh
def dscat(xdat, ydat):
# Todo: erstellen ScatterHistogramplot
import matplotlib.pyplot as plt, numpy as np, numpy.random, scipy
#data definition
N = 1000
#xdat, ydat = np.random.normal(size=N), np.random.normal(size=N)
#xdat, ydat = np.sin(np.arange(1, N)),np.cos(np.arange(1, N)+10)
#xdat, ydat = np.array([1,2,3,4,4,4,4,4,4,4]), np.array([1,2,3,4,4,4,4,4,4,4])
#histogram definition
xyrange = [[np.nanmin(xdat),np.nanmax(xdat)],[np.nanmin(ydat),np.nanmax(ydat)]] # data range
bins = [100,100] # number of bins
thresh = 0.1 #density threshold
#linear regression
from scipy import stats
mask = ~np.isnan(xdat) & ~np.isnan(xdat)
slope, intercept, r_value, p_value, std_err = stats.linregress(xdat[mask], ydat[mask])
line = slope * xdat + intercept
from scipy import stats
#xdat = xdat.reshape(xdat.shape[0]*xdat.shape[1],)
#ydat = ydat.reshape(ydat.shape[0]*ydat.shape[1],)
# histogram the data
hh, locx, locy = scipy.histogram2d(xdat, ydat, range=xyrange, bins=bins)
posx = np.digitize(xdat, locx)
posy = np.digitize(ydat, locy)
#select points within the histogram
ind = (posx > 0) & (posx <= bins[0]) & (posy > 0) & (posy <= bins[1])
hhsub = hh[posx[ind] - 1, posy[ind] - 1] # values of the histogram where the points are
xdat1 = xdat[ind][hhsub < thresh] # low density points
ydat1 = ydat[ind][hhsub < thresh]
hh[hh < thresh] = np.nan # fill the areas with low density by NaNs
plt.imshow(np.flipud(hh.T),cmap='jet',extent=np.array(xyrange).flatten(), interpolation='none', origin='upper')
plt.colorbar()
plt.plot(xdat1, ydat1, '.',color='darkblue')
plt.plot(xdat, line, 'r', label = 'Korrelation \n' + str(round(r_value, 3))
+ r'$\pm$' + str(round(std_err, 3)))
print xdat, xdat1
plt.legend(loc='upper left', ncol=1, fancybox=True, shadow=True,
fontsize='20')
plt.xlim(xyrange[0])
plt.ylim(xyrange[1])
plt.grid()
def hist2d(ax,
xdat,
ydat,
xyrange,
bins,
thresh=2,
cmap=plt.cm.Greys,
log=False,
scatterother=False):
import scipy
tt = ax.get_aspect()
# histogram the data
hh, locx, locy = scipy.histogram2d(xdat, ydat, range=xyrange, bins=bins)
mhh = np.mean(hh)
shh = np.std(hh)
if log:
lhh = np.log10(hh)
else:
lhh = hh
posx = np.digitize(xdat, locx)
posy = np.digitize(ydat, locy)
#select points within the histogram
ind = (posx > 0) & (posx <= bins[0]) & (posy > 0) & (posy <= bins[1])
hhsub = hh[posx[ind] - 1,
posy[ind] - 1] # values of the histogram where the points are
xdat1 = xdat[ind][hhsub < thresh] # low density points
ydat1 = ydat[ind][hhsub < thresh]
lhh[hh < thresh] = np.nan # fill the areas with low density by NaNs
ar = (0.6 / 0.65) * (np.diff(xyrange[0]) / np.diff(xyrange[1]))[0]
c = ax.imshow(np.flipud(lhh.T),
extent=np.array(xyrange).flatten(),
interpolation='none',
cmap=cmap,
aspect=ar)
ax.set_aspect(tt)
if scatterother:
ax.plot(xdat1, ydat1, 'k,')
return c
def make2dhist(testpath, xaxis=TE, yaxis=TI, figmplf=None, curax=None):
"""
This will plot a 2-D histogram of two variables.
Args:
testpath (obj:`str`): The path where the SimISR data is stored.
npulses (obj:`int`): The number of pulses.
xaxis (obj: `dict`): default TE, Dictionary that holds the parameter info along the x axis of the distribution.
yaxis (obj: `dict`): default TE, Dictionary that holds the parameter info along the y axis of the distribution.
figmplf (obj: `matplotb figure`): default None, Figure that the plot will be placed on.
curax (obj: `matplotlib axis`): default None, Axis that the plot will be made on.
Returns:
figmplf (obj: `matplotb figure`), curax (obj: `matplotlib axis`):,hist_h (obj: `matplotlib axis`)):
The figure handle the plot is made on, the axis handle the plot is on, the plot handle itself.
"""
sns.set_style("whitegrid")
sns.set_context("notebook")
params = [xaxis['param'], yaxis['param']]
datadict, _, _, _ = makehistdata(params, testpath)
if (figmplf is None) and (curax is None):
(figmplf, curax) = plt.subplots(1, 1, figsize=(6, 6), facecolor='w')
b1 = sp.linspace(*xaxis['lims'])
b2 = sp.linspace(*yaxis['lims'])
bins = [b1, b2]
d1 = sp.column_stack((datadict[params[0]], datadict[params[1]]))
H, xe, ye = sp.histogram2d(d1[:, 0].real,
d1[:, 1].real,
bins=bins,
normed=True)
hist_h = curax.pcolor(xe[:-1],
ye[:-1],
sp.transpose(H),
cmap='viridis',
vmin=0)
curax.set_xlabel(r'$' + xaxis['paramLT'] + '$')
curax.set_ylabel(r'$' + yaxis['paramLT'] + '$')
curax.set_title(r'Joint distributions for $' + xaxis['paramLT'] + '$' +
' and $' + yaxis['paramLT'] + '$')
plt.colorbar(hist_h, ax=curax, label='Probability', format='%1.1e')
return (figmplf, curax, hist_h)
def doInvTCHist(xvals, yvals, xbins, ybins, sat=1):
h = sp.histogram2d(xvals,yvals,[xbins,ybins])[0]
lh = log10(h + 1).T
#print lh.shape
X,Y = sp.meshgrid(xbins[:-1], ybins[:-1])
c = 1 - cm.RdYlGn(sp.minimum(sp.maximum(X/(X + Y), 0),1))
#print c.shape
sc = sp.minimum(sat*lh/lh.max(), 1)
r = c[:,:,:3]
r[:,:,0] = r[:,:,0]*sc
r[:,:,1] = r[:,:,1]*sc
r[:,:,2] = r[:,:,2]*sc
return 1-r
def doInvTCHist(xvals, yvals, xbins, ybins, sat=1):
h = sp.histogram2d(xvals, yvals, [xbins, ybins])[0]
lh = log10(h + 1).T
#print lh.shape
X, Y = sp.meshgrid(xbins[:-1], ybins[:-1])
c = 1 - cm.RdYlGn(sp.minimum(sp.maximum(X / (X + Y), 0), 1))
#print c.shape
sc = sp.minimum(sat * lh / lh.max(), 1)
r = c[:, :, :3]
r[:, :, 0] = r[:, :, 0] * sc
r[:, :, 1] = r[:, :, 1] * sc
r[:, :, 2] = r[:, :, 2] * sc
return 1 - r
def xy_density(xdat, ydat, cmap='jet', marker='.', imshow_kwargs={},
bins=[100, 100], density_thresh=0, xyrange=None, plot_kwargs={}):
'''
graphs the density of (x,y) points in the plane, using color (defined by cmap) except when density is below
density_thresh, in which case the points themselves are ploted
'''
# validation
assert len(xdat) == len(ydat), 'xdat and ydat must have the same length'
#histogram definition
xyrange = xyrange or [[np.min(xdat), np.max(xdat)], [np.min(ydat), np.max(ydat)]] # data range
if isinstance(bins, int):
bins = [bins, bins]
# else:
# if isinstance(bins[0], int):
# bins[0] = range(bins[0])
# if isinstance(bins[1], int):
# bins[1] = range(bins[1])
if density_thresh < 1: # if density_thresh is a float, interpret it as a ratio of the number of bins
density_thresh = density_thresh * bins[0] * bins[1]
# histogram the data
hh, locx, locy = scipy.histogram2d(xdat, ydat, range=xyrange, bins=bins)
# return hh, locx, locy
posx = np.digitize(xdat, locx)
posy = np.digitize(ydat, locy)
#select points within the histogram
# ind = (posx > 0) & (posx <= np.max(bins[0])) & (posy > 0) & (posy <= np.max(bins[1]))
ind = (posx > 0) & (posx <= bins[0]) & (posy > 0) & (posy <= bins[1])
hhsub = hh[posx[ind] - 1, posy[ind] - 1] # values of the histogram where the points are
xdat1 = xdat[ind][hhsub < density_thresh] # low density points
ydat1 = ydat[ind][hhsub < density_thresh]
# hh[hh < density_thresh] = np.nan # fill the areas with low density by NaNs
plt.imshow(np.flipud(hh.T), cmap=cmap, interpolation='none', extent=np.array(xyrange).flatten(), **imshow_kwargs)
plt.colorbar()
plot_kwargs = dict({'color': plt.cm.get_cmap(cmap)(0)}, **plot_kwargs)
plt.plot(xdat1, ydat1, marker, **plot_kwargs)
plt.show()
return hh
def showpos(self,pos=0):
"this function shows the position results of the raytracing"
if pos < self.surface_pos[-1]:
self.interpol_data(pos=pos)
plot_data = self.dummy_surface
else:
print("Please choose a position smaller than the end position of the simulation")
data_list = [x[0][:].T[0:2] for x in plot_data]
f,axarr = plt.subplots(2,2,figsize=(10,10))
for i,ax in enumerate(flatten(axarr)):
ax.clear()
data=data_list[i]
xdat, ydat = data[0],data[1]
x_range=np.array([-2,2])+np.mean(xdat)
y_range=np.array([-2,2])+np.mean(ydat)
xyrange = [x_range,y_range] # data range
bins = [100,100] # number of bins
thresh = 10 #density threshold
# histogram the data
hh, locx, locy = scipy.histogram2d(xdat, ydat, range=xyrange, bins=bins)
posx = np.digitize(xdat, locx)
posy = np.digitize(ydat, locy)
#select points within the histogram
ind = (posx > 0) & (posx <= bins[0]) & (posy > 0) & (posy <= bins[1])
hhsub = hh[posx[ind] - 1, posy[ind] - 1] # values of the histogram where the points are
xdat1 = xdat[ind][hhsub < thresh] # low density points
ydat1 = ydat[ind][hhsub < thresh]
hh[hh < thresh] = np.nan # fill the areas with low density by NaNs
ax.imshow(np.flipud(hh.T),cmap='jet',extent=np.array(xyrange).flatten(), interpolation='none', origin='upper')
ax.set_title("xm ={:.2f}, xs ={:.2f}, ym = {:.2f}, ys ={:.2f}".format(np.mean(xdat),np.std(xdat),np.mean(ydat),np.std(ydat)))
# ax.set_fontsize(20)
# ax.colorbar()
plt.show()
def Mclast():
fig = plt.figure(1, figsize=(6,9))
gs = gridspec.GridSpec(2,1,height_ratios=[4,1])
ax = plt.subplot(gs[0])
mvir, mclast, mratio = ioformat.rcol(fwind, [1,5,14], linestart=1)
print("Mvir Range: ", min(mvir), max(mvir))
x, y = [], []
for i in range(len(mclast)):
if(mclast[i] > mclast_cut and mratio[i] < 20.0):
x.append(mvir[i])
y.append(mclast[i])
xbins = linspace(11.0, 13.5, 30)
ybins = linspace(0.05, 1.0, 40)
xgrid, ygrid = meshgrid(xbins, ybins)
z, edx, edy = histogram2d(x, y, bins=[xbins,ybins])
z = z.T + 0.01
zf = ndimage.gaussian_filter(z, sigma=1.0, order=0)
cont = ax.contour(xbins[1:], ybins[1:], zf, colors="red")
#plt.pcolor(xgrid, ygrid, z, cmap="Purples", norm=LogNorm(vmin=z.min(), vmax=z.max()))
ax.pcolor(xgrid, ygrid, z, cmap="Purples")
setp(ax.get_xticklabels(), visible=False)
ax.set_ylabel("Mc (Rejoin)")
plt.title(modelname+", Z~1.0")
ax = plt.subplot(gs[1])
plt.subplots_adjust(hspace=0.0, top=0.9, bottom=0.15)
hist1, bins = histogram(mvir, bins=linspace(11.0,13.5,30))
hist2, bins = histogram(x, bins=linspace(11.0,13.5,30))
hist = []
for i in range(len(hist1)):
if(hist1[i] > 0): hist.append(float(hist2[i])/float(hist1[i]))
else: hist.append(0.0)
#width=0.8*(bins[1]-bins[0])
center = (bins[:-1] + bins[1:]) / 2.0
ax.plot(center, hist, "b.-")
ax.set_ylim(0.0, 1.0)
ax.set_xlabel("Mvir")
ax.set_ylabel("f_rej")
plt.show()
def make2dhist(testpath, xaxis=TE, yaxis=TI, figmplf=None, curax=None):
"""
This will plot a 2-D histogram of two variables.
Args:
testpath (obj:`str`): The path where the SimISR data is stored.
npulses (obj:`int`): The number of pulses.
xaxis (obj: `dict`): default TE, Dictionary that holds the parameter info along the x axis of the distribution.
yaxis (obj: `dict`): default TE, Dictionary that holds the parameter info along the y axis of the distribution.
figmplf (obj: `matplotb figure`): default None, Figure that the plot will be placed on.
curax (obj: `matplotlib axis`): default None, Axis that the plot will be made on.
Returns:
figmplf (obj: `matplotb figure`), curax (obj: `matplotlib axis`):,hist_h (obj: `matplotlib axis`)):
The figure handle the plot is made on, the axis handle the plot is on, the plot handle itself.
"""
sns.set_style("whitegrid")
sns.set_context("notebook")
params = [xaxis['param'], yaxis['param']]
datadict, _, _, _ = makehistdata(params, testpath)
if (figmplf is None) and (curax is None):
(figmplf, curax) = plt.subplots(1, 1, figsize=(6, 6), facecolor='w')
b1 = sp.linspace(*xaxis['lims'])
b2 = sp.linspace(*yaxis['lims'])
bins = [b1, b2]
d1 = sp.column_stack((datadict[params[0]],datadict[params[1]]))
H, xe, ye = sp.histogram2d(d1[:,0].real, d1[:,1].real, bins=bins, normed=True)
hist_h = curax.pcolor(xe[:-1], ye[:-1], sp.transpose(H), cmap='viridis', vmin=0)
curax.set_xlabel(r'$'+xaxis['paramLT']+'$')
curax.set_ylabel(r'$'+yaxis['paramLT']+'$')
curax.set_title(r'Joint distributions for $'+ xaxis['paramLT']+'$'+' and $'+
yaxis['paramLT']+'$')
plt.colorbar(hist_h, ax=curax, label='Probability', format='%1.1e')
return (figmplf, curax, hist_h)
def __plot_scatter_singular(self, max_points=None, fileout=None, title="Precision Recall Scatter Plot"):
"""
:param max_points: max number of tuples to plot
:param fileout: output filename
:param title: plot title
"""
prs = [i[0] for i in self.prl]
recs = [i[1] for i in self.prl]
if max_points is not None:
prs = prs[:max_points]
recs = recs[:max_points]
# histogram definition
xyrange = [[0, 1],[0, 1]] # data range
bins = [50, 50] # number of bins
thresh = 1 # density threshold
xdat, ydat = np.array(prs), np.array(recs)
# histogram the data
hh, locx, locy = scipy.histogram2d(xdat, ydat, range=xyrange, bins=bins)
# select points within the histogram
hh[hh < thresh] = np.nan # fill the areas with low density by NaNs
plt.imshow(np.flipud(hh.T), cmap='jet',
extent=np.array(xyrange).flatten(), interpolation='none', origin='upper', alpha=0.8)
plt.colorbar()
plt.title(title)
plt.ylabel("Recall", fontsize=20, labelpad=15)
plt.xlabel("Precision", fontsize=20)
if fileout is not None:
plt.savefig("%s" % fileout)
else:
return plt.show()
def plot_3d_population(self, figname=None):
"""
Makes density contour plots of all the cloud population.
.. note::
Set `figname` to the path of your file where to save the plot, otherwise it outputs to screen.
"""
from matplotlib import gridspec, colors
x0 = self.cartesian_galactocentric.x.value
x1 = self.cartesian_galactocentric.y.value
x2 = self.cartesian_galactocentric.z.value
planes = {'x-y': [x0, x1], 'x-z': [x0, x2], 'y-z': [x1, x2]}
c = 1
fig = plt.figure(figsize=(15, 15))
gs = gridspec.GridSpec(
3, 1) #width_ratios=[1.5, 2,1.5],height_ratios=[1.5,2,1.5])
for a in list(planes.keys()):
x, y = planes[a]
a1, a2 = a.split("-", 2)
xyrange = [[min(x), max(x)], [min(y), max(y)]]
nybins, nxbins = 50, 50
bins = [nybins, nxbins]
thresh = 2 #density threshold
hh, locx, locy = histogram2d(x,
y,
range=xyrange,
bins=[nybins, nxbins])
posx = np.digitize(x, locx)
posy = np.digitize(y, locy)
ind = (posx > 0) & (posx <= bins[0]) & (posy > 0) & (posy <=
bins[1])
hhsub = hh[posx[ind] - 1, posy[ind] -
1] # values of the histogram where the points are
xdat1 = x[ind][hhsub < thresh] # low density points
ydat1 = y[ind][hhsub < thresh]
hh[hh < thresh] = np.nan # fill the areas with low density by NaNs
ax = plt.subplot(gs[c - 1])
ax.set_xlabel(a1 + ' [kpc]', fontsize=20)
ax.set_ylabel(a2 + ' [kpc]', fontsize=20)
if a2 == 'z' and self.model != 'Spherical':
im = ax.imshow(np.flipud(hh.T),
cmap='jet',
vmin=0,
vmax=hhsub.max() / 2,
extent=[-15, 15, -1, 1],
interpolation='gaussian',
origin='upper')
ax.set_yticks((-.5, 0, .5))
else:
im = ax.imshow(np.flipud(hh.T),
cmap='jet',
vmin=0,
vmax=hhsub.max() / 2,
extent=np.array(xyrange).flatten(),
interpolation='gaussian',
origin='upper')
ax.plot(xdat1, ydat1, '.', color='darkblue')
c += 1
fig.subplots_adjust(right=0.8)
cbar_ax = fig.add_axes([0.85, 0.15, 0.05, 0.7])
fig.colorbar(im, cax=cbar_ax)
if figname is None:
plt.show()
else:
plt.savefig(figname)
plt.close()
things['Rs'] = []
things['Q_star2s'] = []
things['Q_star3s'] = []
things['Q_star4s'] = []
things['Q_gmms'] = []
things['Q_kdes'] = []
things['data'] = []
# Iterate through different Ns
for n,N in enumerate(Ns):
# Draw data from distribution
xis, yis = draw_from_Q_true(N, bbox)
# Define grid
[R, xxx, yyy] = sp.histogram2d(xis, yis, [xedges, yedges], normed='True')
# Flatten H into R matrix
things['Rs'].append(R)
plt.subplot(num_Ns,7,2+7*n)
plt.imshow(R, interpolation='nearest', cmap=cmap)
plt.clim(clim)
plt.axis('off')
plt.title(r'R, N=%d'%N)
# Do DEFT calculation (alpha = 2)
Q_star_func2, results2 = deft_2d(xis, yis, bbox, G=G, alpha=2.0, details=True, verbose=True)
Q_star2 = results2.Q_star
things['Q_star2s'].append(Q_star2)
plt.subplot(num_Ns,7,3+7*n)
plt.imshow(Q_star_func2(xfine,yfine), interpolation='nearest', cmap=cmap)
def tvhist2d (x, y, xmin=None, xmax=None, ymin=None, ymax=None,
vmin=None, vmax=None, bins=[100,100],
xtitle="", ytitle="", title=None,
noerase=False, weights=None, zlog=False,
xflip=False, yflip=False,
smooth=None, quick=False,
cmap='gray_r', normx=None, normy=None,
xlog=False, ylog=False, weight_norm=False,
vminfrac=None, vmaxfrac=None, position=None,
xaxis_formatter=None,yaxis_formatter=None,
bar=False, bar_label='', bar_fraction=0.05,
bar_pad=0.05, bar_ticks_locator=None,
bar_formatter=None, apply_func = None, zsqrt=False,
ret_hist=False, interpolation='nearest', scatter_thresh=None,
scatter_opt={},
subplot=None,
**kw):
""" Plot the 2D histogram of the data
Example:
>> tvhist2d(xs,ys,bins=[30,30])
>> tvhist2d(xs,ys,0,10,-1,2,bins=[30,30])
Keyword arguments:
-----------------
xmin,xmax,ymin,ymax
the ranges for x,y where the histogram is constructed.
These params can be specified not only as keywords but also as normal arguments
>> tvhist2d(xs,ys,xmin=0,ymin=10,ymin=-1,ymax=2,bins=[30,30])
>> tvhist2d(xs,ys,0,10,-1,2,bins=[30,30])
vmin,vmax
the ranges of intensities shown on the plot
bins
the list of two integers specifying how many bins in x,y you want
smooth
if not None this parameter controls additional smoothing of the 2D histogram
xflip, yflip
boolean parameters allowing to flip x,y axes
normx, normy
params controlling the normalization of the histogram along X or Y axes
if normx=='sum' then the normalization is done in such way that the sum
is the same for each row/column
if normx=='max' then the normalization is don in such way that
the brightest pixel value will be the same for each row/column
weight_norm
if True the value in each bin is mean weight of points within
the bin
bar
boolean parameter for switching on/off the plotting of the color-bar
bar_pad, bar_fraction
padding and fraction for bar placement
bar_ticks_locator
locator for the tickmarks on the colorbar
"""
x1 = listToArrFlat(x)
y1 = listToArrFlat(y)
#ind = numpy.isfinite(x1) & numpy.isfinite(y1)
if xmin is None:
xmin = numpy.nanmin(x1)
if ymin is None:
ymin = numpy.nanmin(y1)
if xmax is None:
xmax = numpy.nanmax(x1)
if ymax is None:
ymax = numpy.nanmax(y1)
range1 = (xmin, xmax, ymin, ymax)
if xlog is True:
x1=numpy.log10(x1)
xmin,xmax=numpy.log10(xmin),numpy.log10(xmax)
if ylog is True:
y1=numpy.log10(y1)
ymin,ymax=numpy.log10(ymin),numpy.log10(ymax)
range = [[ymin, ymax],[xmin, xmax]]
binsRev = bins[::-1]
if not quick:
hh, yedges, xedges = scipy.histogram2d(y1, x1, range=range,
bins=binsRev, weights=weights)
if weight_norm:
hh1, yedges, xedges = scipy.histogram2d(y1, x1, range=range,
bins=binsRev, weights=None)
hh = hh*1./hh1
else:
import quick_hist
hh = quick_hist.quick_hist((y1, x1), range=range, nbins=binsRev,
weights=weights)
if weight_norm:
hh1 = quick_hist.quick_hist((y1, x1), range=range, nbins=binsRev)
hh = hh*1./hh1
if apply_func is not None:
hh = apply_func (hh)
if normx is not None:
if normx == 'sum':
hh = hh*1./hh.sum(axis=0)[numpy.newaxis,:]#/numpy.maximum(hh.sum(axis=0),1)[numpy.newaxis,:]
elif normx == 'max':
hh = hh*1./numpy.max(hh,axis=0)[numpy.newaxis,:]
else:
raise Exception('unknown normx mode')
if normy is not None:
if normy == 'sum':
hh = hh*1./hh.sum(axis=1)[:,numpy.newaxis]#/numpy.maximum(hh.sum(axis=1),1)[:,numpy.newaxis]
elif normy == 'max':
hh = hh*1./numpy.max(hh,axis=1)[:,numpy.newaxis]
else:
raise Exception('unknown normy mode')
if scatter_thresh is not None:
locx = np.linspace(xmin,xmax, bins[0]+1, True)
locy = np.linspace(ymin, ymax, bins[1]+1, True)
posx = np.digitize(x1, locx)
posy = np.digitize(y1, locy)
#select points within the histogram
ind = (posx > 0) & (posx <= bins[0]) & (posy > 0) & (posy <= bins[1])
hhsub = hh.T[posx[ind] - 1, posy[ind] - 1] # values of the histogram where the points are
x1 = x1[ind][hhsub < scatter_thresh] # low density points
y1 = y1[ind][hhsub < scatter_thresh]
hh[hh < scatter_thresh] = np.nan # fill the areas with low density by NaNs
if xflip:
range1 = (range1[1], range1[0], range1[2], range1[3])
hh = numpy.fliplr(hh)
if yflip:
range1 = (range1[0], range1[1], range1[3], range1[2])
hh = numpy.flipud(hh)
if subplot is not None:
noerase=True
plt.subplot(subplot)
if not noerase:
plt.gcf().clf()
if position is not None:
mypos=position[:]
mypos[2]=position[2]-position[0]
mypos[3]=position[3]-position[1]
plt.axes(mypos)
axis = plt.gca()
axis.set_xlabel(xtitle)
axis.set_ylabel(ytitle)
axis.minorticks_on()
if xaxis_formatter is not None:
axis.xaxis.set_major_formatter(xaxis_formatter)
if yaxis_formatter is not None:
axis.yaxis.set_major_formatter(yaxis_formatter)
if smooth is not None:
hh = scipy.ndimage.filters.gaussian_filter(hh, [smooth, smooth])
if vminfrac is not None and vmin is None:
vmin = scipy.stats.scoreatpercentile(hh.flat, 100 * vminfrac)
if vmaxfrac is not None and vmax is None:
vmax = scipy.stats.scoreatpercentile(hh.flat, 100 * vmaxfrac)
if vmin is not None and vmax is not None and vmin>=vmax:
warnings.warn("vmin is >= vmax... Resetting their values",RuntimeWarning)
vmin=None
vmax=None
if zlog:
norm = matplotlib.colors.LogNorm(vmin=vmin, vmax=vmax)
elif zsqrt:
norm = matplotlib.colors.SqrtNorm(vmin=vmin, vmax=vmax)
else:
norm = matplotlib.colors.Normalize(vmin=vmin, vmax=vmax)
axim=plt.imshow(hh, extent=range1, aspect='auto', interpolation=interpolation,
cmap=cmap, norm=norm, origin='lower', **kw)
if scatter_thresh is not None:
if 'ps' not in scatter_opt:
scatter_opt=copy.copy(scatter_opt)
scatter_opt['ps']=3
oplot(x1,y1,**scatter_opt)
if bar:
if bar_formatter is None:
bar_formatter = matplotlib.ticker.ScalarFormatter()
if int(''.join((matplotlib.__version__).split('.')[:2]))>=11:
kw={'use_gridspec':True}
else:
kw={}
cb=plt.colorbar(fraction=bar_fraction, pad=bar_pad,
norm=axim.norm, ax=axis, format=bar_formatter,
ticks=bar_ticks_locator,**kw)
cb.set_label(bar_label)
if xlog:
plt.gca().set_xscale('log')
if ylog:
plt.gca().set_yscale('log')
if title is not None:
plt.title(title)
if ret_hist:
return hh
return axim
things['Rs'] = []
things['Q_star2s'] = []
things['Q_star3s'] = []
things['Q_star4s'] = []
things['Q_gmms'] = []
things['Q_kdes'] = []
things['data'] = []
# Iterate through different Ns
for n, N in enumerate(Ns):
# Draw data from distribution
xis, yis = draw_from_Q_true(N, bbox)
# Define grid
[R, xxx, yyy] = sp.histogram2d(xis, yis, [xedges, yedges], normed='True')
# Flatten H into R matrix
things['Rs'].append(R)
plt.subplot(num_Ns, 7, 2 + 7 * n)
plt.imshow(R, interpolation='nearest', cmap=cmap)
plt.clim(clim)
plt.axis('off')
plt.title(r'R, N=%d' % N)
# Do DEFT calculation (alpha = 2)
Q_star_func2, results2 = deft_2d(xis,
yis,
bbox,
G=G,
alpha=2.0,
def plot_params(args):
"""Plot alpha, theta, and the emission probabilities"""
old_err = sp.seterr(under='ignore')
oldsize = matplotlib.rcParams['font.size']
K, L = args.emit_probs.shape if not args.continuous_observations else args.means.shape
# alpha
#matplotlib.rcParams['font.size'] = 12
pyplot.figure()
_, xedges, yedges = sp.histogram2d([0,K], [0,K], bins=[K,K])
extent = [xedges[0], xedges[-1], yedges[0], yedges[-1]]
pyplot.imshow(args.alpha.astype(sp.float64), extent=extent, interpolation='nearest',
vmin=0, vmax=1, cmap='OrRd', origin='lower')
pyplot.xticks(sp.arange(K) + .5, sp.arange(K)+1)
pyplot.gca().set_xticks(sp.arange(K)+1, minor=True)
pyplot.yticks(sp.arange(K) + .5, sp.arange(K)+1)
pyplot.gca().set_yticks(sp.arange(K)+1, minor=True)
pyplot.grid(which='minor', alpha=.2)
for line in pyplot.gca().yaxis.get_ticklines() + pyplot.gca().xaxis.get_ticklines() + pyplot.gca().xaxis.get_ticklines(minor=True) + pyplot.gca().yaxis.get_ticklines(minor=True):
# label is a Text instance
line.set_markersize(0)
pyplot.ylabel('Horizontal parent state')
pyplot.xlabel('Node state')
pyplot.title(r"Top root transition ($\alpha$) for {approx} iteration {iteration}".
format(approx=args.approx, iteration=args.iteration))
b = pyplot.colorbar(shrink=.9)
b.set_label("Probability")
outfile = (args.out_params + '_it{iteration}.png').format(param='alpha', **args.__dict__)
pyplot.savefig(os.path.join(args.out_dir, outfile), dpi=240)
# beta
pyplot.figure()
_, xedges, yedges = sp.histogram2d([0,K], [0,K], bins=[K,K])
extent = [xedges[0], xedges[-1], yedges[0], yedges[-1]]
pyplot.clf()
pyplot.imshow(args.beta.astype(sp.float64), extent=extent, interpolation='nearest',
vmin=0, vmax=1, cmap='OrRd', origin='lower')
pyplot.xticks(sp.arange(K) + .5, sp.arange(K)+1)
pyplot.gca().set_xticks(sp.arange(K)+1, minor=True)
pyplot.yticks(sp.arange(K) + .5, sp.arange(K)+1)
pyplot.gca().set_yticks(sp.arange(K)+1, minor=True)
pyplot.grid(which='minor', alpha=.2)
for line in pyplot.gca().yaxis.get_ticklines() + pyplot.gca().xaxis.get_ticklines() + pyplot.gca().xaxis.get_ticklines(minor=True) + pyplot.gca().yaxis.get_ticklines(minor=True):
# label is a Text instance
line.set_markersize(0)
pyplot.ylabel('Vertical parent state')
pyplot.xlabel('Node state')
pyplot.title(r"Left root transition ($\beta$) for {approx} iteration {iteration}".
format(approx=args.approx, iteration=args.iteration))
b = pyplot.colorbar(shrink=.9)
b.set_label("Probability")
outfile = (args.out_params + '_it{iteration}.png').format(param='beta', **args.__dict__)
pyplot.savefig(os.path.join(args.out_dir, outfile), dpi=240)
# theta
if args.separate_theta:
theta_tmp = args.theta
for i in range((args.theta.shape)[0]):
setattr(args, 'theta_%s'%(i+1), args.theta[i,:,:,:])
for theta_name in ['theta'] + ['theta_%s' % i for i in range(20)]:
#print 'trying', theta_name
if not hasattr(args, theta_name):
#print 'missing', theta_name
continue
_, xedges, yedges = sp.histogram2d([0,K], [0,K], bins=[K,K])
extent = [xedges[0], xedges[-1], yedges[0], yedges[-1]]
if K == 18:
numx_plots = 6
numy_plots = 3
elif K == 15:
numx_plots = 5
numy_plots = 3
else:
numx_plots = int(ceil(sp.sqrt(K)))
numy_plots = int(ceil(sp.sqrt(K)))
matplotlib.rcParams['font.size'] = 8
fig, axs = pyplot.subplots(numy_plots, numx_plots, sharex=True, sharey=True, figsize=(numx_plots*2.5,numy_plots*2.5))
for k in xrange(K):
pltx, plty = k // numx_plots, k % numx_plots
#axs[pltx,plty].imshow(args.theta[k,:,:], extent=extent, interpolation='nearest',
axs[pltx,plty].imshow(getattr(args, theta_name)[:,k,:].astype(sp.float64), extent=extent, interpolation='nearest',
vmin=0, vmax=1, cmap='OrRd', aspect='auto', origin='lower')
#if k < numx_plots:
#axs[pltx,plty].text(0 + .5, K - .5, 'vp=%s' % (k+1), horizontalalignment='left', verticalalignment='top', fontsize=10)
axs[pltx,plty].text(0 + .5, K - .5, 'hp=%s' % (k+1), horizontalalignment='left', verticalalignment='top', fontsize=10)
#axs[pltx,plty].xticks(sp.arange(K) + .5, sp.arange(K))
#axs[pltx,plty].yticks(sp.arange(K) + .5, sp.arange(K))
axs[pltx,plty].set_xticks(sp.arange(K) + .5)
axs[pltx,plty].set_xticks(sp.arange(K)+1, minor=True)
axs[pltx,plty].set_xticklabels(sp.arange(K) + 1)
axs[pltx,plty].set_yticks(sp.arange(K) + .5)
axs[pltx,plty].set_yticks(sp.arange(K)+1, minor=True)
axs[pltx,plty].set_yticklabels(sp.arange(K) + 1)
for line in axs[pltx,plty].yaxis.get_ticklines() + axs[pltx,plty].xaxis.get_ticklines() + axs[pltx,plty].yaxis.get_ticklines(minor=True) + axs[pltx,plty].xaxis.get_ticklines(minor=True):
line.set_markersize(0)
axs[pltx,plty].grid(True, which='minor', alpha=.2)
#fig.suptitle(r"$\Theta$ with fixed parents for {approx} iteration {iteration}".
# format(approx=args.approx, iteration=args.iteration),
# fontsize=14, verticalalignment='top')
fig.suptitle('Node state', y=.03, fontsize=14, verticalalignment='center')
#fig.suptitle('Horizontal parent state', y=.5, x=.02, rotation=90,
fig.suptitle('Vertical parent state', y=.5, x=.02, rotation=90,
verticalalignment='center', fontsize=14)
matplotlib.rcParams['font.size'] = 6.5
fig.subplots_adjust(wspace=.05, hspace=.05, left=.05, right=.95)
#b = fig.colorbar(shrink=.9)
#b.set_label("Probability")
outfile = (args.out_params + '_vertparent_it{iteration}.png').format(param=theta_name, **args.__dict__)
pyplot.savefig(os.path.join(args.out_dir, outfile), dpi=240)
fig, axs = pyplot.subplots(numy_plots, numx_plots, sharex=True, sharey=True, figsize=(numx_plots*2.5,numy_plots*2.5))
for k in xrange(K):
pltx, plty = k // numx_plots, k % numx_plots
axs[pltx,plty].imshow(getattr(args, theta_name)[k,:,:].astype(sp.float64), extent=extent, interpolation='nearest',
#axs[pltx,plty].imshow(args.theta[:,k,:], extent=extent, interpolation='nearest',
vmin=0, vmax=1, cmap='OrRd', aspect='auto', origin='lower')
#if k < numx_plots:
axs[pltx,plty].text(0 + .5, K - .5, 'vp=%s' % (k+1), horizontalalignment='left', verticalalignment='top', fontsize=10)
#axs[pltx,plty].xticks(sp.arange(K) + .5, sp.arange(K))
#axs[pltx,plty].yticks(sp.arange(K) + .5, sp.arange(K))
axs[pltx,plty].set_xticks(sp.arange(K) + .5)
axs[pltx,plty].set_xticks(sp.arange(K)+1, minor=True)
axs[pltx,plty].set_xticklabels(sp.arange(K) + 1)
axs[pltx,plty].set_yticks(sp.arange(K) + .5)
axs[pltx,plty].set_yticks(sp.arange(K)+1, minor=True)
axs[pltx,plty].set_yticklabels(sp.arange(K) + 1)
for line in axs[pltx,plty].yaxis.get_ticklines() + axs[pltx,plty].xaxis.get_ticklines() + axs[pltx,plty].yaxis.get_ticklines(minor=True) + axs[pltx,plty].xaxis.get_ticklines(minor=True):
line.set_markersize(0)
axs[pltx,plty].grid(True, which='minor', alpha=.2)
#fig.suptitle(r"$\Theta$ with fixed parents for {approx} iteration {iteration}".
# format(approx=args.approx, iteration=args.iteration),
# fontsize=14, verticalalignment='top')
fig.suptitle('Node state', y=.03, fontsize=14, verticalalignment='center')
fig.suptitle('Horizontal parent state', y=.5, x=.02, rotation=90,
#fig.suptitle('Vertical parent state', y=.5, x=.02, rotation=90,
verticalalignment='center', fontsize=14)
matplotlib.rcParams['font.size'] = 6.5
fig.subplots_adjust(wspace=.05, hspace=.05, left=.05, right=.95)
#b = fig.colorbar(shrink=.9)
#b.set_label("Probability")
outfile = (args.out_params + '_it{iteration}.png').format(param=theta_name, **args.__dict__)
pyplot.savefig(os.path.join(args.out_dir, outfile), dpi=240)
# emission probabilities
if args.continuous_observations:
# plot mean values
matplotlib.rcParams['font.size'] = 8
pyplot.figure(figsize=(max(1,round(L/3.)),max(1, round(K/3.))))
print (max(1,round(L/3.)),max(1, round(K/3.)))
pyplot.imshow(args.means.astype(sp.float64), interpolation='nearest', aspect='auto',
vmin=0, vmax=args.means.max(), cmap='OrRd', origin='lower')
for k in range(K):
for l in range(L):
pyplot.text(l, k, '%.1f' % (args.means[k,l]), horizontalalignment='center', verticalalignment='center', fontsize=5)
pyplot.yticks(sp.arange(K), sp.arange(K)+1)
pyplot.gca().set_yticks(sp.arange(K)+.5, minor=True)
pyplot.xticks(sp.arange(L), valid_marks, rotation=30, horizontalalignment='right')
pyplot.gca().set_xticks(sp.arange(L)+.5, minor=True)
pyplot.grid(which='minor', alpha=.2)
for line in pyplot.gca().yaxis.get_ticklines() + pyplot.gca().xaxis.get_ticklines() + pyplot.gca().xaxis.get_ticklines(minor=True) + pyplot.gca().yaxis.get_ticklines(minor=True):
# label is a Text instance
line.set_markersize(0)
pyplot.ylabel('Hidden State')
pyplot.title("Emission Mean")
#b = pyplot.colorbar(shrink=.7)
#b.set_label("Probability")
outfile = (args.out_params + '_it{iteration}.png').format(param='emission_means', **args.__dict__)
pyplot.savefig(os.path.join(args.out_dir, outfile), dpi=240)
# plot variances
pyplot.figure(figsize=(max(1,round(L/3.)),max(1, round(K/3.))))
print (L/3,K/3.)
pyplot.imshow(args.variances.astype(sp.float64), interpolation='nearest', aspect='auto',
vmin=0, vmax=args.variances.max(), cmap='OrRd', origin='lower')
for k in range(K):
for l in range(L):
pyplot.text(l, k, '%.1f' % (args.variances[k,l]), horizontalalignment='center', verticalalignment='center', fontsize=5)
pyplot.yticks(sp.arange(K), sp.arange(K)+1)
pyplot.gca().set_yticks(sp.arange(K)+.5, minor=True)
pyplot.xticks(sp.arange(L), valid_marks, rotation=30, horizontalalignment='right')
pyplot.gca().set_xticks(sp.arange(L)+.5, minor=True)
pyplot.grid(which='minor', alpha=.2)
for line in pyplot.gca().yaxis.get_ticklines() + pyplot.gca().xaxis.get_ticklines() + pyplot.gca().xaxis.get_ticklines(minor=True) + pyplot.gca().yaxis.get_ticklines(minor=True):
# label is a Text instance
line.set_markersize(0)
pyplot.ylabel('Hidden State')
pyplot.title("Emission Variance")
#b = pyplot.colorbar(shrink=.7)
#b.set_label("Probability")
outfile = (args.out_params + '_it{iteration}.png').format(param='emission_variances', **args.__dict__)
pyplot.savefig(os.path.join(args.out_dir, outfile), dpi=240)
else:
matplotlib.rcParams['font.size'] = 8
pyplot.figure(figsize=(max(1,round(L/3.)),max(1, round(K/3.))))
print (L/3,K/3.)
pyplot.imshow(args.emit_probs.astype(sp.float64), interpolation='nearest', aspect='auto',
vmin=0, vmax=1, cmap='OrRd', origin='lower')
for k in range(K):
for l in range(L):
pyplot.text(l, k, '%2.0f' % (args.emit_probs[k,l] * 100), horizontalalignment='center', verticalalignment='center')
pyplot.yticks(sp.arange(K), sp.arange(K)+1)
pyplot.gca().set_yticks(sp.arange(K)+.5, minor=True)
pyplot.xticks(sp.arange(L), valid_marks, rotation=30, horizontalalignment='right')
pyplot.gca().set_xticks(sp.arange(L)+.5, minor=True)
pyplot.grid(which='minor', alpha=.2)
for line in pyplot.gca().yaxis.get_ticklines() + pyplot.gca().xaxis.get_ticklines() + pyplot.gca().xaxis.get_ticklines(minor=True) + pyplot.gca().yaxis.get_ticklines(minor=True):
# label is a Text instance
line.set_markersize(0)
pyplot.ylabel('Hidden State')
pyplot.title("Emission probabilities")
#b = pyplot.colorbar(shrink=.7)
#b.set_label("Probability")
outfile = (args.out_params + '_it{iteration}.png').format(param='emission', **args.__dict__)
pyplot.savefig(os.path.join(args.out_dir, outfile), dpi=240)
#broad_paper_enrichment = sp.array([[16,2,2,6,17,93,99,96,98,2],
# [12,2,6,9,53,94,95,14,44,1],
# [13,72,0,9,48,78,49,1,10,1],
# [11,1,15,11,96,99,75,97,86,4],
# [5,0,10,3,88,57,5,84,25,1],
# [7,1,1,3,58,75,8,6,5,1],
# [2,1,2,1,56,3,0,6,2,1],
# [92,2,1,3,6,3,0,0,1,1],
# [5,0,43,43,37,11,2,9,4,1],
# [1,0,47,3,0,0,0,0,0,1],
# [0,0,3,2,0,0,0,0,0,0],
# [1,27,0,2,0,0,0,0,0,0],
# [0,0,0,0,0,0,0,0,0,0],
# [22,28,19,41,6,5,26,5,13,37],
# [85,85,91,88,76,77,91,73,85,78],
# [float('nan'), float('nan'),float('nan'),float('nan'),float('nan'),float('nan'),float('nan'),float('nan'),float('nan'),float('nan')]
# ]) / 100.
#mapping_from_broad = dict(zip(range(K), (5,2,0,14,4,6,9,1,12,-1,3,12,8,7,10,12,11,13)))
#broad_paper_enrichment = broad_paper_enrichment[tuple(mapping_from_broad[i] for i in range(K)), :]
#broad_names = ['Active promoter', 'Weak promoter', 'Inactive/poised promoter', 'Strong enhancer',
# 'Strong enhancer', 'weak/poised enhancer', 'Weak/poised enhancer', 'Insulator',
# 'Transcriptional transition', 'Transcriptional elongation', 'Weak transcribed',
# 'Polycomb repressed', 'Heterochrom; low signal', 'Repetitive/CNV', 'Repetitive/CNV',
# 'NA', 'NA', 'NA']
#pyplot.figure(figsize=(L/3,K/3.))
#print (L/3,K/3.)
#pyplot.imshow(broad_paper_enrichment, interpolation='nearest', aspect='auto',
# vmin=0, vmax=1, cmap='OrRd', origin='lower')
#for k in range(K):
# for l in range(L):
# pyplot.text(l, k, '%2.0f' % (broad_paper_enrichment[k,l] * 100), horizontalalignment='center', verticalalignment='center')
# pyplot.text(L, k, broad_names[mapping_from_broad[k]], horizontalalignment='left', verticalalignment='center', fontsize=6)
#pyplot.yticks(sp.arange(K), sp.arange(K)+1)
#pyplot.gca().set_yticks(sp.arange(K)+.5, minor=True)
#pyplot.xticks(sp.arange(L), valid_marks, rotation=30, horizontalalignment='right')
#pyplot.gca().set_xticks(sp.arange(L)+.5, minor=True)
#pyplot.grid(which='minor', alpha=.2)
#for line in pyplot.gca().yaxis.get_ticklines() + pyplot.gca().xaxis.get_ticklines() + pyplot.gca().xaxis.get_ticklines(minor=True) + pyplot.gca().yaxis.get_ticklines(minor=True):
## label is a Text instance
# line.set_markersize(0)
#pyplot.ylabel('Hidden State')
#pyplot.title("Broad paper Emission probabilities")
##b = pyplot.colorbar(shrink=.7)
##b.set_label("Probability")
#pyplot.subplots_adjust(right=.7)
#outfile = (args.out_params + '_broadpaper.png').format(param='emission', **args.__dict__)
#pyplot.savefig(os.path.join(args.out_dir, outfile), dpi=240)
pyplot.close('all')
sp.seterr(**old_err)
matplotlib.rcParams['font.size'] = oldsize
def single_pixel_study(xs, ys, field_label, profile, nbins=None, **kwargs):
"""Plot a study of single-pixel and general behaviour of profile data.
Args:
xs (np.array): pixel (relative) distance from nuclear lamina.
ys (np.array): pixel channel intensity.
field_label (string): y-axis label.
profile (dict): profile to study.
nbins (int): study precision (opt, def 200).
Returns:
plt.figure: current figure canvas.
"""
# CHECK PARAMS =============================================================
# Default values
if None == nbins:
nbins = 200
# PREPARE DATA =============================================================
# XY range
xyrange = [[xs.min(), xs.max()], [ys.min(), ys.max()]]
# Bin data
assigned_bins = np.digitize(xs, np.linspace(xs.min(), xs.max(), nbins))
binned_ydat = [[] for i in range(nbins)]
for bin_id in range(nbins):
binned_ydat[bin_id] = ys[np.where(assigned_bins == bin_id)]
# Calculate local density
hh, locx, locy = scipy.histogram2d(xs,
ys,
range=xyrange,
bins=[nbins, nbins])
posx = np.digitize(xs, locx)
posy = np.digitize(ys, locy)
# Hide empty bins
pp = (hh.T / hh.T.max(0))
pp[:, hh.T.sum(0) == 0] = 0
# MULTIPLOT ================================================================
# Init figure --------------------------------------------------------------
fig = plt.figure(figsize=[40, 15])
rc('font', **{'size': 15})
# Number of pixels plot ----------------------------------------------------
plt.subplot2grid((4, 3), (0, 0))
plt.plot(range(nbins), hh.sum(1))
plt.ylabel('Number of pixels')
plt.xlabel(kwargs['dlabel'])
# Adjust ticks
ax = plt.gca()
ticks = ax.get_xticks() / float(nbins) * xyrange[0][1]
ticks = [const.SCI_FORMAT % n for n in ticks]
ax.get_xaxis().set_visible(False)
plt.ticklabel_format(style='sci', axis='y', scilimits=(0, 0))
# Boxplot ------------------------------------------------------------------
ax = plt.subplot2grid((4, 3), (1, 0))
plt.boxplot(binned_ydat, sym='k.')
plt.ylabel(field_label)
ax.get_xaxis().set_visible(False)
plt.ticklabel_format(style='sci', axis='y', scilimits=(0, 0))
# STD plot -----------------------------------------------------------------
plt.subplot2grid((4, 3), (2, 0))
plt.plot(profile['std_raw'], 'r')
plt.plot(profile['std_raw'], 'r.')
plt.ylabel('std(' + field_label + ')')
plt.hold(True)
plt.plot(profile['std'], 'k')
plt.xlabel(kwargs['dlabel'])
# Adjust ticks
ax = plt.gca()
ax.set_xticklabels(ticks)
plt.ticklabel_format(style='sci', axis='y', scilimits=(0, 0))
# Mean plot ----------------------------------------------------------------
plt.subplot2grid((4, 3), (0, 1))
plt.plot(profile['mean_raw'], 'b')
plt.plot(profile['mean_raw'], 'b.')
plt.ylabel('mean(' + field_label + ')')
plt.hold(True)
plt.plot(profile['mean'], 'y')
# Adjust ticks
ax = plt.gca()
ax.get_xaxis().set_visible(False)
plt.ticklabel_format(style='sci', axis='y', scilimits=(0, 0))
# Median plot --------------------------------------------------------------
plt.subplot2grid((4, 3), (1, 1))
plt.plot(profile['median_raw'], 'r')
plt.plot(profile['median_raw'], 'r.')
plt.ylabel('median(' + field_label + ')')
plt.hold(True)
plt.plot(profile['median'], 'g')
# Adjust ticks
ax = plt.gca()
ax.get_xaxis().set_visible(False)
plt.ticklabel_format(style='sci', axis='y', scilimits=(0, 0))
# Mode plot ----------------------------------------------------------------
plt.subplot2grid((4, 3), (2, 1))
plt.plot(profile['mode_raw'], 'c')
plt.plot(profile['mode_raw'], 'c.')
plt.ylabel('mode(' + field_label + ')')
plt.hold(True)
plt.plot(profile['mode'], 'm')
plt.xlabel(kwargs['dlabel'])
# Adjust ticks
ax = plt.gca()
ax.get_xaxis().set_visible(False)
plt.ticklabel_format(style='sci', axis='y', scilimits=(0, 0))
# Max plot -----------------------------------------------------------------
plt.subplot2grid((4, 3), (3, 1))
plt.plot(profile['max_raw'], 'c')
plt.plot(profile['max_raw'], 'c.')
plt.ylabel('max(' + field_label + ')')
plt.hold(True)
plt.plot(profile['max'], 'b')
plt.xlabel(kwargs['dlabel'])
# Adjust ticks
ax = plt.gca()
ax.set_xticklabels(ticks)
plt.ticklabel_format(style='sci', axis='y', scilimits=(0, 0))
# Local density scatterplot ------------------------------------------------
plt.subplot2grid((4, 3), (0, 2), rowspan=4, colspan=2)
profile_density_scatterplot(pp,
field_label,
nbins,
xyrange,
profile,
dlabel=kwargs['dlabel'],
new_figure=False)
# SINGLE PLOTS =============================================================
if kwargs['plotting']:
# Density scatterplot --------------------------------------------------
# Retrieve tmp figure
tmp_fig = profile_density_scatterplot(pp,
field_label,
nbins,
xyrange,
profile,
dlabel=kwargs['dlabel'])
# Get filename suffix
suffix = field_label.lower().split(' ')[0].replace('/', '_')
# Get condition description
if kwargs['cond_name'] in kwargs['cdescr'].keys():
suptitle = kwargs['cdescr'][kwargs['cond_name']]
else:
suptitle = kwargs['cond_name']
plt.suptitle(suptitle)
# Set output name
fname = kwargs['outdir'] + const.OUTDIR_PNG_REPORT + kwargs['cond_name']
fname += '.' + suffix + '.density_profile'
# Add partial suffix
if 'partial' in kwargs.keys():
if kwargs['partial']:
fname += '.part'
# Add general suffix
fname += kwargs['suffix'] + '.png'
# Export
export(fname, 'png')
# Close tmp figure
plt.close(tmp_fig)
# Number of pixels plot -----------------------------------------------
# Init tmp figure
tmp_fig = plt.figure()
# Plot
plt.plot(range(nbins), hh.sum(1))
plt.ylabel('Number of pixels')
plt.xlabel(kwargs['dlabel'])
ticks = ax.get_xticks() / float(nbins) * xyrange[0][1]
ticks = [const.SCI_FORMAT % n for n in ticks]
plt.gca().set_xticklabels(ticks)
plt.ticklabel_format(style='sci', axis='y', scilimits=(0, 0))
# Set suptitle
if 'partial' in kwargs.keys():
if kwargs['partial']:
suptitle += ' [partial volume '
suptitle += str(kwargs['part_n_erosion']) + ']'
plt.suptitle(suptitle)
# Set output name
fname = kwargs['outdir'] + const.OUTDIR_PNG_REPORT + kwargs['cond_name']
fname += '.' + suffix + '.npx'
# Add partial suffix
if 'partial' in kwargs.keys():
if kwargs['partial']:
fname += '.part'
# Add general suffix
fname += kwargs['suffix'] + '.png'
# Export
export(fname, 'png')
# Close tmp figure
plt.close(tmp_fig)
# Return MULTIPLOT canvas
return (fig)
def deft_2d(xis_raw, yis_raw, bbox, G=20, alpha=2, num_samples=0, tol=1E-3, ti_shift=-1, tf_shift=0, verbose=False, details=False):
'''
Performs DEFT density estimation in 2D
Args:
xis_raw: The x-data, comprising a list (or scipy array) of real numbers.
Data falling outside bbox is discarded.
yis_raw: The y-data, comprising a list (or scipy array) of real numbers.
Data falling outside bbox is discarded.
bbox: The domain (bounding box) on which density estimation is
performed. This should be of the form [xmin, xmax, ymin, ymax].
Density estimation will be performed on G grid points evenly spaced
within this interval. Max and min grid points are placed half a grid
spacing in from the boundaries.
G: Number of grid points to use in each dimension. Total number of
gridpoints used in the calculation is G**2.
alpha: The smoothness parameter, specifying which derivative of the
filed is constrained by the prior. May be any integer >= 2.
num_samples: The number of density estimates to sample from the Bayesian
posterior. If zero is passed, no sample are drawn.
num_ts: The number of lengthscales at which to compute Q_\ell.
ti_shift: Initial integration t is set to
t_i = log[ (2*pi)**(2*alpha) * L**(2-2*alpha)] + ti_shift
(see Eq. 10)
tf_shift: Final integration t is set to
t_f = log[ (2*pi*G)**(2*alpha) * L**(2-2*alpha)] + tf_shift
(see Eq. 10)
verbose: If True, user feedback is provided
details: If True, calculation details are returned along with the
density estimate Q_star_func
Returns:
Q_star_func: A function, defined within bbox, providing a cubic spline
interpolation of the maximum a posteriori density estimate.
results: Returned only if `details' is set to be True. Contains detailed
information about the density estimation calculation.
'''
# Time execution
start_time = time.clock()
# Check data types
L = G
V = G**2
assert G==sp.floor(G)
assert len(xis_raw) > 1
assert num_samples >= 0 and num_samples==sp.floor(num_samples)
assert 2*alpha-2 >= 1
assert len(bbox) == 4
###
### Draw grid and histogram data
###
# Make sure xis is a numpy array
xis_raw = sp.array(xis_raw)
# If xdomain are specified, check them, and keep only data that falls within
xlb = bbox[0]
xub = bbox[1]
assert(xub-xlb > 0)
ylb = bbox[2]
yub = bbox[3]
assert(yub-ylb > 0)
# Throw away data not within bbox
indices = (xis_raw >= xlb) & (xis_raw <= xub) & (yis_raw >= ylb) & (yis_raw <= yub)
xis = xis_raw[indices]
yis = yis_raw[indices]
# Determine number of data points within bbox
N = len(xis)
# Compute edges for histogramming
xedges = sp.linspace(xlb,xub,L+1)
dx = xedges[1] - xedges[0]
yedges = sp.linspace(ylb,yub,L+1)
dy = yedges[1] - yedges[0]
# Compute grid for binning
xgrid = xedges[:-1]+dx/2
ygrid = yedges[:-1]+dy/2
# Convert to z = normalized x
xzis = (xis-xlb)/dx
yzis = (yis-ylb)/dx
xzedges = (xedges-xlb)/dx
yzedges = (yedges-ylb)/dy
# Histogram data
[H, xxx, yyy] = sp.histogram2d(xzis, yzis, [xzedges, yzedges])
R = H/N
R_flat = R.flatten()
R_col = sp.mat(R_flat).T
###
### Use weak-field approximation to get phi_0
###
# Compute fourier transform of data
R_ks = fft2(R)
# Set constant component of data FT to 0
R_ks[0] = 0
# Set mode numbers seperately for even G and for odd G
if G%2 == 0: # If G is even
# ks = [0, 1, ... , G/2, -G/2+1, ..., -1]
ks = sp.concatenate((sp.arange(0, G/2 + 1), sp.arange(1 - G/2, 0)))
else: # If G is odd
# ks = [0, 1, ... , (G-1)/2, -(G-1)/2, ..., -1]
ks = sp.concatenate((sp.arange(0, (G-1)/2 + 1), sp.arange(-(G-1)/2, 0)))
# Mode numbers corresponding to fourier transform
A = sp.mat(sp.tile((2.0*sp.pi*ks/G)**2.0,[G,1]))
B = A.T
tau_ks = sp.log(V*sp.array(A+B)**alpha)
tau_ks[0,0] = 1 # This number is actually irrelevant since R_hat[0,0] = 0
# Set t_i to the \ell_i vale in Eq. 10 plus shift
t_i = sp.log(((2*sp.pi)**(2*alpha))*G**(2.0-2.0*alpha)) + ti_shift
# Set t_f to the \ell_f value in Eq. 10 plus shift
t_f = sp.log(((2*sp.pi*G)**(2*alpha))*G**(2.0-2.0*alpha)) + tf_shift
# Compute Fourier components of phi
phi_ks = -(G**2)*sp.array(R_ks)/sp.array((1.0 + sp.exp(tau_ks - t_i)))
# Invert the Fourier transform to get phi weak field approx
phi0 = sp.ravel(sp.real(ifft2(phi_ks)))
###
### Integrate ODE
###
# Build 2D Laplacian matrix
delsq_2d = (-4.0*sp.eye(V) +
sp.eye(V,V,-1) + sp.eye(V,V,+1) +
sp.eye(V,V,-G) + sp.eye(V,V,+G) +
sp.eye(V,V,-V+1) + sp.eye(V,V,V-1) +
sp.eye(V,V,-V+G) + sp.eye(V,V,V-G))
# Make Delta, and make it sparse
Delta = csr_matrix(-delsq_2d)**alpha
# This is the key function: computes deriviate of phi for ODE integration
def this_flow(t, phi):
# Compute distribution Q corresponding to phi
Q = sp.exp(-phi)/V
A = Delta + sp.exp(t)*diags(Q,0)
dphidt = sp.real(spsolve(A, sp.exp(t)*(Q-R_flat)))
# Return the time derivative of phi
return dphidt
backend = 'vode'
solver = ode(this_flow).set_integrator(backend, nsteps=1, atol=tol, rtol=tol)
solver.set_initial_value(phi0, t_i)
# suppress Fortran-printed warning
solver._integrator.iwork[2] = -1
warnings.filterwarnings("ignore", category=UserWarning)
# Make containers
phis = []
ts = []
log_evidence = []
Qs = []
ells = []
log_dets = []
# Will keep initial phi to check that integration is working well
phi0_col = sp.mat(phi0).T
kinetic0 = (phi0_col.T*Delta*phi0_col)[0,0]
# Integrate phi over specified t range.
integration_start_time = time.clock()
keep_going = True
max_log_evidence = -sp.Inf
coeff = 1.0
while solver.t < t_f and keep_going:
# Step integrator
solver.integrate(t_f, step=True)
# Compute deteriminant.
phi = solver.y
t = solver.t
beta = N*sp.exp(-t)
ell = (N/sp.exp(t))**(1.0/(2.0*alpha-2.0))
# Compute new distribution
Q = sp.exp(-phi)/sum(sp.exp(-phi))
phi_col = sp.mat(phi).T
# Check that S[phi] < S[phi0]. If not, phi might just be f****d up, due to the
# integration having to solve a degenerate system of equations.
# In this case, set phi = phi0 and restart integration from there.
S = 0.5*(phi_col.T*Delta*phi_col)[0,0] + sp.exp(t)*(R_col.T*phi_col)[0,0] + sp.exp(t)*sum(sp.exp(-phi)/G)
S0 = 0.5*(phi0_col.T*Delta*phi0_col)[0,0] + sp.exp(t)*(R_col.T*phi0_col)[0,0] + sp.exp(t)*sum(sp.exp(-phi0)/G)
if S0 < S:
t_i = t_i + 0.5
solver = ode(this_flow).set_integrator(backend, nsteps=1, atol=tol, rtol=tol)
solver.set_initial_value(phi0, t_i)
# Reset containers
phis = []
ts = []
log_evidence = []
Qs = []
ells = []
log_dets = []
keep_going = True
max_log_evidence = -sp.Inf
#print 'Restarting integration at t_i = %f'%t_i
else:
# Compute betaS directly again to minimize multiplying very large
# numbers by very small numbers. Also, subtract initial kinetic term
betaS = beta*0.5*(phi_col.T*Delta*phi_col-kinetic0)[0,0] + N*(R_col.T*phi_col)[0,0] + N
# Compute the log determinant of Lambda
Lambda = Delta + sp.exp(t)*sp.diag(Q)
log_det_Lambda, coeff = get_log_det_Lambda(Lambda, coeff)
log_evidence_value = - betaS - 0.5*log_det_Lambda + 0.5*alpha*t #sp.log(beta)
if log_evidence_value > max_log_evidence:
max_log_evidence = log_evidence_value
if (log_evidence_value < max_log_evidence - 300) and (ell < G):
keep_going = False
# Record shit
phis.append(phi)
ts.append(t)
ells.append(ell)
Qs.append(Q)
log_evidence.append(log_evidence_value)
log_dets.append(log_det_Lambda)
warnings.resetwarnings()
if verbose:
print 'Integration took %0.2f seconds.'%(time.clock()-integration_start_time)
# Set ts and ells
ts = sp.array(ts)
phis = sp.array(phis)
Qs = sp.array(Qs)
ells = sp.array(ells)
log_evidence = sp.array(log_evidence)
# Noramlize weights for different ells and save
ell_weights = sp.exp(log_evidence) - max(log_evidence)
ell_weights[ell_weights < -100] = 0.0
assert(not sum(ell_weights) == 0)
assert(all(sp.isfinite(ell_weights)))
ell_weights /= sum(ell_weights)
# Find the best lengthscale
i_star = sp.argmax(log_evidence)
phi_star = phis[i_star,:]
Q_star = sp.reshape(sp.exp(-phi_star)/sum(sp.exp(-phi_star)), [G, G])
ell_star = ells[i_star]
t_star = ts[i_star]
M_star = sp.exp(t_star)
###
### Sample from posterior (only if requirested)
###
# Get ell range
log_ells_raw = sp.log(ells)[::-1]
log_ell_i = max(log_ells_raw)
log_ell_f = min(log_ells_raw)
# Interploate Qs at 300 different ells
K = 1000
phis_raw = phis[::-1,:]
log_ells_grid = sp.linspace(log_ell_f, log_ell_i, K)
# Create function to get interpolated phis
phis_interp_func = interp1d(log_ells_raw, phis_raw, axis=0, kind='cubic')
# Compute weights for each ell on the fine grid
log_weights_func = interp1d(log_ells_raw, sp.log(ell_weights[::-1]), kind='cubic')
log_weights_grid = log_weights_func(log_ells_grid)
weights_grid = sp.exp(log_weights_grid)
weights_grid /= sum(weights_grid)
# If user requests samples
if num_samples > 0:
Lambda_star = (ell_star**(2.0*alpha-1.0))*(Delta + M_star*sp.diag(Q_star))
eigvals, eigvecs = eig(Lambda_star)
# Lambda_star is Hermetian; shouldn't need to do this
eigvals = sp.real(eigvals)
eigvecs = sp.real(eigvecs)
# Initialize container variables
Qs_sampled = sp.zeros([num_samples, G, G])
phis_sampled = sp.zeros([num_samples, V])
#is_sampled = sp.zeros([num_samples])
log_ells_sampled = sp.zeros([num_samples])
for j in range(num_samples):
# First choose a classical path based on
i = choice(K, p=weights_grid)
phi_cl = phis_interp_func(log_ells_grid[i])
#is_sampled[j] = int(i)
log_ells_sampled[j] = log_ells_grid[i]
# Draw random amplitudes for all modes and compute dphi
etas = randn(L)
dphi = sp.ravel(sp.real(sp.mat(eigvecs)*sp.mat(etas/sp.sqrt(eigvals)).T))
# Record final sampled phi
phi = phi_cl + dphi
Qs_sampled[j,:,:] = sp.reshape(sp.exp(-phi)/sum(sp.exp(-phi)), [G,G])
###
### Return results (with everything in correct lenght units!)
###
results = Results()
results.G = G
results.ts = ts
results.N = N
results.alpha = alpha
results.num_samples = num_samples
results.xgrid = xgrid
results.bbox = bbox
results.dx = dx
results.dy = dy
# Everything with units of length gets multiplied by dx!!!
results.ells = [ells*dx, ells*dy]
results.L = [G*dx, G*dy]
results.V = V*dx*dy
results.h = [dx, dy]
results.Qs = Qs/(dx*dy)
results.phis = phis
results.log_evidence = log_evidence
# Comptue star results
phi_star = sp.reshape(phi_star,[G,G])
results.phi_star = deepcopy(phi_star)
results.Q_star = deepcopy(Q_star)/(dx*dy)
results.i_star = i_star
results.ell_star = [ells[i_star]*dx, ells[i_star]*dy]
bbox = [xlb, xub, ylb, yub]
# Create interpolated Q_star. Man this is a pain in the ass!
extended_xgrid = sp.zeros(G+2)
extended_xgrid[1:-1] = xgrid
extended_xgrid[0] = xlb
extended_xgrid[-1] = xub
extended_ygrid = sp.zeros(L+2)
extended_ygrid[1:-1] = ygrid
extended_ygrid[0] = ylb
extended_ygrid[-1] = yub
# Extend grid for phi_star for interpolating function
extended_phi_star = sp.zeros([G+2, G+2])
extended_phi_star[1:-1,1:-1] = phi_star
# Get rows
row = 0.5*(phi_star[0,:] + phi_star[-1,:])
extended_phi_star[0,1:-1] = row
extended_phi_star[-1,1:-1] = row
# Get cols
col = 0.5*(phi_star[:,0] + phi_star[:,-1])
extended_phi_star[1:-1,0] = col
extended_phi_star[1:-1,-1] = col
# Get remaining corners, which share the same value
corner= 0.25*(row[0]+row[-1]+col[0]+col[-1])
extended_phi_star[0,0] = corner
extended_phi_star[0,-1] = corner
extended_phi_star[-1,0] = corner
extended_phi_star[-1,-1] = corner
# Finally, compute interpolated function
phi_star_func = RectBivariateSpline(extended_xgrid, extended_ygrid, extended_phi_star, bbox=bbox)
Z = sp.sum((dx*dy)*sp.exp(-phi_star))
def Q_star_func(x,y):
return sp.exp(-phi_star_func(x,y))/Z
# If samples are requested, return those too
if num_samples > 0:
results.phis_sampled = phis_sampled
results.Qs_sampled = Qs_sampled/(dx*dy)
results.ells_sampled = sp.exp(log_ells_sampled)*dx*dy
# Stop time
time_elapsed = time.clock() - start_time
if verbose:
print 'deft_2d: %1.2f sec for alpha = %d, G = %d, N = %d'%(time_elapsed, alpha, G, N)
if details:
return Q_star_func, results
else:
return Q_star_func
def volWeightBeam(beam, rgrid, zgrid, trace=True, ds=2e-3, toroidal=None, **kwargs):
r"""Generate a tuple of Beam objects from tuples of surface objects
Args:
beam: tuple of Surfaces or a Surface object
Beam origin surfaces, based on the coordinate system
of the surfaces. Center position is accessible through Beam(0),
Beam.x()[...,0] or Beam.r()[...,0] (last two options create
numpy arrays, the first generats a geometry.Vec object).
rgrid: tuple of Surfaces or a Surface object
Direction of the ray can be defined by a vector object (assumed
to be in the space of the pt1 origin) from pt1, or a point, which
generates a vector pointing from pt1 to pt2.
zgrid: tuple of Surfaces or a Surface object
Direction of the ray can be defined by a vector object (assumed
to be in the space of the pt1 origin) from pt1, or a point, which
generates a vector pointing from pt1 to pt2.
Returns:
output: tuple of beam objects.
Examples:
Accepts all surface or surface-derived object inputs, though all data
is stored as a python object.
Generate an y direction Ray in cartesian coords using a Vec from (0,0,1)::
cen = geometry.Center(flag=True)
ydir = geometry.Vecx((0,1,0))
zpt = geometry.Point((0,0,1),cen)
"""
out = scipy.zeros((len(rgrid)-1,len(zgrid)-1))
try:
if toroidal is None:
if trace:
temp = beam(scipy.mgrid[beam.norm.s[-2]:beam.norm.s[-1]:ds]).r()
else:
temp = beam(scipy.mgrid[beam.norm.s[0]:beam.norm.s[-1]:ds]).r()
out += scipy.histogram2d(temp[0],
temp[2],
bins = [rgrid, zgrid],
weights=scipy.ones(temp[0].shape)*beam.etendue*ds)[0]
else:
if trace:
temp = beam(scipy.mgrid[beam.norm.s[-2]:beam.norm.s[-1]:ds]).t(toroidal[0],toroidal[1])
else:
temp = beam(scipy.mgrid[beam.norm.s[0]:beam.norm.s[-1]:ds]).t(toroidal[0],toroidal[1])
out += scipy.histogram2d(temp[2],
temp[0],
bins = [rgrid, zgrid],
weights=scipy.ones(temp[0].shape)*beam.etendue*ds)[0]
except AttributeError:
for i in beam:
try:
out += volWeightBeam(i, rgrid, zgrid, trace=trace, ds=ds, toroidal=toroidal, **kwargs)
except TypeError:
pass
return out
def plot():
mycm = cm.get_cmap('YlOrRd_r')
mycm.set_under('w')
mycm = truncate_colormap(mycm, 0.0, 0.8)
cset = brewer2mpl.get_map('YlOrRd', 'sequential', 9).mpl_colors
set_plot_properties() # change style
tab = Table.read('/data/lc585/QSOSED/Results/150217/sample2/out_add.fits')
tab = tab[~np.isnan(tab['BBT_STDERR'])]
tab = tab[tab['BBT_STDERR'] < 500.]
tab = tab[tab['BBT_STDERR'] > 5.0]
tab = tab[(tab['LUM_IR_SIGMA'] * tab['RATIO_IR_UV']) < 0.4]
fig = plt.figure(figsize=figsize(1, 0.6))
gs = gridspec.GridSpec(6, 13)
ax1 = fig.add_subplot(gs[1:5, 0:4])
ax3 = fig.add_subplot(gs[0, 0:4])
ax4 = fig.add_subplot(gs[1:5, 4:8])
ax6 = fig.add_subplot(gs[0, 4:8])
ax7 = fig.add_subplot(gs[1:5, 8:12])
ax9 = fig.add_subplot(gs[0, 8:12])
ax10 = fig.add_subplot(gs[1:5, 12])
fig.subplots_adjust(wspace=0.0)
fig.subplots_adjust(hspace=0.0)
#histogram definition
xyrange = [[43, 47], [0, 2]] # data range
bins = [60, 40] # number of bins
thresh = 4 #density threshold
#data definition
xdat, ydat = tab['LUM_UV'], tab['RATIO_IR_UV']
# histogram the data
hh, locx, locy = histogram2d(xdat, ydat, range=xyrange, bins=bins)
posx = np.digitize(xdat, locx)
posy = np.digitize(ydat, locy)
#select points within the histogram
ind = (posx > 0) & (posx <= bins[0]) & (posy > 0) & (posy <= bins[1])
hhsub = hh[posx[ind] - 1,
posy[ind] - 1] # values of the histogram where the points are
xdat1 = xdat[ind][hhsub < thresh] # low density points
ydat1 = ydat[ind][hhsub < thresh]
hh[hh < thresh] = np.nan # fill the areas with low density by NaNs
im = ax1.imshow(np.flipud(hh.T),
cmap=mycm,
extent=np.array(xyrange).flatten(),
interpolation='none',
aspect='auto',
vmin=thresh)
ax1.scatter(xdat1, ydat1, color=cset[-1])
ax1.set_ylabel(r'$R_{1-3{\mu}m/UV}$')
ax1.set_ylim(-0.4, 1)
ax1.set_xlim(45, 47)
ax3.hist(tab['LUM_UV'], color=cset[-1], bins=20)
ax3.set_xlim(ax1.get_xlim())
ax3.set_axis_off()
#ax1.get_xaxis().set_ticks([46,46.5,47])
#ax1.get_yaxis().set_ticks([0.05,0.15,0.25,0.35,0.45])
############################################################################
#histogram definition
xyrange = [[7.5, 10.5], [0, 2]] # data range
bins = [40, 40] # number of bins
thresh = 4 #density threshold
#data definition
xdat, ydat = tab['LOGBH'], tab['RATIO_IR_UV']
# histogram the data
hh, locx, locy = histogram2d(xdat, ydat, range=xyrange, bins=bins)
posx = np.digitize(xdat, locx)
posy = np.digitize(ydat, locy)
#select points within the histogram
ind = (posx > 0) & (posx <= bins[0]) & (posy > 0) & (posy <= bins[1])
hhsub = hh[posx[ind] - 1,
posy[ind] - 1] # values of the histogram where the points are
xdat1 = xdat[ind][hhsub < thresh] # low density points
ydat1 = ydat[ind][hhsub < thresh]
hh[hh < thresh] = np.nan # fill the areas with low density by NaNs
im = ax4.imshow(np.flipud(hh.T),
cmap=mycm,
extent=np.array(xyrange).flatten(),
interpolation='none',
aspect='auto',
vmin=thresh)
ax4.scatter(xdat1, ydat1, color=cset[-1])
ax4.set_ylim(ax1.get_ylim())
ax4.set_xlim(7.9, 10.5)
ax4.set_xticklabels([7.5, 8, 8.5, 9, 9.5, 10])
#ax4.set_yticks([0.05,0.15,0.25,0.35,0.45])
ax4.set_yticklabels([])
ax6.hist(tab[tab['LOGBH'] > 6]['LOGBH'], color=cset[-1], bins=20)
ax6.set_xlim(ax4.get_xlim())
ax4.set_ylim(ax1.get_ylim())
ax6.hist(tab['LOGBH'][tab['LOGBH'] > 6], color=cset[-1], bins=20)
ax6.set_xlim(ax4.get_xlim())
ax6.set_axis_off()
######################################################################################
#histogram definition
xyrange = [[-2, 0.5], [0, 2]] # data range
bins = [40, 40] # number of bins
thresh = 4 #density threshold
#data definition
xdat, ydat = tab['LOGEDD_RATIO'], tab['RATIO_IR_UV']
# histogram the data
hh, locx, locy = histogram2d(xdat, ydat, range=xyrange, bins=bins)
posx = np.digitize(xdat, locx)
posy = np.digitize(ydat, locy)
#select points within the histogram
ind = (posx > 0) & (posx <= bins[0]) & (posy > 0) & (posy <= bins[1])
hhsub = hh[posx[ind] - 1,
posy[ind] - 1] # values of the histogram where the points are
xdat1 = xdat[ind][hhsub < thresh] # low density points
ydat1 = ydat[ind][hhsub < thresh]
hh[hh < thresh] = np.nan # fill the areas with low density by NaNs
im = ax7.imshow(np.flipud(hh.T),
cmap=mycm,
extent=np.array(xyrange).flatten(),
interpolation='none',
aspect='auto',
vmin=thresh)
ax7.scatter(xdat1, ydat1, color=cset[-1])
ax7.set_ylim(ax1.get_ylim())
ax7.set_xlim(-1.8, 0.7)
ax9.hist(tab[tab['LOGEDD_RATIO'] > -2.5]['LOGEDD_RATIO'],
color=cset[-1],
bins=20)
ax9.set_xlim(ax7.get_xlim())
ax9.set_axis_off()
plt.tick_params(axis='both', which='major')
#ax7.set_yticks([0.05,0.15,0.25,0.35,0.45])
ax7.set_yticklabels([])
#ax7.set_xticklabels([])
ax9.hist(tab['LOGBH'][tab['LOGBH'] > 6], color=cset[-1], bins=20)
ax9.set_xlim(ax7.get_xlim())
ax9.set_axis_off()
ax10.hist(tab['RATIO_IR_UV'],
orientation='horizontal',
color=cset[-1],
bins=25)
ax10.set_ylim(ax1.get_ylim())
ax10.set_axis_off()
ax4.set_xlabel(r'Log$_{10}$ (BH Mass $M_{\rm BH}$)')
ax1.set_xlabel(r'Log$_{10} (L_{\rm UV} ({\rm erg/s}))$')
ax7.set_xlabel(r'Log$_{10}$ ($\lambda_{\rm Edd}$)')
axcb = fig.add_axes([0.33, 0.1, 0.33, 0.03])
clb = fig.colorbar(im, cax=axcb, orientation='horizontal')
clb.set_label('Number of Objects', fontsize=14)
s1 = spearmanr(tab[tab['LOGEDD_RATIO'] > -2.5]['LOGEDD_RATIO'],
tab[tab['LOGEDD_RATIO'] > -2.5]['RATIO_IR_UV'])[0]
s2 = spearmanr(tab[tab['LOGBH'] > 6]['LOGBH'],
tab[tab['LOGBH'] > 6]['RATIO_IR_UV'])[0]
s3 = spearmanr(tab['LUM_UV'], tab['RATIO_IR_UV'])[0]
p1 = spearmanr(tab[tab['LOGEDD_RATIO'] > -2.5]['LOGEDD_RATIO'],
tab[tab['LOGEDD_RATIO'] > -2.5]['RATIO_IR_UV'])[1]
p2 = spearmanr(tab[tab['LOGBH'] > 6]['LOGBH'],
tab[tab['LOGBH'] > 6]['RATIO_IR_UV'])[1]
p3 = spearmanr(tab['LUM_UV'], tab['RATIO_IR_UV'])[1]
ax1.text(45.6, -0.2, r'$\rho =$ {0:.2f} ({1:.2f})'.format(s3, p3))
ax4.text(8.7, -0.2, r'$\rho =$ {0:.2f} ({1:.2f})'.format(s2, p2))
ax7.text(-1.1, -0.2, r'$\rho =$ {0:.2f} ({1:.2f})'.format(s1, p1))
fig.savefig(
'/home/lc585/thesis/figures/chapter06/ratio_lowz_correlations.pdf')
plt.show()
return None
def deft_2d(xis_raw, yis_raw, bbox, G=20, alpha=2, num_samples=0, num_ts=100, ti_shift=-5, tf_shift=0, verbose=False, details=False):
'''
Performs DEFT density estimation in 2D
Args:
xis_raw: The x-data, comprising a list (or scipy array) of real numbers.
Data falling outside bbox is discarded.
yis_raw: The y-data, comprising a list (or scipy array) of real numbers.
Data falling outside bbox is discarded.
bbox: The domain (bounding box) on which density estimation is
performed. This should be of the form [xmin, xmax, ymin, ymax].
Density estimation will be performed on G grid points evenly spaced
within this interval. Max and min grid points are placed half a grid
spacing in from the boundaries.
G: Number of grid points to use in each dimension. Total number of
gridpoints used in the calculation is G**2.
alpha: The smoothness parameter, specifying which derivative of the
filed is constrained by the prior. May be any integer >= 2.
num_samples: The number of density estimates to sample from the Bayesian
posterior. If zero is passed, no sample are drawn.
num_ts: The number of lengthscales at which to compute Q_\ell.
ti_shift: Initial integration t is set to
t_i = log[ (2*pi)**(2*alpha) * L**(2-2*alpha)] + ti_shift
(see Eq. 10)
tf_shift: Final integration t is set to
t_f = log[ (2*pi*G)**(2*alpha) * L**(2-2*alpha)] + tf_shift
(see Eq. 10)
verbose: If True, user feedback is provided
details: If True, calculation details are returned along with the
density estimate Q_star_func
Returns:
Q_star_func: A function, defined within bbox, providing a cubic spline
interpolation of the maximum a posteriori density estimate.
results: Returned only if `details' is set to be True. Contains detailed
information about the density estimation calculation.
'''
# Time execution
start_time = time.clock()
# Check data types
L = G
V = G**2
assert G==sp.floor(G)
assert len(xis_raw) > 1
assert num_samples >= 0 and num_samples==sp.floor(num_samples)
assert 2*alpha-2 >= 1
assert len(bbox) == 4
###
### Draw grid and histogram data
###
# Make sure xis is a numpy array
xis_raw = sp.array(xis_raw)
# If xdomain are specified, check them, and keep only data that falls within
xlb = bbox[0]
xub = bbox[1]
assert(xub-xlb > 0)
ylb = bbox[2]
yub = bbox[3]
assert(yub-ylb > 0)
# Throw away data not within bbox
indices = (xis_raw >= xlb) & (xis_raw <= xub) & (yis_raw >= ylb) & (yis_raw <= yub)
xis = xis_raw[indices]
yis = yis_raw[indices]
# Determine number of data points within bbox
N = len(xis)
# Compute edges for histogramming
xedges = sp.linspace(xlb,xub,L+1)
dx = xedges[1] - xedges[0]
yedges = sp.linspace(ylb,yub,L+1)
dy = yedges[1] - yedges[0]
# Compute grid for binning
xgrid = xedges[:-1]+dx/2
ygrid = yedges[:-1]+dy/2
# Convert to z = normalized x
xzis = (xis-xlb)/dx
yzis = (yis-ylb)/dx
xzedges = (xedges-xlb)/dx
yzedges = (yedges-ylb)/dy
# Histogram data
[H, xxx, yyy] = sp.histogram2d(xzis, yzis, [xzedges, yzedges])
R = H/N
R_flat = R.flatten()
###
### Use weak-field approximation to get phi_0
###
# Compute fourier transform of data
R_ks = fft2(R)
# Set constant component of data FT to 0
R_ks[0] = 0
# Set mode numbers seperately for even G and for odd G
if G%2 == 0: # If G is even
# ks = [0, 1, ... , G/2, -G/2+1, ..., -1]
ks = sp.concatenate((sp.arange(0, G/2 + 1), sp.arange(1 - G/2, 0)))
else: # If G is odd
# ks = [0, 1, ... , (G-1)/2, -(G-1)/2, ..., -1]
ks = sp.concatenate((sp.arange(0, (G-1)/2 + 1), sp.arange(-(G-1)/2, 0)))
# Mode numbers corresponding to fourier transform
A = sp.mat(sp.tile((2.0*sp.pi*ks/G)**2.0,[G,1]))
B = A.T
tau_ks = sp.log(V*sp.array(A+B)**alpha)
tau_ks[0,0] = 1 # This number is actually irrelevant since R_hat[0,0] = 0
# Set t_i to the \ell_i vale in Eq. 10 plus shift
t_i = sp.log(((2*sp.pi)**(2*alpha))*G**(2.0-2.0*alpha)) + ti_shift
# Set t_f to the \ell_f value in Eq. 10 plus shift
t_f = sp.log(((2*sp.pi*G)**(2*alpha))*G**(2.0-2.0*alpha)) + tf_shift
# Compute Fourier components of phi
phi_ks = -(G**2)*sp.array(R_ks)/sp.array((1.0 + sp.exp(tau_ks - t_i)))
# Invert the Fourier transform to get phi weak field approx
phi0 = sp.ravel(sp.real(ifft2(phi_ks)))
# Let user know integration interval
# if verbose:
# print 'Integrating from t_i == %f to t_f == %f'%(t_i, t_f)
# Set integrationt times
ts = sp.linspace(t_i,t_f,num_ts)
# Compute ells in terms of ts: exp(ts) = N/ell^(2 alpha - 2)
# Note: this is in dx = 1 units!
ells = (N/sp.exp(ts))**(1.0/(2.0*alpha-2.0))
###
### Integrate ODE
###
# Initialize phis
phis = sp.zeros([V,num_ts])
# Build 2D Laplacian matrix
delsq_2d = (-4.0*sp.eye(V) +
sp.eye(V,V,-1) + sp.eye(V,V,+1) +
sp.eye(V,V,-G) + sp.eye(V,V,+G) +
sp.eye(V,V,-V+1) + sp.eye(V,V,V-1) +
sp.eye(V,V,-V+G) + sp.eye(V,V,V-G))
# Make Delta, and make it sparse
Delta = csr_matrix(-delsq_2d)**alpha
# This is the key function: computes deriviate of phi for ODE integration
def this_flow(phi, t):
# Compute distribution Q corresponding to phi
Q = sp.exp(-phi)/V
# This matrix in invertible for all t > 0
A = Delta*sp.exp(-t) + diags(Q, 0)
# Solve A*phidt = Q-R
dphidt = spsolve(A, Q - R_flat)
# Return the time derivative of phi
return dphidt
# Integrate phi over specified ts
phis = odeint(this_flow, phi0, ts)
###
### Identify optimal lengthscale
###
# Initialize containers
log_evidence = sp.nan*sp.zeros(num_ts)
S = sp.zeros(num_ts)
log_det_Lambda = sp.zeros(num_ts)
Qs = sp.zeros([num_ts, G, G])
R_col = sp.mat(sp.ravel(R)).T
coeff = 1.0
# Compute log evidence for each t in ts
for i in range(num_ts):
# Get M corresonding to time ts[i]
t = ts[i]
# Get lengthscale correspondin to time ts[i]
ell = ells[i]
# Get field at time ts[i]
phi = phis[i,:]
# Renormalize phi just in case
Q = sp.exp(-phi)/sum(sp.exp(-phi))
phi = -sp.log(V*Q);
phi_col = sp.mat(phi).T;
Qs[i,:,:] = sp.reshape(Q,[L,L])
beta = ell**(2.0*alpha-2.0)
# Compute the value of the action of classical path at ts[i]
if all(sp.isfinite(phi_col)):
S[i] = 0.5*(phi_col.T*Delta*phi_col) + sp.exp(t)*(R_col.T*phi_col)[0,0] + sp.exp(t)
else:
S[i] = sp.inf
# Compute exact fluctuating determinant
# This requires dynamically fiddling with the overall scale of lambda
# to avoid infinities. Should find a more sensible way to do this
Lambda = Delta + sp.exp(t)*sp.diag(Q)
ok_value = False
while not ok_value:
f = sp.log(det(coeff*Lambda))
if f == -sp.inf:
coeff *= 1.5
#print '%d:+'%i
elif f == sp.inf:
coeff /= 1.5
#print '%d:-'%i
else:
ok_value = True
log_det_Lambda[i] = f - V*sp.log(coeff)
# If i = 0, i.e. largest value of \ell, compute the deteriminant ratio
# in weak field approximation. Use to compute log_det_nc_Delta
if i == 0:
fluct_wf = t + sum(sum(sp.log(1.0 + sp.exp(t - tau_ks))))
log_det_nc_Delta = log_det_Lambda[i] - fluct_wf
# Compute evidence for each ell with correct proportionality constant
log_evidence[i] = (N + 0.5*sp.sqrt(N) - N*sp.log(V)) - beta*S[i] - 0.5*sp.log(beta) - 0.5*(log_det_Lambda[i] - log_det_nc_Delta)
# Noramlize weights for different ells and save
ell_weights = sp.exp(log_evidence) - max(log_evidence)
ell_weights[ell_weights < -100] = 0.0
assert(not sum(ell_weights) == 0)
assert(all(sp.isfinite(ell_weights)))
ell_weights /= sum(ell_weights)
# Find the best lengthscale
i_star = sp.argmax(log_evidence)
phi_star = phis[i_star,:]
Q_star = sp.reshape(sp.exp(-phi_star)/sum(sp.exp(-phi_star)), [G, G])
ell_star = ells[i_star]
t_star = ts[i_star]
M_star = sp.exp(t_star)
###
### Sample from posterior (only if requirested)
###
# If user requests samples
if num_samples > 0:
Lambda_star = (ell_star**(2.0*alpha-120))*(Delta + M_star*sp.diag(Q_star))
eigvals, eigvecs = eig(Lambda_star)
# Lambda_star is Hermetian; shouldn't need to do this
eigvals = sp.real(eigvals)
eigvecs = sp.real(eigvecs)
# Initialize container variables
Qs_sampled = sp.zeros([num_samples, G, G])
phis_sampled = sp.zeros([num_samples, V])
is_sampled = sp.zeros([num_samples])
for j in range(num_samples):
# First choose a classical path based on
i = choice(num_ts, p=ell_weights)
phi_cl = phis[i,:]
is_sampled[j] = i
# Draw random amplitudes for all modes and compute dphi
etas = randn(L)
dphi = sp.ravel(sp.real(sp.mat(eigvecs)*sp.mat(etas/sp.sqrt(eigvals)).T))
# Record final sampled phi
phi = phi_cl + dphi
Qs_sampled[j,:,:] = sp.reshape(sp.exp(-phi)/sum(sp.exp(-phi)), [G,G])
###
### Return results (with everything in correct lenght units!)
###
results = Results()
results.G = G
results.ts = ts
results.N = N
results.alpha = alpha
results.num_samples = num_samples
results.xgrid = xgrid
results.bbox = bbox
results.dx = dx
results.dy = dy
# Everything with units of length gets multiplied by dx!!!
results.ells = [ells*dx, ells*dy]
results.L = [G*dx, G*dy]
results.V = V*dx*dy
results.h = [dx, dy]
results.Qs = Qs/(dx*dy)
results.phis = phis
results.log_evidence = log_evidence
# Comptue star results
results.phi_star = deepcopy(phi_star)
results.Q_star = deepcopy(Q_star)/(dx*dy)
results.i_star = i_star
results.ell_star = [ells[i_star]*dx, ells[i_star]*dy]
bbox = [xlb, xub, ylb, yub]
# Create interpolated Q_star. Man this is a pain in the ass!
extended_xgrid = sp.zeros(G+2)
extended_xgrid[1:-1] = xgrid
extended_xgrid[0] = xlb
extended_xgrid[-1] = xub
extended_ygrid = sp.zeros(L+2)
extended_ygrid[1:-1] = ygrid
extended_ygrid[0] = ylb
extended_ygrid[-1] = yub
extended_Q_star = sp.zeros([G+2, G+2])
extended_Q_star[1:-1,1:-1] = Q_star
# Get rows
row = 0.5*(Q_star[0,:] + Q_star[-1,:])
extended_Q_star[0,1:-1] = row
extended_Q_star[-1,1:-1] = row
# Get cols
col = 0.5*(Q_star[:,0] + Q_star[:,-1])
extended_Q_star[1:-1,0] = col
extended_Q_star[1:-1,-1] = col
# Get remaining corners, which share the same value
corner= 0.25*(row[0]+row[-1]+col[0]+col[-1])
extended_Q_star[0,0] = corner
extended_Q_star[0,-1] = corner
extended_Q_star[-1,0] = corner
extended_Q_star[-1,-1] = corner
# Finally, compute interpolated function
Q_star_func = RectBivariateSpline(extended_xgrid, extended_ygrid, extended_Q_star/(dx*dy), bbox=bbox)
# If samples are requested, return those too
if num_samples > 0:
results.phis_sampled = phis_sampled
results.Qs_sampled = Qs_sampled/(dx*dy)
results.is_sampled = is_sampled
# Stop time
time_elapsed = time.clock() - start_time
if verbose:
print 'deft_2d: %1.2f sec for alpha = %d, G = %d, N = %d'%(time_elapsed, alpha, G, N)
if details:
return Q_star_func, results
else:
return Q_star_func
# Compute source plane positions
xs = x.copy()
ys = y.copy()
plt.ion()
plt.hold(False)
for i in xrange(0, num_stars):
rsq = (x - x_stars[i])**2 + (y - y_stars[i])**2
xs -= mass*(x - x_stars[i])/(np.pi*rsq)
ys -= mass*(y - y_stars[i])/(np.pi*rsq)
print('{i}/{n}'.format(i=(i+1), n=num_stars))
if (i+1)%50 == 0:
counts = scipy.histogram2d(xs.flatten(), ys.flatten(), bins=250,\
range=[[x_min, x_max], [y_min, y_max]])
plt.imshow(counts[0])
plt.title('{a}/{b}'.format(a=(i+1), b=num_stars))
plt.draw()
rays = np.empty((xs.size, 2))
rays[:,0] = xs.flatten()
rays[:,1] = ys.flatten()
# Save the rays
import cPickle as pickle
print('Saving...')
pickle.dump(rays, open('rays.pickle', 'wb'))
print('Done.')
plt.ioff()
def transition_matrix(y, nstates, dt=1):
"""
Return transition matrix histogram (i.e., counts)
"""
tr, tmp, tmp = scipy.histogram2d(y[:-dt], y[dt:], bins=range(nstates+1))
return tr
def plot_scatter(
dataframe, x_series, y_series,
output_name = 'scatter',
output_directory = None,
output_format = None,
verbose = True,
dropna = True,
density_plot = False,
plot_title = None,
fig_dpi = 300,
fig_width = None,
fig_height = None,
fig_grid = True,
axis_label_size = 12.0,
):
if not output_directory:
output_directory = tempfile.mkdtemp( prefix = '%s-%s-plots_' % (time.strftime("%y%m%d"), getpass.getuser()) )
fig, ax = plt.subplots()
ax.grid(fig_grid)
if dropna:
dataframe = dataframe[[x_series, y_series]].replace([np.inf, -np.inf], np.nan).dropna()
if not output_format:
# If there are many points, save figure as a PNG (since PDFs perform poorly with many points)
if max( len(dataframe.as_matrix([x_series])), len(dataframe.as_matrix([y_series])) ) >= 1500:
output_format = 'png'
else:
output_format = 'pdf'
output_path = os.path.join(output_directory, output_name + '.' + output_format)
if density_plot:
xdat = dataframe.as_matrix([x_series]).flatten()
ydat = dataframe.as_matrix([y_series]).flatten()
#histogram definition
xyrange = [ [np.min(xdat), np.max(xdat)], [np.min(ydat), np.max(ydat)] ] # data range
bins = [100, 100] # number of bins
thresh = 3 #density threshold
# histogram the data
hh, locx, locy = scipy.histogram2d(xdat, ydat, range=xyrange, bins=bins)
posx = np.digitize(xdat, locx)
posy = np.digitize(ydat, locy)
#select points within the histogram
ind = (posx > 0) & (posx <= bins[0]) & (posy > 0) & (posy <= bins[1])
hhsub = hh[posx[ind] - 1, posy[ind] - 1] # values of the histogram where the points are
xdat_low = xdat[ind][hhsub < thresh] # low density points
ydat_low = ydat[ind][hhsub < thresh]
xdat_high = xdat[ind][hhsub >= thresh] # low density points
ydat_high = ydat[ind][hhsub >= thresh]
hh[hh < thresh] = np.nan # fill the areas with low density by NaNs
plt.scatter(xdat_low, ydat_low, s = 10, alpha = 0.6, linewidth = 0.1)
plt.scatter(xdat_high, ydat_high, s = 0.6, alpha = 0.15, linewidth = 0.1, color='white')
plt.imshow(np.flipud(hh.T),cmap='jet',extent=np.array(xyrange).flatten(), interpolation='none')
plt.colorbar(label = 'Counts per (high point density) histogram region')
else:
plt.scatter(dataframe[[x_series]], dataframe[[y_series]], s = 10, alpha = 0.6)
plt.ylabel( make_latex_safe(y_series), fontsize = axis_label_size )
plt.xlabel( make_latex_safe(x_series), fontsize = axis_label_size )
if plot_title:
plt.title( make_latex_safe(plot_title) )
if verbose:
print 'Saving scatterplot figure to:', output_path
if fig_height and fig_width:
plt.gcf().set_size_inches(fig_width, fig_height)
plt.savefig(
output_path, dpi = fig_dpi, format = output_format
)
plt.close()
return output_path
def plot():
cs = palettable.colorbrewer.qualitative.Set1_3.mpl_colors
# Load parameter file
with open('/home/lc585/qsosed/input.yml', 'r') as f:
parfile = yaml.load(f)
ebvmin = -0.075
ebvmax = 0.075
zmin = 1
zmax = 3.0
# Load stuff
fittingobj = load(parfile)
wavlen = fittingobj.get_wavlen()
# Load data
df = get_data()
df = df[(df.z_HW >= zmin) & (df.z_HW <= zmax)]
colmin, colmax = [], []
zs = np.arange(zmin, zmax + 0.025, 0.025)
mycm = cm.get_cmap('Blues_r')
mycm.set_under('w')
cset = brewer2mpl.get_map('Blues', 'sequential', 9).mpl_colors
mycm = truncate_colormap(mycm, 0.0, 0.8)
fig = plt.figure(figsize=(figsize(1, vscale=0.8)))
ax = fig.add_subplot(1, 1, 1)
plt.tight_layout()
#histogram definition
xyrange = [[0., 3.0], [0, 3]] # data range
bins = [100, 60] # number of bins
thresh = 7 #density threshold
#data definition
xdat, ydat = df.z_HW, df.iVEGA - df.KVEGA
# histogram the data
hh, locx, locy = histogram2d(xdat, ydat, range=xyrange, bins=bins)
posx = np.digitize(xdat, locx)
posy = np.digitize(ydat, locy)
#select points within the histogram
ind = (posx > 0) & (posx <= bins[0]) & (posy > 0) & (posy <= bins[1])
hhsub = hh[posx[ind] - 1,
posy[ind] - 1] # values of the histogram where the points are
xdat1 = xdat[ind][hhsub < thresh] # low density points
ydat1 = ydat[ind][hhsub < thresh]
hh[hh < thresh] = np.nan # fill the areas with low density by NaNs
im = ax.imshow(np.flipud(hh.T),
cmap=mycm,
extent=np.array(xyrange).flatten(),
interpolation='none',
aspect='auto',
vmin=thresh,
vmax=45)
clb = fig.colorbar(im)
clb.set_label('Number of Objects')
ax.scatter(xdat1, ydat1, color=cset[-1], s=2)
plt.ylim(0, 3.5)
col = []
for z in zs:
parfile['ext']['EBV'] = ebvmax
magtmp, wavlentmp, fluxtmp = model(redshift=z, parfile=parfile)
col.append(magtmp[3] - magtmp[8])
plt.plot(zs, col, label='E(B-V) = 0.075', color=cs[0], linewidth=1.0)
upper_bound_1 = col[np.argmin(np.abs(zs - 1.0)):np.argmin(np.abs(zs -
1.5))]
upper_bound_2 = col[np.argmin(np.abs(zs - 2.0)):np.argmin(np.abs(zs -
2.7))]
col = []
for z in zs:
parfile['ext']['EBV'] = ebvmin
magtmp, wavlentmp, fluxtmp = model(redshift=z, parfile=parfile)
col.append(magtmp[3] - magtmp[8])
plt.plot(zs, col, label='E(B-V) = -0.075', color=cs[0], linewidth=1.0)
lower_bound_1 = col[np.argmin(np.abs(zs - 1.0)):np.argmin(np.abs(zs -
1.5))]
lower_bound_2 = col[np.argmin(np.abs(zs - 2.0)):np.argmin(np.abs(zs -
2.7))]
plt.fill_between(zs[np.argmin(np.abs(zs - 2.0)):np.argmin(np.abs(zs -
2.7))],
lower_bound_2,
upper_bound_2,
facecolor='None',
edgecolor=cs[0],
linewidth=3.0)
col = []
for z in zs:
parfile['ext']['EBV'] = 0.0
magtmp, wavlentmp, fluxtmp = model(redshift=z, parfile=parfile)
col.append(magtmp[3] - magtmp[8])
plt.plot(zs, col, label='E(B-V) = 0.0', color=cs[0], linewidth=1.0)
plt.xlim(0.75, 3.5)
plt.text(3.33821,
2.5,
'E(B-V)=',
horizontalalignment='right',
verticalalignment='center',
color='black')
plt.text(3.09608,
2.2,
'0.075',
horizontalalignment='left',
verticalalignment='center',
color='black')
plt.text(3.09608,
1.2,
'-0.075',
horizontalalignment='left',
verticalalignment='center',
color='black')
plt.text(3.09608,
1.7,
'0.0',
horizontalalignment='left',
verticalalignment='center',
color='black')
plt.xlabel(r'Redshift $z$')
plt.ylabel(r'$i$-$K$')
plt.tight_layout()
fig.savefig('/home/lc585/thesis/figures/chapter05/ik_versus_z_low_ext.pdf')
plt.show()
return None
#update flies
print('t: {0:1.2f}'.format(t))
#update the swarm
# swarm.update(t,dt,importedWind,importedConc,traps,pre_stored=True) #for presaved conc
swarm.update(t,
dt,
importedWind,
importedPlumes,
traps,
pre_stored=True) #for presaved plumes
t += dt
time.sleep(0.001)
#Save density to logger
grid_hist, _, _ = scipy.histogram2d( # Compute location bin membership using 2d hist function
swarm.x_position,
swarm.y_position,
bins=[density_logger_bin_edges, density_logger_bin_edges])
imd.set_data(grid_hist)
data = {'grid_hist': grid_hist}
timecourse_logger.add(data)
# Update live display
'''plot the flies'''
#plot the wind vector field
if plotting:
u, v = importedWind.quiver_at_time(t)
u,v = u[0:full_size-1:shrink_factor,0:full_size-1:shrink_factor],\
v[0:full_size-1:shrink_factor,0:full_size-1:shrink_factor]
fly_dots.set_offsets(scipy.c_[swarm.x_position, swarm.y_position])
def main(argv):
datafiles = [
"../test/mg5pythia8_hp200.root.test3.csv",
"../test/mg5pythia8_hp300.root.test.csv",
"../test/mg5pythia8_hp400.root.test.csv",
]
frames = []
for df in datafiles:
frames.append(getData(df))
data = pd.concat(frames)
predictors = sp.array([
"et(met)", "phi(met)",
#"nbjet", "njet",
"pt(reco tau1)", "eta(reco tau1)", "phi(reco tau1)", "m(reco tau1)",
"pt(reco bjet1)", "eta(reco bjet1)", "phi(reco bjet1)", "m(reco bjet1)",
"pt(reco jet1)", "eta(reco jet1)", "phi(reco jet1)", "m(reco jet1)",
"pt(reco jet2)", "eta(reco jet2)", "phi(reco jet2)", "m(reco jet2)",
], dtype=str)
target = "pt(mc nuH)"
plotdir = "correlations"
mkdir(plotdir)
nplots = len(predictors)
w = sp.ceil(sp.sqrt(nplots))
h = sp.floor(sp.sqrt(nplots))
nplots_total = nplots*len(frames)
iplot = 0
for j, frame in enumerate(frames):
x = sp.array(frame[target], dtype=float)
xmin = x.min()
xmax = x.max()
xedges = sp.linspace(xmin, xmax, 100)
masspoint = datafiles[j].split('_')[1].split('.')[0].replace('hp', '')
for i, pred in enumerate(predictors):
y = sp.array(frame[pred], dtype=float)
ymin = y.min()
ymax = y.max()
yedges = sp.linspace(ymin, ymax, 100)
r, p = pearsonr(x, y)
fig = plt.figure()
ax = fig.add_subplot(111)
Z, xedges, yedges = sp.histogram2d(x, y, bins=100)
X, Y = sp.meshgrid(xedges, yedges, indexing='ij')
ax.pcolormesh(X, Y, Z, norm=LogNorm(vmin=1e-4, vmax=Z.max()))
ax.set_xlabel(target)
ax.set_ylabel(pred)
ax.set_title("H+ {} GeV, rho = {:.2f}%, p = {:.2f}%".format(masspoint, 100*r, 100*p))
ax.autoscale_view(tight=True)
fig.tight_layout()
plotname = "hp{}_{}".format(masspoint, pred.replace('(', '_').replace(')', ''))
plt.savefig("{}/{}.png".format(plotdir, plotname))
plt.savefig("{}/{}.pdf".format(plotdir, plotname))
plt.close(fig)
iplot += 1
print("--- {} / {}".format(iplot, nplots_total))
def plot():
tab = Table.read('/data/lc585/QSOSED/Results/141203/sample1/out_add.fits')
tab = tab[tab['BBT_STDERR'] < 200.0]
tab = tab[tab['BBFLXNRM_STDERR'] < 0.05]
tab = tab[tab['CHI2_RED'] < 3.0]
tab = tab[tab['LUM_IR'] > 40.0] # just gets rid of one annoying point
mycm = cm.get_cmap('YlOrRd_r')
mycm.set_under('w')
cset = brewer2mpl.get_map('YlOrRd', 'sequential', 9).mpl_colors
mycm = truncate_colormap(mycm, 0.0, 0.8)
set_plot_properties() # change style
fig = plt.figure(figsize=figsize(1.5, 0.7))
gs = gridspec.GridSpec(9, 13)
ax1 = fig.add_subplot(gs[1:5, 0:4])
ax2 = fig.add_subplot(gs[5:, 0:4])
ax3 = fig.add_subplot(gs[0, 0:4])
ax4 = fig.add_subplot(gs[1:5, 4:8])
ax5 = fig.add_subplot(gs[5:, 4:8])
ax6 = fig.add_subplot(gs[0, 4:8])
ax7 = fig.add_subplot(gs[1:5, 8:12])
ax8 = fig.add_subplot(gs[5:, 8:12])
ax9 = fig.add_subplot(gs[0, 8:12])
ax10 = fig.add_subplot(gs[1:5, 12])
ax11 = fig.add_subplot(gs[5:, 12])
fig.subplots_adjust(wspace=0.0)
fig.subplots_adjust(hspace=0.0)
#histogram definition
xyrange = [[44, 48], [0, 2]] # data range
bins = [80, 40] # number of bins
thresh = 4 #density threshold
#data definition
xdat, ydat = tab['LUM_UV'], tab['RATIO_IR_UV']
# histogram the data
hh, locx, locy = histogram2d(xdat, ydat, range=xyrange, bins=bins)
posx = np.digitize(xdat, locx)
posy = np.digitize(ydat, locy)
#select points within the histogram
ind = (posx > 0) & (posx <= bins[0]) & (posy > 0) & (posy <= bins[1])
hhsub = hh[posx[ind] - 1,
posy[ind] - 1] # values of the histogram where the points are
xdat1 = xdat[ind][hhsub < thresh] # low density points
ydat1 = ydat[ind][hhsub < thresh]
hh[hh < thresh] = np.nan # fill the areas with low density by NaNs
im = ax1.imshow(np.flipud(hh.T),
cmap=mycm,
extent=np.array(xyrange).flatten(),
interpolation='none',
aspect='auto',
vmin=thresh,
vmax=45)
ax1.scatter(xdat1, ydat1, color=cset[-1])
ax1.set_ylabel(r'$R_{{\rm NIR}/{\rm UV}}$', fontsize=14)
ax1.set_ylim(0., 2.)
ax1.set_xlim(45.25, 47)
ax3.hist(tab['LUM_UV'], color=cset[-1], bins=20)
ax3.set_xlim(ax1.get_xlim())
ax3.set_axis_off()
#histogram definition
xyrange = [[44, 48], [200, 2000]] # data range
bins = [90, 35] # number of bins
thresh = 4 #density threshold
#data definition
xdat, ydat = tab['LUM_UV'], tab['BBT']
# histogram the data
hh, locx, locy = histogram2d(xdat, ydat, range=xyrange, bins=bins)
posx = np.digitize(xdat, locx)
posy = np.digitize(ydat, locy)
#select points within the histogram
ind = (posx > 0) & (posx <= bins[0]) & (posy > 0) & (posy <= bins[1])
hhsub = hh[posx[ind] - 1,
posy[ind] - 1] # values of the histogram where the points are
xdat1 = xdat[ind][hhsub < thresh] # low density points
ydat1 = ydat[ind][hhsub < thresh]
hh[hh < thresh] = np.nan # fill the areas with low density by NaNs
im = ax2.imshow(np.flipud(hh.T),
cmap=mycm,
extent=np.array(xyrange).flatten(),
interpolation='none',
aspect='auto',
vmin=thresh,
vmax=45)
ax2.scatter(xdat1, ydat1, color=cset[-1])
ax2.set_xlabel(r'Log$_{10} (L_{\rm UV} ({\rm erg/s}))$', fontsize=14)
ax2.set_ylabel(r'$T_{\mathrm{BB}}$', fontsize=14)
ax2.set_xlim(ax1.get_xlim())
ax2.set_ylim(500, 2100)
plt.tick_params(axis='both', which='major', labelsize=10)
ax1.get_xaxis().set_ticks([])
ax2.get_yaxis().set_ticks([600, 800, 1000, 1200, 1400, 1600, 1800, 2000])
ax2.get_xaxis().set_ticks([45.4, 45.6, 45.8, 46.0, 46.2, 46.4, 46.6, 46.8])
#############################################################################
#histogram definition
xyrange = [[7.5, 10.5], [0, 2]] # data range
bins = [30, 30] # number of bins
thresh = 4 #density threshold
#data definition
xdat, ydat = tab['LOGBH'], tab['RATIO_IR_UV']
# histogram the data
hh, locx, locy = histogram2d(xdat, ydat, range=xyrange, bins=bins)
posx = np.digitize(xdat, locx)
posy = np.digitize(ydat, locy)
#select points within the histogram
ind = (posx > 0) & (posx <= bins[0]) & (posy > 0) & (posy <= bins[1])
hhsub = hh[posx[ind] - 1,
posy[ind] - 1] # values of the histogram where the points are
xdat1 = xdat[ind][hhsub < thresh] # low density points
ydat1 = ydat[ind][hhsub < thresh]
hh[hh < thresh] = np.nan # fill the areas with low density by NaNs
im = ax4.imshow(np.flipud(hh.T),
cmap=mycm,
extent=np.array(xyrange).flatten(),
interpolation='none',
aspect='auto',
vmin=thresh,
vmax=45)
ax4.scatter(xdat1, ydat1, color=cset[-1])
ax4.set_ylim(ax1.get_ylim())
ax4.set_xlim(7.5, 10.5)
ax4.set_xticklabels([])
ax4.set_yticklabels([])
ax6.hist(tab[tab['LOGBH'] > 6]['LOGBH'], color=cset[-1], bins=20)
ax6.set_xlim(ax4.get_xlim())
ax6.set_axis_off()
#histogram definition
xyrange = [[7.5, 10.5], [500, 2000]] # data range
bins = [50, 30] # number of bins
thresh = 4 #density threshold
#data definition
xdat, ydat = tab['LOGBH'], tab['BBT']
# histogram the data
hh, locx, locy = histogram2d(xdat, ydat, range=xyrange, bins=bins)
posx = np.digitize(xdat, locx)
posy = np.digitize(ydat, locy)
#select points within the histogram
ind = (posx > 0) & (posx <= bins[0]) & (posy > 0) & (posy <= bins[1])
hhsub = hh[posx[ind] - 1,
posy[ind] - 1] # values of the histogram where the points are
xdat1 = xdat[ind][hhsub < thresh] # low density points
ydat1 = ydat[ind][hhsub < thresh]
hh[hh < thresh] = np.nan # fill the areas with low density by NaNs
im = ax5.imshow(np.flipud(hh.T),
cmap=mycm,
extent=np.array(xyrange).flatten(),
interpolation='none',
aspect='auto',
vmin=thresh,
vmax=45)
ax5.scatter(xdat1, ydat1, color=cset[-1])
ax5.set_xlabel(r'Log$_{10}$ (Black Hole Mass $M_{\rm BH}$)')
ax5.set_yticklabels([])
# ax5.hist(tab['BBT'], orientation='horizontal',color=cset[-1],bins=20)
# ax5.set_ylim(ax2.get_ylim())
# ax5.set_axis_off()
plt.tick_params(axis='both', which='major')
ax4.set_ylim(ax1.get_ylim())
ax5.set_ylim(ax2.get_ylim())
ax5.set_xlim(ax4.get_xlim())
ax6.hist(tab['LOGBH'][tab['LOGBH'] > 6], color=cset[-1], bins=20)
ax6.set_xlim(ax5.get_xlim())
ax6.set_axis_off()
######################################################################################
#histogram definition
xyrange = [[-2, 0.5], [0, 2]] # data range
bins = [30, 30] # number of bins
thresh = 4 #density threshold
#data definition
xdat, ydat = tab['LOGEDD_RATIO'], tab['RATIO_IR_UV']
# histogram the data
hh, locx, locy = histogram2d(xdat, ydat, range=xyrange, bins=bins)
posx = np.digitize(xdat, locx)
posy = np.digitize(ydat, locy)
#select points within the histogram
ind = (posx > 0) & (posx <= bins[0]) & (posy > 0) & (posy <= bins[1])
hhsub = hh[posx[ind] - 1,
posy[ind] - 1] # values of the histogram where the points are
xdat1 = xdat[ind][hhsub < thresh] # low density points
ydat1 = ydat[ind][hhsub < thresh]
hh[hh < thresh] = np.nan # fill the areas with low density by NaNs
im = ax7.imshow(np.flipud(hh.T),
cmap=mycm,
extent=np.array(xyrange).flatten(),
interpolation='none',
aspect='auto',
vmin=thresh,
vmax=45)
ax7.scatter(xdat1, ydat1, color=cset[-1])
ax7.set_ylim(ax1.get_ylim())
ax7.set_xlim(-2, 0.5)
ax9.hist(tab[tab['LOGEDD_RATIO'] > -2.5]['LOGEDD_RATIO'],
color=cset[-1],
bins=20)
ax9.set_xlim(ax7.get_xlim())
ax9.set_axis_off()
#histogram definition
xyrange = [[-2, 0.5], [500, 2000]] # data range
bins = [50, 30] # number of bins
thresh = 4 #density threshold
#data definition
xdat, ydat = tab['LOGEDD_RATIO'], tab['BBT']
# histogram the data
hh, locx, locy = histogram2d(xdat, ydat, range=xyrange, bins=bins)
posx = np.digitize(xdat, locx)
posy = np.digitize(ydat, locy)
#select points within the histogram
ind = (posx > 0) & (posx <= bins[0]) & (posy > 0) & (posy <= bins[1])
hhsub = hh[posx[ind] - 1,
posy[ind] - 1] # values of the histogram where the points are
xdat1 = xdat[ind][hhsub < thresh] # low density points
ydat1 = ydat[ind][hhsub < thresh]
hh[hh < thresh] = np.nan # fill the areas with low density by NaNs
im = ax8.imshow(np.flipud(hh.T),
cmap=mycm,
extent=np.array(xyrange).flatten(),
interpolation='none',
aspect='auto',
vmin=thresh,
vmax=45)
ax8.scatter(xdat1, ydat1, color=cset[-1])
axcb = fig.add_axes([0.33, 0.05, 0.33, 0.02])
clb = fig.colorbar(im, cax=axcb, orientation='horizontal')
clb.set_label('Number of Objects')
ax8.set_xlabel(r'Log$_{10}$ (Eddington Ratio $\lambda$)')
ax8.set_ylim(ax2.get_ylim())
ax8.set_xlim(ax7.get_xlim())
plt.tick_params(axis='both', which='major')
ax8.set_yticklabels([])
ax8.set_xticklabels(['', '-1.5', '-1.0', '-0.5', '0.0', '0.5'])
ax7.set_yticklabels([])
ax7.set_xticklabels([])
ax9.hist(tab['LOGBH'][tab['LOGBH'] > 6], color=cset[-1], bins=20)
ax9.set_xlim(ax8.get_xlim())
ax9.set_axis_off()
ax10.hist(tab['RATIO_IR_UV'],
orientation='horizontal',
color=cset[-1],
bins=np.arange(0, 2, 0.1))
ax10.set_ylim(ax1.get_ylim())
ax10.set_axis_off()
ax11.hist(tab['BBT'], orientation='horizontal', color=cset[-1], bins=20)
ax11.set_ylim(ax2.get_ylim())
ax11.set_axis_off()
s1 = spearmanr(tab[tab['LOGEDD_RATIO'] > -2.5]['LOGEDD_RATIO'],
tab[tab['LOGEDD_RATIO'] > -2.5]['RATIO_IR_UV'])[0]
s2 = spearmanr(tab[tab['LOGBH'] > 6]['LOGBH'],
tab[tab['LOGBH'] > 6]['RATIO_IR_UV'])[0]
s3 = spearmanr(tab['LUM_UV'], tab['RATIO_IR_UV'])[0]
s4 = spearmanr(tab[tab['LOGEDD_RATIO'] > -2.5]['LOGEDD_RATIO'],
tab[tab['LOGEDD_RATIO'] > -2.5]['BBT'])[0]
s5 = spearmanr(tab[tab['LOGBH'] > 6]['LOGBH'],
tab[tab['LOGBH'] > 6]['BBT'])[0]
s6 = spearmanr(tab['LUM_UV'], tab['BBT'])[0]
ax1.text(46.5, 0.2, r'$\rho =$ {0:.2f}'.format(s3))
ax4.text(9.7, 0.2, r'$\rho =$ {0:.2f}'.format(s2))
ax7.text(-0.1, 0.2, r'$\rho =$ {0:.2f}'.format(s1))
ax2.text(46.5, 1900, r'$\rho =$ {0:.2f}'.format(s6))
ax5.text(9.7, 1900, r'$\rho =$ {0:.2f}'.format(s5))
ax8.text(-0.1, 1900, r'$\rho =$ {0:.2f}'.format(s4))
fig.savefig('/home/lc585/thesis/figures/chapter06/correlations.pdf')
plt.show()
return None
def plot_params(args):
"""Plot alpha, theta, and the emission probabilities"""
old_err = sp.seterr(under='ignore')
oldsize = matplotlib.rcParams['font.size']
K, L = args.emit_probs.shape if not args.continuous_observations else args.means.shape
# alpha
#matplotlib.rcParams['font.size'] = 12
pyplot.figure()
_, xedges, yedges = sp.histogram2d([0, K], [0, K], bins=[K, K])
extent = [xedges[0], xedges[-1], yedges[0], yedges[-1]]
pyplot.imshow(args.alpha.astype(sp.float64),
extent=extent,
interpolation='nearest',
vmin=0,
vmax=1,
cmap='OrRd',
origin='lower')
pyplot.xticks(sp.arange(K) + .5, sp.arange(K) + 1)
pyplot.gca().set_xticks(sp.arange(K) + 1, minor=True)
pyplot.yticks(sp.arange(K) + .5, sp.arange(K) + 1)
pyplot.gca().set_yticks(sp.arange(K) + 1, minor=True)
pyplot.grid(which='minor', alpha=.2)
for line in pyplot.gca().yaxis.get_ticklines() + pyplot.gca(
).xaxis.get_ticklines() + pyplot.gca().xaxis.get_ticklines(
minor=True) + pyplot.gca().yaxis.get_ticklines(minor=True):
# label is a Text instance
line.set_markersize(0)
pyplot.ylabel('Horizontal parent state')
pyplot.xlabel('Node state')
pyplot.title(
r"Top root transition ($\alpha$) for {approx} iteration {iteration}".
format(approx=args.approx, iteration=args.iteration))
b = pyplot.colorbar(shrink=.9)
b.set_label("Probability")
outfile = (args.out_params + '_it{iteration}.png').format(param='alpha',
**args.__dict__)
pyplot.savefig(os.path.join(args.out_dir, outfile), dpi=240)
# beta
pyplot.figure()
_, xedges, yedges = sp.histogram2d([0, K], [0, K], bins=[K, K])
extent = [xedges[0], xedges[-1], yedges[0], yedges[-1]]
pyplot.clf()
pyplot.imshow(args.beta.astype(sp.float64),
extent=extent,
interpolation='nearest',
vmin=0,
vmax=1,
cmap='OrRd',
origin='lower')
pyplot.xticks(sp.arange(K) + .5, sp.arange(K) + 1)
pyplot.gca().set_xticks(sp.arange(K) + 1, minor=True)
pyplot.yticks(sp.arange(K) + .5, sp.arange(K) + 1)
pyplot.gca().set_yticks(sp.arange(K) + 1, minor=True)
pyplot.grid(which='minor', alpha=.2)
for line in pyplot.gca().yaxis.get_ticklines() + pyplot.gca(
).xaxis.get_ticklines() + pyplot.gca().xaxis.get_ticklines(
minor=True) + pyplot.gca().yaxis.get_ticklines(minor=True):
# label is a Text instance
line.set_markersize(0)
pyplot.ylabel('Vertical parent state')
pyplot.xlabel('Node state')
pyplot.title(
r"Left root transition ($\beta$) for {approx} iteration {iteration}".
format(approx=args.approx, iteration=args.iteration))
b = pyplot.colorbar(shrink=.9)
b.set_label("Probability")
outfile = (args.out_params + '_it{iteration}.png').format(param='beta',
**args.__dict__)
pyplot.savefig(os.path.join(args.out_dir, outfile), dpi=240)
# theta
if args.separate_theta:
theta_tmp = args.theta
for i in range((args.theta.shape)[0]):
setattr(args, 'theta_%s' % (i + 1), args.theta[i, :, :, :])
for theta_name in ['theta'] + ['theta_%s' % i for i in range(20)]:
#print 'trying', theta_name
if not hasattr(args, theta_name):
#print 'missing', theta_name
continue
_, xedges, yedges = sp.histogram2d([0, K], [0, K], bins=[K, K])
extent = [xedges[0], xedges[-1], yedges[0], yedges[-1]]
if K == 18:
numx_plots = 6
numy_plots = 3
elif K == 15:
numx_plots = 5
numy_plots = 3
else:
numx_plots = int(ceil(sp.sqrt(K)))
numy_plots = int(ceil(sp.sqrt(K)))
matplotlib.rcParams['font.size'] = 8
fig, axs = pyplot.subplots(numy_plots,
numx_plots,
sharex=True,
sharey=True,
figsize=(numx_plots * 2.5,
numy_plots * 2.5))
for k in xrange(K):
pltx, plty = k // numx_plots, k % numx_plots
#axs[pltx,plty].imshow(args.theta[k,:,:], extent=extent, interpolation='nearest',
axs[pltx,
plty].imshow(getattr(args,
theta_name)[:, k, :].astype(sp.float64),
extent=extent,
interpolation='nearest',
vmin=0,
vmax=1,
cmap='OrRd',
aspect='auto',
origin='lower')
#if k < numx_plots:
#axs[pltx,plty].text(0 + .5, K - .5, 'vp=%s' % (k+1), horizontalalignment='left', verticalalignment='top', fontsize=10)
axs[pltx, plty].text(0 + .5,
K - .5,
'hp=%s' % (k + 1),
horizontalalignment='left',
verticalalignment='top',
fontsize=10)
#axs[pltx,plty].xticks(sp.arange(K) + .5, sp.arange(K))
#axs[pltx,plty].yticks(sp.arange(K) + .5, sp.arange(K))
axs[pltx, plty].set_xticks(sp.arange(K) + .5)
axs[pltx, plty].set_xticks(sp.arange(K) + 1, minor=True)
axs[pltx, plty].set_xticklabels(sp.arange(K) + 1)
axs[pltx, plty].set_yticks(sp.arange(K) + .5)
axs[pltx, plty].set_yticks(sp.arange(K) + 1, minor=True)
axs[pltx, plty].set_yticklabels(sp.arange(K) + 1)
for line in axs[pltx, plty].yaxis.get_ticklines() + axs[
pltx, plty].xaxis.get_ticklines() + axs[
pltx, plty].yaxis.get_ticklines(
minor=True) + axs[pltx, plty].xaxis.get_ticklines(
minor=True):
line.set_markersize(0)
axs[pltx, plty].grid(True, which='minor', alpha=.2)
#fig.suptitle(r"$\Theta$ with fixed parents for {approx} iteration {iteration}".
# format(approx=args.approx, iteration=args.iteration),
# fontsize=14, verticalalignment='top')
fig.suptitle('Node state',
y=.03,
fontsize=14,
verticalalignment='center')
#fig.suptitle('Horizontal parent state', y=.5, x=.02, rotation=90,
fig.suptitle('Vertical parent state',
y=.5,
x=.02,
rotation=90,
verticalalignment='center',
fontsize=14)
matplotlib.rcParams['font.size'] = 6.5
fig.subplots_adjust(wspace=.05, hspace=.05, left=.05, right=.95)
#b = fig.colorbar(shrink=.9)
#b.set_label("Probability")
outfile = (args.out_params + '_vertparent_it{iteration}.png').format(
param=theta_name, **args.__dict__)
pyplot.savefig(os.path.join(args.out_dir, outfile), dpi=240)
fig, axs = pyplot.subplots(numy_plots,
numx_plots,
sharex=True,
sharey=True,
figsize=(numx_plots * 2.5,
numy_plots * 2.5))
for k in xrange(K):
pltx, plty = k // numx_plots, k % numx_plots
axs[pltx, plty].imshow(
getattr(args, theta_name)[k, :, :].astype(sp.float64),
extent=extent,
interpolation='nearest',
#axs[pltx,plty].imshow(args.theta[:,k,:], extent=extent, interpolation='nearest',
vmin=0,
vmax=1,
cmap='OrRd',
aspect='auto',
origin='lower')
#if k < numx_plots:
axs[pltx, plty].text(0 + .5,
K - .5,
'vp=%s' % (k + 1),
horizontalalignment='left',
verticalalignment='top',
fontsize=10)
#axs[pltx,plty].xticks(sp.arange(K) + .5, sp.arange(K))
#axs[pltx,plty].yticks(sp.arange(K) + .5, sp.arange(K))
axs[pltx, plty].set_xticks(sp.arange(K) + .5)
axs[pltx, plty].set_xticks(sp.arange(K) + 1, minor=True)
axs[pltx, plty].set_xticklabels(sp.arange(K) + 1)
axs[pltx, plty].set_yticks(sp.arange(K) + .5)
axs[pltx, plty].set_yticks(sp.arange(K) + 1, minor=True)
axs[pltx, plty].set_yticklabels(sp.arange(K) + 1)
for line in axs[pltx, plty].yaxis.get_ticklines() + axs[
pltx, plty].xaxis.get_ticklines() + axs[
pltx, plty].yaxis.get_ticklines(
minor=True) + axs[pltx, plty].xaxis.get_ticklines(
minor=True):
line.set_markersize(0)
axs[pltx, plty].grid(True, which='minor', alpha=.2)
#fig.suptitle(r"$\Theta$ with fixed parents for {approx} iteration {iteration}".
# format(approx=args.approx, iteration=args.iteration),
# fontsize=14, verticalalignment='top')
fig.suptitle('Node state',
y=.03,
fontsize=14,
verticalalignment='center')
fig.suptitle(
'Horizontal parent state',
y=.5,
x=.02,
rotation=90,
#fig.suptitle('Vertical parent state', y=.5, x=.02, rotation=90,
verticalalignment='center',
fontsize=14)
matplotlib.rcParams['font.size'] = 6.5
fig.subplots_adjust(wspace=.05, hspace=.05, left=.05, right=.95)
#b = fig.colorbar(shrink=.9)
#b.set_label("Probability")
outfile = (args.out_params + '_it{iteration}.png').format(
param=theta_name, **args.__dict__)
pyplot.savefig(os.path.join(args.out_dir, outfile), dpi=240)
# emission probabilities
if args.continuous_observations:
# plot mean values
matplotlib.rcParams['font.size'] = 8
pyplot.figure(figsize=(max(1, round(L / 3.)), max(1, round(K / 3.))))
print(max(1, round(L / 3.)), max(1, round(K / 3.)))
pyplot.imshow(args.means.astype(sp.float64),
interpolation='nearest',
aspect='auto',
vmin=0,
vmax=args.means.max(),
cmap='OrRd',
origin='lower')
for k in range(K):
for l in range(L):
pyplot.text(l,
k,
'%.1f' % (args.means[k, l]),
horizontalalignment='center',
verticalalignment='center',
fontsize=5)
pyplot.yticks(sp.arange(K), sp.arange(K) + 1)
pyplot.gca().set_yticks(sp.arange(K) + .5, minor=True)
pyplot.xticks(sp.arange(L),
valid_marks,
rotation=30,
horizontalalignment='right')
pyplot.gca().set_xticks(sp.arange(L) + .5, minor=True)
pyplot.grid(which='minor', alpha=.2)
for line in pyplot.gca().yaxis.get_ticklines() + pyplot.gca(
).xaxis.get_ticklines() + pyplot.gca().xaxis.get_ticklines(
minor=True) + pyplot.gca().yaxis.get_ticklines(minor=True):
# label is a Text instance
line.set_markersize(0)
pyplot.ylabel('Hidden State')
pyplot.title("Emission Mean")
#b = pyplot.colorbar(shrink=.7)
#b.set_label("Probability")
outfile = (args.out_params + '_it{iteration}.png').format(
param='emission_means', **args.__dict__)
pyplot.savefig(os.path.join(args.out_dir, outfile), dpi=240)
# plot variances
pyplot.figure(figsize=(max(1, round(L / 3.)), max(1, round(K / 3.))))
print(L / 3, K / 3.)
pyplot.imshow(args.variances.astype(sp.float64),
interpolation='nearest',
aspect='auto',
vmin=0,
vmax=args.variances.max(),
cmap='OrRd',
origin='lower')
for k in range(K):
for l in range(L):
pyplot.text(l,
k,
'%.1f' % (args.variances[k, l]),
horizontalalignment='center',
verticalalignment='center',
fontsize=5)
pyplot.yticks(sp.arange(K), sp.arange(K) + 1)
pyplot.gca().set_yticks(sp.arange(K) + .5, minor=True)
pyplot.xticks(sp.arange(L),
valid_marks,
rotation=30,
horizontalalignment='right')
pyplot.gca().set_xticks(sp.arange(L) + .5, minor=True)
pyplot.grid(which='minor', alpha=.2)
for line in pyplot.gca().yaxis.get_ticklines() + pyplot.gca(
).xaxis.get_ticklines() + pyplot.gca().xaxis.get_ticklines(
minor=True) + pyplot.gca().yaxis.get_ticklines(minor=True):
# label is a Text instance
line.set_markersize(0)
pyplot.ylabel('Hidden State')
pyplot.title("Emission Variance")
#b = pyplot.colorbar(shrink=.7)
#b.set_label("Probability")
outfile = (args.out_params + '_it{iteration}.png').format(
param='emission_variances', **args.__dict__)
pyplot.savefig(os.path.join(args.out_dir, outfile), dpi=240)
else:
matplotlib.rcParams['font.size'] = 8
pyplot.figure(figsize=(max(1, round(L / 3.)), max(1, round(K / 3.))))
print(L / 3, K / 3.)
pyplot.imshow(args.emit_probs.astype(sp.float64),
interpolation='nearest',
aspect='auto',
vmin=0,
vmax=1,
cmap='OrRd',
origin='lower')
for k in range(K):
for l in range(L):
pyplot.text(l,
k,
'%2.0f' % (args.emit_probs[k, l] * 100),
horizontalalignment='center',
verticalalignment='center')
pyplot.yticks(sp.arange(K), sp.arange(K) + 1)
pyplot.gca().set_yticks(sp.arange(K) + .5, minor=True)
pyplot.xticks(sp.arange(L),
valid_marks,
rotation=30,
horizontalalignment='right')
pyplot.gca().set_xticks(sp.arange(L) + .5, minor=True)
pyplot.grid(which='minor', alpha=.2)
for line in pyplot.gca().yaxis.get_ticklines() + pyplot.gca(
).xaxis.get_ticklines() + pyplot.gca().xaxis.get_ticklines(
minor=True) + pyplot.gca().yaxis.get_ticklines(minor=True):
# label is a Text instance
line.set_markersize(0)
pyplot.ylabel('Hidden State')
pyplot.title("Emission probabilities")
#b = pyplot.colorbar(shrink=.7)
#b.set_label("Probability")
outfile = (args.out_params + '_it{iteration}.png').format(
param='emission', **args.__dict__)
pyplot.savefig(os.path.join(args.out_dir, outfile), dpi=240)
#broad_paper_enrichment = sp.array([[16,2,2,6,17,93,99,96,98,2],
# [12,2,6,9,53,94,95,14,44,1],
# [13,72,0,9,48,78,49,1,10,1],
# [11,1,15,11,96,99,75,97,86,4],
# [5,0,10,3,88,57,5,84,25,1],
# [7,1,1,3,58,75,8,6,5,1],
# [2,1,2,1,56,3,0,6,2,1],
# [92,2,1,3,6,3,0,0,1,1],
# [5,0,43,43,37,11,2,9,4,1],
# [1,0,47,3,0,0,0,0,0,1],
# [0,0,3,2,0,0,0,0,0,0],
# [1,27,0,2,0,0,0,0,0,0],
# [0,0,0,0,0,0,0,0,0,0],
# [22,28,19,41,6,5,26,5,13,37],
# [85,85,91,88,76,77,91,73,85,78],
# [float('nan'), float('nan'),float('nan'),float('nan'),float('nan'),float('nan'),float('nan'),float('nan'),float('nan'),float('nan')]
# ]) / 100.
#mapping_from_broad = dict(zip(range(K), (5,2,0,14,4,6,9,1,12,-1,3,12,8,7,10,12,11,13)))
#broad_paper_enrichment = broad_paper_enrichment[tuple(mapping_from_broad[i] for i in range(K)), :]
#broad_names = ['Active promoter', 'Weak promoter', 'Inactive/poised promoter', 'Strong enhancer',
# 'Strong enhancer', 'weak/poised enhancer', 'Weak/poised enhancer', 'Insulator',
# 'Transcriptional transition', 'Transcriptional elongation', 'Weak transcribed',
# 'Polycomb repressed', 'Heterochrom; low signal', 'Repetitive/CNV', 'Repetitive/CNV',
# 'NA', 'NA', 'NA']
#pyplot.figure(figsize=(L/3,K/3.))
#print (L/3,K/3.)
#pyplot.imshow(broad_paper_enrichment, interpolation='nearest', aspect='auto',
# vmin=0, vmax=1, cmap='OrRd', origin='lower')
#for k in range(K):
# for l in range(L):
# pyplot.text(l, k, '%2.0f' % (broad_paper_enrichment[k,l] * 100), horizontalalignment='center', verticalalignment='center')
# pyplot.text(L, k, broad_names[mapping_from_broad[k]], horizontalalignment='left', verticalalignment='center', fontsize=6)
#pyplot.yticks(sp.arange(K), sp.arange(K)+1)
#pyplot.gca().set_yticks(sp.arange(K)+.5, minor=True)
#pyplot.xticks(sp.arange(L), valid_marks, rotation=30, horizontalalignment='right')
#pyplot.gca().set_xticks(sp.arange(L)+.5, minor=True)
#pyplot.grid(which='minor', alpha=.2)
#for line in pyplot.gca().yaxis.get_ticklines() + pyplot.gca().xaxis.get_ticklines() + pyplot.gca().xaxis.get_ticklines(minor=True) + pyplot.gca().yaxis.get_ticklines(minor=True):
## label is a Text instance
# line.set_markersize(0)
#pyplot.ylabel('Hidden State')
#pyplot.title("Broad paper Emission probabilities")
##b = pyplot.colorbar(shrink=.7)
##b.set_label("Probability")
#pyplot.subplots_adjust(right=.7)
#outfile = (args.out_params + '_broadpaper.png').format(param='emission', **args.__dict__)
#pyplot.savefig(os.path.join(args.out_dir, outfile), dpi=240)
pyplot.close('all')
sp.seterr(**old_err)
matplotlib.rcParams['font.size'] = oldsize
def generateThumbnail(inputFile, thumbSize):
f1 = inpFilt.h5rSource(inputFile)
threeD = False
stack = False
split = False
# print f1.keys()
if "fitResults_Ag" in f1.keys():
# if we used the splitter set up a mapping so we can filter on total amplitude and ratio
f1_ = inpFilt.mappingFilter(
f1, A="fitResults_Ag + fitResults_Ar", gFrac="fitResults_Ag/(fitResults_Ag + fitResults_Ar)"
)
# f2 = inpFilt.resultsFilter(f1_, error_x=[0,30], A=[5, 1e5], sig=[100/2.35, 350/2.35])
split = True
else:
f1_ = f1
if "fitResults_sigma" in f1.keys():
f2 = inpFilt.resultsFilter(f1_, error_x=[0, 30], A=[5, 1e5], sig=[100 / 2.35, 350 / 2.35])
else:
f2 = inpFilt.resultsFilter(f1_, error_x=[0, 30], A=[5, 1e5])
if "fitResults_z0" in f1_.keys():
threeD = True
if "Events" in dir(f1.h5f.root):
events = f1.h5f.root.Events[:]
evKeyNames = set()
for e in events:
evKeyNames.add(e["EventName"])
if "ProtocolFocus" in evKeyNames:
stack = True
xmax = f2["x"].max()
ymax = f2["y"].max()
if xmax > ymax:
step = xmax / thumbSize
else:
step = ymax / thumbSize
im, edx, edy = histogram2d(f2["x"], f2["y"], [arange(0, xmax, step), arange(0, ymax, step)])
f1.close()
im = minimum(2 * (255 * im) / im.max(), 255).T
im = concatenate((im[:, :, newaxis], im[:, :, newaxis], im[:, :, newaxis]), 2)
if stack:
im[-10:, -10:, 0] = 180
if threeD:
im[-10:, -10:, 1] = 180
if split:
im[-10:-5, :10, 1] = 210
im[-5:, :10, 0] = 210
return im.astype("uint8")
def calc_sum_xyz(datafile, scale=None, bins=100, filter=None, invert=False,
clip=None):
"""A function to take a file of data and create an XYZ data object.
Take values at random x,y points and bin into summed values.
datafile a file containing (lat, lon, val) values
scale amount to scale the data (ie, divide)
bins number of bins in X and Y direction
if an int, # bins in X and Y direction
if [int, int] the X and Y bin counts may be different
filter a function to extract (lon, lat, val) from one datafile line
if not supplied an internal function is used
invert if 'filter' not supplied, an internal filter is used. if
'invert' is True, switch the order of lat/lon in data lines.
clip if defined is a dictionary defining clip limits. Recognised
keys in the dictionary are:
'high' sets high limit above which values are clipped
'low' sets low limit below which values are clipped
At least one of the above keys must exist.
The clipping is done before any scaling.
Returns a numpy array of XYZ data [[lon, lat, val], ...].
Values that are 0 replaced with Nan.
"""
# handle optional parameters
try:
bin_len = len(bins)
except TypeError:
bins_x = bins_y = bins
else:
try:
(bins_x, bins_y) = bins
except ValueError:
raise RuntimeError("Bad 'bins' value, expected int or [int, int]")
# if user didn't supply a data filter, use our own
if filter is None:
# define a default filter and use it
if invert:
def default_filter(line):
return (float(line[1]), float(line[0]), float(line[2]))
else:
def default_filter(line):
return (float(line[0]), float(line[1]), float(line[2]))
filter = default_filter
# get data into memory
data = []
for line in file(datafile):
# ignore blank or comment lines
line = line.strip()
if line == '' or line[0] in '%#':
continue
# get lon+lat+value from line
data.append(filter(SplitPattern.split(line)))
data = scipy.array(data)
# do clipping, if required
if clip:
low_clip = clip.get('low', None)
high_clip = clip.get('high', None)
if low_clip is None:
low_clip = scipy.min(data[:,2])
if high_clip is None:
high_clip = scipy.max(data[:,2])
data[:,2] = scipy.clip(data[:,2], low_clip, high_clip)
# handle any scaling
if scale:
scale = int(scale)
data[:,2] = data[:,2] / scale
# get extent of data (tight first, then with margin)
(tll_lat, tll_lon, tur_lat, tur_lon) = ge.get_extent(data, margin=0)
tr_opt = '-R%f/%f/%f/%f' % (tll_lon, tur_lon, tll_lat, tur_lat)
(ll_lat, ll_lon, ur_lat, ur_lon) = ge.get_extent(data)
r_opt = '-R%f/%f/%f/%f' % (ll_lon, ur_lon, ll_lat, ur_lat)
# now generate a binned dataset
(binned_data, xedges, yedges) = scipy.histogram2d(data[:,0], data[:,1],
bins=bins, normed=False,
weights=data[:,2])
# create XYZ object
# make sure X+Y is *centre* of each bin
xyz = []
xedges = scipy.array(xedges)
xedges = xedges[:-1] + (xedges[1] - xedges[0])/2
yedges = scipy.array(yedges)
yedges = yedges[:-1] + (yedges[1] - yedges[0])/2
for (xi, x) in enumerate(xedges):
for (yi, y) in enumerate(yedges):
xyz.append([x, y, binned_data[xi,yi]])
return scipy.array(xyz)
def visualize2PEC2(maxT, sumM, name, nozero=False, k=2.2e35, useall=False, lim=1e-2, conc='c_w_l',loc='data_to_Ian.txt',idx2=None):
""" log plot of the data with only PEC data"""
val, c_w_l, idx = conditionVal(name=name, nozero=nozero, conc=conc)
data = scipy.io.readsav('W_Abundances_grid_puestu_adpak_fitscaling_74_0.00000_5.00000_1000_idlsave')
y = time.time()
xax = scipy.logspace(2,5,201)
yax = scipy.logspace(-5,1,101)
xtemp = xax[1:]/2.+xax[:-1]/2.
ytemp = yax[1:]/2.+yax[:-1]/2.
Y,X = scipy.meshgrid(xtemp, ytemp)
if idx2 is None:
histdata, xed, yed = scipy.histogram2d(maxT,val/c_w_l/sumM*k,bins=[xax,yax])
else:
histdata, xed, yed = scipy.histogram2d(maxT[idx2],(val/c_w_l/sumM*k)[idx2],bins=[xax,yax])
extent = [xed[0], xed[-1], yed[0], yed[-1]]
plt.pcolormesh(xax,yax,histdata.T, cmap='viridis', rasterized=True,vmin=1.,norm=LogNorm())
plt.gca().set_yscale('log')
plt.gca().set_xscale('log')
plt.gca().set_ylim(1e-4,1e1)
plt.gca().set_xlim(1e3,2e4)
cmap = plt.cm.get_cmap('viridis')
cmap.set_under('white')
inp = adsre.lineInfo(loc)
if True:
alpha=.8
idx = scipy.logical_and(data['en'] > 1e3, data['en'] < 5e4)
output = scipy.zeros(data['en'].shape)
output2 = scipy.zeros(data['en'].shape)
temp = scipy.arange(len(output))[idx]
for i in temp:
PEC = inp(data['en'], 0)
output[i] = scipy.sum(testM.normMn2(10, scipy.log(data['en']), scipy.log(5e2), scipy.log(data['en'][i]), .125)*data['abundance'][:,46]*PEC)
output2[i] = scipy.sum(testM.normMn2(10, scipy.log(data['en']), scipy.log(5e2), scipy.log(data['en'][i]), .125)*data['abundance'][:,46])
plt.loglog(data['en'],output/output.max(),lw=3,color='crimson',alpha=alpha,linestyle='--',label=r'W$^{46+}$')
output = scipy.zeros(data['en'].shape)
temp = scipy.arange(len(output))[idx]
for i in temp:
PEC = inp(data['en'], 3)
output[i] = scipy.sum(testM.normMn2(10, scipy.log(data['en']), scipy.log(5e2), scipy.log(data['en'][i]), .125)*data['abundance'][:,45]*PEC)
output2[i] = scipy.sum(testM.normMn2(10, scipy.log(data['en']), scipy.log(5e2), scipy.log(data['en'][i]), .125)*data['abundance'][:,45])
plt.loglog(data['en'],output/output.max(),lw=3,color='magenta',alpha=alpha,linestyle='--',label=r'W$^{45+}$')
output = scipy.zeros(data['en'].shape)
temp = scipy.arange(len(output))[idx]
for i in temp:
PEC = inp(data['en'], 6)
output[i] = scipy.sum(testM.normMn2(10, scipy.log(data['en']), scipy.log(5e2), scipy.log(data['en'][i]), .125)*data['abundance'][:,44]*PEC)
output2[i] = scipy.sum(testM.normMn2(10, scipy.log(data['en']), scipy.log(5e2), scipy.log(data['en'][i]), .125)*data['abundance'][:,44])
plt.loglog(data['en'],output/output.max(),lw=3,color='cyan',alpha=alpha,linestyle='--',label=r'W$^{44+}$')
plt.xlabel(r'$T_e$ [eV]')
plt.ylabel(r'const$\cdot I_{CSXR} / c_W \sum M$')
colorbar = plt.colorbar()
colorbar.ax.set_ylabel('datapoint density (a.u.)')
leg = plt.legend(loc=4,fontsize=20,title=r'\underline{\hspace{1em} \emph{with PEC} \hspace{1em}}')
plt.setp(leg.get_title(),fontsize=14)
plt.subplots_adjust(bottom=.12,right=1.)
def main(argv):
datafile = "../test/mg5pythia8_hp200.root.test3.csv"
try:
datafile = argv[1]
except IndexError:
pass
ds = getData(datafile)
ds.fillna(-999.)
predictors = sp.array([
"et(met)", "phi(met)",
#"nbjet", "njet",
"pt(reco tau1)", "eta(reco tau1)", "phi(reco tau1)", "m(reco tau1)",
"pt(reco bjet1)", "eta(reco bjet1)", "phi(reco bjet1)", "m(reco bjet1)",
"pt(reco jet1)", "eta(reco jet1)", "phi(reco jet1)", "m(reco jet1)",
"pt(reco jet2)", "eta(reco jet2)", "phi(reco jet2)", "m(reco jet2)",
], dtype=str)
target = "pt(mc nuH)"
#target = "m(true h+)"
ds["m(true h+)"] = sp.sqrt(
2*(ds["pt(mc nuH)"])
*(ds["pt(mc tau)"])
*(sp.cosh(ds["eta(mc nuH)"] - ds["eta(mc tau)"])
- sp.cos(ds["phi(mc nuH)"] - ds["phi(mc tau)"])
)
)
selector = SelectKBest(score_func=f_regression, k=5)
selector.fit(ds[predictors], ds[target])
ind = selector.get_support(indices=True)
final_predictors = predictors[ind]
print(final_predictors)
ds = ds[:1000]
folds = KFold(ds.shape[0], n_folds=3, random_state=123)
models = {}
models["gp"] = GaussianProcess(theta0=1e-2, thetaL=1e-4, thetaU=1e-1, nugget=1e-1)
models["krr"] = KernelRidge(kernel='linear')
models["svr"] = SVR(kernel='rbf', gamma=0.001, C=1e5)
models["nusvr"] = NuSVR(kernel='linear')
models["linearsvr"] = LinearSVR(C=1e1, loss='epsilon_insensitive',
max_iter=1e4, verbose=True, tol=1e-1)
model = models["gp"]
predictions = []
try:
for train, test in folds:
training_sample = ds[final_predictors].iloc[train, :]
target_sample = ds[target].iloc[train]
testing_sample = ds[final_predictors].iloc[test, :]
model.fit(training_sample, target_sample)
pred = model.predict(testing_sample)
predictions.append(pred)
except MemoryError as e:
print("MemoryError")
type_, value_, traceback_ = sys.exc_info()
traceback.print_tb(traceback_)
sys.exit(1)
except Exception as e:
print("Unexpected exception")
print(e)
type_, value_, traceback_ = sys.exc_info()
traceback.print_tb(traceback_)
sys.exit(2)
predictions = sp.concatenate(predictions, axis=0)
t = sp.array(ds[target], dtype=float)
met = sp.array(ds["et(met)"], dtype=float)
resolution = (predictions - t)*100/t
resolution_met = (met - t)*100/t
print("")
print("Prediction resolution: mean (sigma): {} ({})"
.format(resolution.mean(),
resolution.std()))
print("MET resolution: mean (sigma): {} ({})"
.format(resolution_met.mean(),
resolution_met.std()))
bins = sp.linspace(0, 400, 50)
plt.subplot(2, 3, 1)
plt.hist(t, bins, facecolor='blue', label='Obs', alpha=0.5, normed=1,
histtype='stepfilled')
plt.hist(predictions, bins, facecolor='orange', label='Pred',
alpha=0.5, normed=1, histtype='stepfilled')
plt.hist(met, bins, edgecolor='red', label='MET', alpha=0.5, normed=1,
histtype='step', linewidth=2)
plt.xlabel(target)
plt.legend(loc='best')
plt.subplot(2, 3, 2)
plt.hist(resolution, sp.linspace(-100, 100, 50),
facecolor='green', label='Res', alpha=0.5, histtype='stepfilled')
plt.xlabel('Resolution')
plt.legend(loc='best')
plt.subplot(2, 3, 4)
hist2d_pred, x_edges, y_edges = sp.histogram2d(t, predictions, bins=bins)
plt.pcolor(hist2d_pred)
plt.xlabel(target)
plt.ylabel('Prediction')
plt.subplot(2, 3, 5)
plt.scatter(t, predictions)
plt.xlabel(target)
plt.ylabel('Prediction')
plt.subplot(2, 3, 3)
plt.hist(resolution_met, sp.linspace(-100, 100, 50),
facecolor='green', label='Res (MET)', alpha=0.5, histtype='stepfilled')
plt.xlabel('Resolution')
plt.legend(loc='best')
plt.subplot(2, 3, 6)
plt.scatter(t, met)
plt.xlabel(target)
plt.ylabel('MET')
plt.show()
def hist2d_contour(a,
x,
y,
w=None,
plot_heatmap=False,
plot_levels_filled=False,
plot_levels=True,
plot_points=False,
filter_contour=True,
filter_heatmap=False,
hist_kwargs={},
pcolor_kwargs={},
contour_kwargs={},
scatter_kwargs={},
filter_kwargs={},
filter_sigma=1.0,
plot_ci=None,
ci_kwargs={},
scatter_fraction=1.0,
use_kde=False,
kde_bw0=None,
kde_bw1=None):
"""Make combined 2d histogram, contour and/or scatter plot.
Parameters
----------
a : axes
The axes to plot on.
x : 1d array
The x data.
y : 1d array
The y data.
w : 1d array, optional
The weights
plot_heatmap : bool, optional
If True, plot the heatmap of the histogram. Default is False.
plot_levels_filled : bool, optional
If True, plot the filled contours of the histogram. Default is False.
plot_levels : bool, optional
If True, plot the contours of the histogram. Default is True.
plot_points : bool, optional
If True, make a scatterplot of the points. Default is False.
filter_contour : bool, optional
If True, filter the histogram before plotting contours. Default is
True.
filter_heatmap : bool, optional
If True, filter the histogram before plotting heatmap. Default is
False.
hist_kwargs : dict, optional
Keyword arguments for scipy.histogram2d.
pcolor_kwargs : dict, optional
Keyword arguments for pcolormesh when plotting heatmap.
contour_kwargs : dict, optional
Keyword arguments for contour and contourf when plotting contours. To
specify the number of contours, use the key 'N'. To use specific
contour levels, use the key 'V'.
scatter_kwargs : dict, optional
Keyword arguments for scatterplot when plotting points.
filter_kwargs : dict, optional
Keyword arguments for filtering of histogram.
filter_sigma : float, optional
The standard deviation for the Gaussian filter used to smooth the
histogram. Default is 2 bins.
plot_ci : float or 1d array, optional
If this is a float, the contour containing this much probability mass
is drawn. Default is None (don't draw contour).
ci_kwargs : dict, optional
Keyword arguments for drawing the confidence interval.
scatter_fraction : float, optional
Fraction of points to include in the scatterplot. Default is 1.0 (use
all points).
use_kde : bool, optional
If True, use a kernel density estimator (KDE) in place of bivariate
histogram when `plot_heatmap=True`. Default is False.
kde_bw0 : float, optional
KDE bandwidth for the zeroth dimension.
kde_bw1 : float, optional
KDE bandwidth for the first dimension.
"""
if 'bins' not in hist_kwargs:
hist_kwargs['bins'] = (100, 101)
if 'normed' not in hist_kwargs:
hist_kwargs['normed'] = True
# Only compute histogram if needed:
if (plot_heatmap or plot_levels or plot_levels_filled
or (plot_ci is not None)):
if use_kde:
xgrid = scipy.linspace(x.min(), x.max(), hist_kwargs['bins'][0])
ygrid = scipy.linspace(y.min(), y.max(), hist_kwargs['bins'][1])
H = wkde.bivariate_weighted_kde(
x, y, w if w is not None else scipy.ones(len(x)), xgrid, ygrid,
kde_bw0, kde_bw1)
# Match pcolormesh's silly format:
dx = xgrid[1] - xgrid[0]
xedges = (xgrid[1:] + xgrid[:-1]) / 2.0
xedges = scipy.concatenate(
([xedges[0] - dx], xedges, [xedges[-1] + dx]))
dy = ygrid[1] - ygrid[0]
yedges = (ygrid[1:] + ygrid[:-1]) / 2.0
yedges = scipy.concatenate(
([yedges[0] - dy], yedges, [yedges[-1] + dy]))
# Doesn't make sense to filter KDE...
Hf = H
else:
H, xedges, yedges = scipy.histogram2d(x,
y,
weights=w,
**hist_kwargs)
if filter_contour or filter_heatmap:
Hf = gaussian_filter(H, filter_sigma, **filter_kwargs)
if plot_heatmap:
XX, YY = scipy.meshgrid(xedges, yedges)
a.pcolormesh(XX, YY, Hf.T if filter_heatmap else H.T, **pcolor_kwargs)
if plot_levels or plot_levels_filled or (plot_ci is not None):
# Convert to bin centers:
xcenters = 0.5 * (xedges[:-1] + xedges[1:])
ycenters = 0.5 * (yedges[:-1] + yedges[1:])
XX, YY = scipy.meshgrid(xcenters, ycenters)
args = []
if 'V' in contour_kwargs:
args += [
scipy.atleast_1d(contour_kwargs.pop('V')),
]
elif 'N' in contour_kwargs:
args += [
contour_kwargs.pop('N'),
]
if plot_levels_filled:
a.contourf(XX, YY, Hf.T if filter_contour else H.T, *args,
**contour_kwargs)
if plot_levels:
a.contour(XX, YY, Hf.T if filter_contour else H.T, *args,
**contour_kwargs)
if plot_ci is not None:
V = prob_contour(H, xedges, yedges, p=plot_ci)
if 'vmin' not in ci_kwargs:
ci_kwargs['vmin'] = 0.0
if 'vmax' not in ci_kwargs:
ci_kwargs['vmax'] = H.max()
a.contour(XX, YY, Hf.T if filter_contour else H.T,
scipy.unique(scipy.atleast_1d(V)), **ci_kwargs)
if plot_points:
# Make the markersize not ridiculous by default:
if 'ms' not in scatter_kwargs and 'markersize' not in scatter_kwargs:
scatter_kwargs['ms'] = 0.1
# Make points transparent by default (approximates heatmap...):
if 'alpha' not in scatter_kwargs:
scatter_kwargs['alpha'] = 0.5
# Do not connect with lines by default:
if 'ls' not in scatter_kwargs and 'linestyle' not in scatter_kwargs:
scatter_kwargs['ls'] = ''
# Plot with circles by default:
if 'marker' not in scatter_kwargs:
scatter_kwargs['marker'] = 'o'
if scatter_fraction != 1.0:
N = int(round(scatter_fraction * len(x)))
indices = scipy.random.choice(range(len(x)), size=N, replace=False)
else:
indices = range(len(x))
a.plot(x[indices], y[indices], **scatter_kwargs)
def hist_2d(vis_x,vis_y):
hh, locx, locy = scipy.histogram2d(vis_x, vis_y, bins=[200,200])
fig = plt.figure(frameon=False)
fig.set_size_inches(30,30)
plt.imshow(np.flipud(hh.T),cmap='jet', interpolation='none', shape = (1,1))
plt.colorbar()
def plotTTCvsMMMD(experiment):
import scipy
fitnessThreshold = -8
def TTCvsMMMD(gridPoint):
if not (gridPoint['compositeClass0']
== 'integerVectorSymmetricRangeMutations'
and gridPoint['probabilityOfMutatingClass0'] == 0.2):
return
# determining the time of convergence
bilogFileName = 'bestIndividual{}.log'.format(gridPoint['randomSeed'])
bilog = np.loadtxt(bilogFileName)
toc = None # time of convergence
# TOME OF CONVERGENCE
for i in range(bilog.shape[0]):
if -1. * bilog[i, 1] <= fitnessThreshold:
toc = i
break
if toc is None:
with open('../results/nonconverged runs', 'a') as ncrf:
ncrf.write(str(gridPoint['randomSeed']) + '\n')
return
# making a scatter of points of TTC vs MMMD
with open('../results/ttcvsmmmd', 'a') as tmfile:
for i in range(evsDefaults['genStopAfter'] + 1):
tmfile.write('{} {}\n'.format(
toc - i, gctools.minParetoFrontHammingDistanceToMMM(i)))
experiment.executeAtEveryGridPointDir(TTCvsMMMD)
os.chdir('results')
ttcmmmddata = np.loadtxt('ttcvsmmmd')
mmmd = ttcmmmddata[:, 1]
ttc = ttcmmmddata[:, 0]
mmmdrange = [0, 6]
ttcrange = [min(ttc), max(ttc)]
mmmdbins = 6
ttcbins = 100
xdat = mmmd
ydat = ttc
xrange = mmmdrange
yrange = ttcrange
xbins = mmmdbins
ybins = ttcbins
# https://stackoverflow.com/questions/10439961
xyrange = [xrange, yrange]
bins = [xbins, ybins]
thresh = 1
hh, locx, locy = scipy.histogram2d(xdat, ydat, range=xyrange, bins=bins)
posx = np.digitize(xdat, locx)
posy = np.digitize(ydat, locy)
# select points within the histogram
ind = (posx > 0) & (posx <= bins[0]) & (posy > 0) & (posy <= bins[1])
hhsub = hh[posx[ind] - 1,
posy[ind] - 1] # values of the histogram where the points are
xdat1 = xdat[ind][hhsub < thresh] # low density points
ydat1 = ydat[ind][hhsub < thresh]
hh[hh < thresh] = np.nan # fill the areas with low density by NaNs
fig, ax = plt.subplots(figsize=(7, 6))
cax = fig.add_axes() # https://stackoverflow.com/questions/32462881
#im = ax.imshow(np.flipud(hh.T),cmap='jet',extent=np.array(xyrange).flatten(), interpolation='none', origin='upper',norm=LogNorm(vmin=1, vmax=10000))
im = ax.imshow(np.flipud(hh.T),
cmap='jet',
extent=np.array(xyrange).flatten(),
interpolation='none',
origin='upper',
norm=LogNorm(vmin=1))
cb = fig.colorbar(im, cax=cax)
cb.set_label('# of points')
ax.plot(xdat1, ydat1, '.', color='darkblue')
ax.set_xlabel(r'$\mu$', x=1.0, fontsize=20)
ax.set_ylabel(r'$\tau_{conv}$', y=1.0, fontsize=20)
# ax.set_ylim([0,420])
ax.set_aspect(0.006)
plt.savefig('ttcvsmmmd.png', dpi=300)
plt.clf()
os.chdir('..')
def make_plot(points,AC,name):
fig=plt.figure(1,figsize=(15,10), dpi=100)
ax1=fig.add_subplot(2,3,1)
ax1.plot(points[:name,0],points[:name,1],'k.')
ax1.set_xlim(0,width)
ax1.set_ylim(0,height)
ax1.set_title('Posterior points')
ax1.set_yticks([])
ax1.set_xticks([])
#just plotting the active points
ax3=fig.add_subplot(2,3,2)
xt=points[:name,0]
yt=points[:name,1]
hh,locx,locy=scipy.histogram2d(xt,yt,bins=[linspace(0,width,width+1),linspace(0,height,height+1)])
#x,y histogram
#more common points will increase the histogram value and be more intense on the image
ax3.imshow(flipud(hh.T),extent=[0,width,0,height],aspect='normal')
ax3.set_title('Pseudo image from posterior')
ax2=fig.add_subplot(2,3,3)
ax2.plot(AC[:,0],AC[:,1],'k.')
ax2.set_xlim(0,width)
ax2.set_ylim(0,height)
ax2.set_yticks([])
ax2.set_xticks([])
ax2.set_title('Active points')
#just plotting the active points
ax4=fig.add_subplot(2,3,4)
ax4.imshow(flipud(data),extent=[0,width,0,height])
ax4.set_title('Original image with noise')
#show input noised image
ax5=fig.add_subplot(2,3,5)
ax5.imshow(flipud(data_or),extent=[0,width,0,height])
ax5.set_title('Original image ')
#show the original image
fname="%05d" % name
ax6=fig.add_subplot(2,3,6)
img=mpimg.imread(output_folder + '/plots/somplot/som_'+fname+'.png')
#earlier we created the som image. now we read it back in
ax6.imshow(img,extent=[0,width,0,height],aspect='normal')
#and display it as one of the panels in the subplot
ax6.set_title('SOM map ')
fig.savefig(output_folder + '/plots/6plot/all6_'+fname+'.png',bbox_inches='tight')
fig.clear()
"""
ax1 = orig
ax2 = orig w/ noise
ax3 = posterior
ax4 = active
"""
fig=plt.figure(2,figsize=(50,50), dpi=100)
ax1 = plt.subplot2grid((2,2), (0,0))
ax1.imshow(flipud(data_or),extent=[0,width,0,height])
ax1.set_title('Original image')
ax1.set_yticks([])
ax1.set_xticks([])
ax2 = plt.subplot2grid((2,2), (0,1))
ax2.imshow(flipud(data),extent=[0,width,0,height])
ax2.set_title('Original image with noise')
ax2.set_yticks([])
ax2.set_xticks([])
ax3 = plt.subplot2grid((2,2), (1,0))
ax3.plot(points[:name,0],points[:name,1],'k.')
ax3.set_xlim(0,width)
ax3.set_ylim(0,height)
ax3.set_title('Posterior points')
ax3.set_yticks([])
ax3.set_xticks([])
ax4 = plt.subplot2grid((2,2), (1,1))
ax4.plot(AC[:,0],AC[:,1],'k.')
ax4.set_xlim(0,width)
ax4.set_ylim(0,height)
ax4.set_yticks([])
ax4.set_xticks([])
ax4.set_title('Active points')
fig.savefig(output_folder + '/plots/4plot/4plot'+fname+'.png',bbox_inches='tight')
fig.clear()