def Get_2DTProfile(ar1, ar2, ar3, nbBinsX, nbBinsY,we):
'''
'''
d = numpy.array(zip(ar1,ar2,ar3))
number, axis = numpy.histogramdd( d, (nbBinsX,nbBinsY,1))
weight, axis = numpy.histogramdd( d, (nbBinsX,nbBinsY,1), weights=we )
mean, axis = numpy.histogramdd( d, (nbBinsX,nbBinsY,1), weights=we*ar3)
err, axis = numpy.histogramdd( d, (nbBinsX,nbBinsY,1), weights=we*(ar3**2.))
mean /= weight
err = numpy.sqrt((err/weight-mean**2.)/number)
mean = mean[:,:,0]
err = err[:,:,0]
number = number[:,:,0]
### find the axis X
#axisX = axis[0]
#axisX = numpy.array([ axisX[i]+(axisX[i+1]-axisX[i])/2. for i in range(0,axisX.size-1) ])
### find the axis Y
#axisY = axis[1]
#axisY = numpy.array([ axisY[i]+(axisY[i+1]-axisY[i])/2. for i in range(0,axisY.size-1) ])
### For test look at the histo
#plt.imshow(mean,origin='lower',extent=[0., 10., 0., 10.],interpolation='None')
#cbar = plt.colorbar()
#plt.show()
return mean, err, number
def MeasureColorVector(self,img,HistSizes):
""" Returns a color vector obtained by histogram
of number of bins mentioned in HistSizes for each color layer """
ImgX,ImgY,ImgZ = img.shape
# First define masks for the partions in the image
Mask0 = np.zeros(img.shape)
Mask0[ImgX/3:2*ImgX/3,ImgY/3:2*ImgY/3,:] = 1 # Central 1/3 region rectangle
Vector0 = np.histogramdd(img[Mask0], bins=HistSizes, normed=True)
Mask1 = np.zeros(img.shape)
Mask1[0:ImgX/2,0:ImgY/2,:] = 1 # 1st quadrent
Mask1[0:ImgX/2,0:ImgY/2,:] -= Mask0[0:ImgX/2,0:ImgY/2,:]
Vector1 = np.histogramdd(img[Mask1], bins=HistSizes, normed=True)
Mask2 = np.zeros(img.shape)
Mask2[ImgX/2:,0:ImgY/2,:] = 1 # 2nd quadrent
Mask2[ImgX/2:,0:ImgY/2,:] -= Mask0[ImgX/2:,0:ImgY/2,:]
Vector2 = np.histogramdd(img[Mask2], bins=HistSizes, normed=True)
Mask3 = np.zeros(img.shape)
Mask3[0:ImgX/2,ImgY/2:,:] = 1 # 3rd quadrent
Mask3[0:ImgX/2,ImgY/2:,:] -= Mask0[0:ImgX/2,ImgY/2:,:]
Vector3 = np.histogramdd(img[Mask3], bins=HistSizes, normed=True)
Mask4 = np.zeros(img.shape)
Mask4[ImgX/2:,ImgY/2:,:] = 1 # 4th quadrent
Mask4[ImgX/2:,ImgY/2:,:] -= Mask0[ImgX/2:,ImgY/2:,:]
Vector4 = np.histogramdd(img[Mask4], bins=HistSizes, normed=True)
return np.concatenate((Vector0,Vector1,Vector2,Vector3,Vector4))
def myhist(x,weights=None,**histkw):
"""
Multidimensional histogram with option for multidimensional weights
"""
# Pars input
x=mmlpars.mml_pars(x,type=np.ndarray)
xshp=x.shape ; N=xshp[0]
# Histogram w/o weights
if weights==None: hist,bins=np.histogramdd(x,**histkw)
# Histogram w/ weights
else:
wshp=weights.shape
# Histogram w/ scaler weights
if wshp==(N,): hist,bins=np.histogramdd(x,weights=weights,**histkw)
# Histogram w/ vector weights
else:
# Handle errors
if wshp[0]!=N: raise Exception('Weights must have same size first dimension as data. (data.shape={},weights.shape={})'.format(xshp,wshp))
if len(wshp)>2: raise Exception('Weights with more than 2 dimensions not supported. (weights.shape={})'.format(wshp))
# Get histograms
owshp=tuple(list(histlist[0].shape)+[wshp[1]])
hist=np.zeros(owshp)
for iw in range(wshp[1]):
ihist,bins=np.histogramdd(x,weights=weights[:,iw],**histkw)
if iw==0: histkw['bins']=bins
if len(xshp)==1: hist[:,iw]=ihist
elif len(xshp)==2: hist[:,:,iw]=ihist
elif len(xshp)==3: hist[:,:,:,iw]=ihist
elif len(xshp)==4: hist[:,:,:,:,iw]=ihist
else: raise Exception('Multidimensional weights only supported for data with <4 dimensions. (data.shape={})'.format(xshp))
# Return output
return hist,bins
def grid_21cm(self,data=None):
deltaT=np.empty_like(data)
for i in np.arange(freq.shape[0]):
deltaT[i]=data[i]-data[i].mean()
nn=np.histogramdd(self.DPM,bins=(self.bin_x,self.bin_y,self.bin_z))[0]
T,edges=np.histogramdd(self.DPM,bins=(self.bin_x,self.bin_y,self.bin_z),weights=deltaT.reshape(-1))
return T
def spline_fit(data,
bins = None,
range = None,
weights = None,
order = None,
filename = 'spline.fits'):
if bins is None:
bins = data.shape[1]*[10]
counts,bin_arrays = np.histogramdd(data,range=range,weights=weights)
vars,bin_arrays = np.histogramdd(data,range=range,weights=weights**2)
else:
counts,bin_arrays = np.histogramdd(data,bins=bins,range=range,weights=weights)
vars,bin_arrays = np.histogramdd(data,bins=bins,range=range,weights=weights**2)
coords = [(b[1:]+b[:-1])/2. for b in bin_arrays]
if order == None:
order = list(np.zeros_like(bins))
knots = pad_knots(bin_arrays, order)
w = 1./np.sqrt(vars)
w[~np.isfinite(w)] = np.nanmin(w)
result = glam.fit(counts,w,coords,knots,order,0)
if not filename is None:
if os.path.exists(filename):
os.system('rm '+filename)
if filename[-5:]=='.fits':
splinefitstable.write(result,filename)
else:
splinefitstable.write(result,filename+'.fits')
return result
def bin_sum(r, f, bins=10):
"""Binned sum of function f(r)
Parameters:
r: independent variable to be binned over
f: function to be summed
bins: (default 10): number of bins or bin edges `len(nbins)+1`
Returns:
total: the total value per bin
count: number of values summed per bin (histogram)
bins: bin edges
"""
multi = isinstance(f, tuple)
if bins is 1:
if r.dtype.kind not in 'iu':
assert np.allclose(r, np.around(r)), 'need integer array for bins=1'
print 'converting to int array'
r = r.astype(int)
count = np.bincount(r)
if multi:
total = [np.bincount(r, weights=fi) for fi in f]
else:
total = np.bincount(r, weights=f)
bins = np.arange(len(count)+1)
else:
count, bins = np.histogramdd(r, bins)
if multi:
total = [np.histogramdd(r, bins, weights=fi)[0] for fi in f]
else:
total = np.histogramdd(r, bins, weights=f)[0]
if len(bins) == 1:
bins = bins[0]
return total, count.astype(int), bins
def __init__(self, samples, recovery, bins=32, range=None,
transit_lnprob_function=None):
# Make sure that the samples have the correct format.
samples = np.atleast_2d(samples)
# Compute the recovery and injection histograms.
img_all, self.bins = np.histogramdd(samples, bins=bins, range=range)
img_yes, tmp = np.histogramdd(samples[recovery], bins=self.bins)
self.setup()
# Compute the completeness asserting zero completeness where there
# were no injections.
lncompleteness = -np.inf + np.zeros(img_yes.shape, dtype=float)
m = img_all > 0
lncompleteness[m] = np.log(img_yes[m]) - np.log(img_all[m])
# Compute the transit probability if a function was given.
if transit_lnprob_function is None:
lnprob = np.array(lncompleteness)
else:
args = np.meshgrid(*(self.bin_centers), indexing="ij")
transit_lnprob = transit_lnprob_function(*args)
lnprob = lncompleteness + transit_lnprob
# Expand the completeness and probability grids to have zeros around
# the edges.
self.lncompleteness = -np.inf + np.zeros(np.array(img_yes.shape)+2,
dtype=float)
self.lncompleteness[[slice(1, -1)] * len(self.bins)] = lncompleteness
self.lnprob = -np.inf + np.zeros(np.array(img_yes.shape)+2,
dtype=float)
self.lnprob[[slice(1, -1)] * len(self.bins)] = lnprob
def test3(self):
print "Testing the user interface, random base_distribution_type"
number_of_particles = 10000
test_grid = self.setup_simple_grid()
sph_particles = convert_grid_to_SPH(test_grid, number_of_particles,
base_distribution_type = "random", seed = 12345)
self.assertEqual(len(sph_particles), number_of_particles)
self.assertAlmostEqual(sph_particles.mass.sum(), 1.5 | units.kg)
self.assertAlmostEqual(sph_particles.velocity, [3.0, 4.0, 0.0] | units.m/units.s)
self.assertAlmostEqual(sph_particles.u, 1.0 | (units.m/units.s)**2)
# For 'random', the number of particles in a cell should scale only on average
# with the amount of mass in the cell:
self.assertAlmostRelativeEqual(
((1.5 | units.kg)/number_of_particles * numpy.histogramdd(
sph_particles.position.value_in(units.m), bins=(4,3,2))[0]).sum(),
(test_grid.rho*test_grid.cellsize().prod()).sum(),
places = 2
)
self.assertRaises(AssertionError,
self.assertAlmostRelativeEqual,
(1.5 | units.kg)/number_of_particles * numpy.histogramdd(sph_particles.position.value_in(units.m), bins=(4,3,2))[0],
test_grid.rho*test_grid.cellsize().prod(),
places = 2,
)
self.assertAlmostEqual(sph_particles.h_smooth, (50.0/number_of_particles)**(1.0/3) | units.m)
def transfer_entropy(ts1, ts2, lag=2, bins=5):
""" D_1<-2 """
ts1, lts1 = multi_lag(ts1, lag)
ts2, lts2 = multi_lag(ts2, lag)
# P(i_n+1, i_(n), j_(n))
joint = np.histogramdd([ts1] + lts1 + lts2, bins=bins)[0]
joint = normalize(joint)
# P(i_n+1, i_(n))
auto = np.histogramdd([ts1] + lts1, bins=bins)[0]
auto = normalize(auto)
# P(i_(n))
lag1 = np.histogramdd(lts1, bins=bins)[0]
lag1 = normalize(lag1)
# P(i_(n), j_(n))
lag12 = np.histogramdd(lts1 + lts2, bins=bins)[0]
lag12 = normalize(lag12)
# P(i_n+1 | i_(n), j_(n))
jcond = np.divide(joint.T, lag12.T).T
jcond = clean(jcond)
jcond = do_cpdf(jcond.T, avg_zeros).T
# P(i_n+1 | i_(n))
acond = np.divide(auto.T, lag1.T).T
acond = clean(acond)
acond = do_cpdf(acond.T, avg_zeros).T
# E[log P(i_n+1 | i_(n), j_(n)) / P(i_n+1 | i_(n))]
transfer = joint * clean(np.log(np.divide(jcond, acond)))
return transfer.sum()
def bin_by_mean(lon, lat, z, bins=10, range=None):
bins = bins[::-1]
range = range[::-1]
w_sum, _ = np.histogramdd((lat, lon), weights=z, bins=bins, range=range)
n_pts, edges = np.histogramdd((lat, lon), bins=bins, range=range)
n_pts[n_pts==0] = np.nan
return (w_sum/n_pts), n_pts, edges[1], edges[0]
def bin_by_mean(lon, lat, z, bins=10, range=None):
bins = bins[::-1]
range = range[::-1]
wsum, _ = np.histogramdd((lat, lon), weights=z, bins=bins, range=range)
ppbin, edges = np.histogramdd((lat, lon), bins=bins, range=range)
#ppbin[ppbin==0] = np.nan
#ppbin = np.ma.masked_equal(ppbin, 0)
return (wsum/ppbin), ppbin, edges[1], edges[0]
def chi2div(y, x, idx1, idx2, edges):
n1, edges1 = np.histogramdd([y[idx1], x[idx1]], bins=edges)
n2, edges2 = np.histogramdd([y[idx2], x[idx2]], bins=edges)
chi2d = sum(len(idx1)*pow(n1 - n2, 2)/(2*n2*(len(idx1)-n2)))
#chi2 = m*(n1 - n2).^2./(2*n2.*(m-n2));
return chi2d
def test_filter_minmax(self):
"""
"""
result_c = histogramnd(self.sample,
self.histo_range,
self.n_bins,
weights=self.weights,
last_bin_closed=True,
weight_min=self.filter_min,
weight_max=self.filter_max)
# !!!!!!!!!!!!!!!!!!!!!!!!!!!!
filter_min = self.dtype_weights(self.filter_min)
filter_max = self.dtype_weights(self.filter_max)
weight_idx = _get_in_range_indices(self.weights,
filter_min, # <------ !!!
filter_max, # <------ !!!
minop=operator.ge,
maxop=operator.le)
result_np = np.histogramdd(self.sample[weight_idx],
bins=self.n_bins,
range=self.histo_range)
result_np_w = np.histogramdd(self.sample[weight_idx],
bins=self.n_bins,
range=self.histo_range,
weights=self.weights[weight_idx])
# comparing "hits"
hits_cmp = np.array_equal(result_c[0],
result_np[0])
# comparing weights
weights_cmp = np.array_equal(result_c[1], result_np_w[0])
self.assertTrue(hits_cmp)
self.assertTrue(weights_cmp)
bins_min = [rng[0] for rng in self.histo_range]
bins_max = [rng[1] for rng in self.histo_range]
inrange_idx = _get_in_range_indices(self.sample[weight_idx],
bins_min,
bins_max,
minop=operator.ge,
maxop=operator.le)
inrange_idx = weight_idx[inrange_idx]
self.assertEqual(result_c[0].sum(), len(inrange_idx),
msg=self.state_msg)
# we have to sum the weights using the same precision as the
# histogramnd function
weights_sum = self.weights[inrange_idx].astype(result_c[1].dtype).sum()
self.assertTrue(self.array_compare(result_c[1].sum(), weights_sum),
msg=self.state_msg)
def feature_dist(self):
pcounts, e = np.histogramdd(self.pfeatures.view((np.float64, len(self.pfeatures.dtype.names))), bins=self.edges[:-1])
hcounts, e = np.histogramdd(self.hfeatures.view((np.float64, len(self.hfeatures.dtype.names))), bins=self.edges[:-1])
#Probability that a halo is present at a particle with features in a particular bin of feature space
#is the number of halos in that bin of feature space over the number of particles in that bin of
#feature space
self.php = hcounts/pcounts
def test_inf_edges(self):
"""Test using +/-inf bin edges works. See #1788."""
x = np.arange(6).reshape(3, 2)
expected = np.array([[1, 0], [0, 1], [0, 1]])
h, e = np.histogramdd(x, bins=[3, [-np.inf, 2, 10]])
assert_allclose(h, expected)
h, e = np.histogramdd(x, bins=[3, np.array([-1, 2, np.inf])])
assert_allclose(h, expected)
h, e = np.histogramdd(x, bins=[3, [-np.inf, 3, np.inf]])
assert_allclose(h, expected)
def plot_lum(self):
#print 'reading file'
#col = rdcol.read('./catalogs/fg_lum_cat/lum_catalog.dat',1,2)
print 'rounding'
self.z_li = np.round(self.z_l,3)*1000-100
self.z_li = self.z_li.astype(int)
print 'size of self.tmag_l '+str(self.tmag_l.shape)
print 'size of self.z_l '+str(self.z_li.shape)
print 'self.z_l'
print self.z_l[0:50]
print 'self.z_li'
print self.z_li[0:50]
print 'self.distances[z_l]'
print self.distances[self.z_li[0:50]]
self.absmag = 5.0 + self.tmag_l - 5.0*np.log10(self.distances[self.z_li])
self.lum_l = 10**((4.65-self.absmag)/2.5)#AB system SDSS r-band Solar AbsMagnitude
#self.lum_l = np.asarray(col['luminosity'])
#self.dec_l = np.asarray(col['dec'])
#self.ra_l = np.asarray(col['ra'])
self.lum_l = self.lum_l - np.mean(self.lum_l.ravel()) #Subtract out mean
self.kappa = self.kappa - np.mean(self.kappa.ravel())
print 'pixelizing'
self.lum2dw, edges = np.histogramdd(np.array([self.dec_l,self.ra_l]).T,
bins=(self.bin_dec, self.bin_ra),
weights=self.lum_l)
self.grid, edges = np.histogramdd(np.array([self.dec_l,self.ra_l]).T,
bins=(self.bin_dec, self.bin_ra))
self.lum2d = self.lum2dw/self.grid
#self.tmag2d, edges = np.histogramdd(np.array([self.dec_l,self.ra_l]).T,
# bins=(self.bin_dec, self.bin_ra),
# weights=self.tmag_l)
self.kappa2dw, edges = np.histogramdd(np.array([self.dec_l,self.ra_l]).T,
bins=(self.bin_dec, self.bin_ra),
weights=self.kappa)
self.kappa2d = self.kappa2dw/self.grid
self.mass2d = self.get_mass_from_kappa()
self.mass2d = self.mass2d - np.mean(self.mass2d.ravel())
print 'saving'
self.save_fits_image(self.lum2d,'/home/dbrout/bccml/maps/luminosity/lum_density'+str(self.file_root)+'.fits')
#self.save_fits_image(self.tmag2d,'/home/dbrout/bccml/maps/luminosity/mag_density.fits')
self.save_fits_image(self.kappa2d,'/home/dbrout/bccml/maps/luminosity/kappa_density'+str(self.file_root)+'.fits')
self.save_fits_image(self.mass2d,'/home/dbrout/bccml/maps/luminosity/mass'+str(self.file_root)+'.fits')
print 'plotting 3'
#plt.figure()
#n, bins, patches = plt.hist(self.lum2d[(self.lum2d < .4*10**10) & (self.lum2d > 1001.0)],100,log=True, histtype='bar')
#plt.xlabel('Solar Luminosity')
#plt.ylabel('# of Pixels')
#print 'saving'
#plt.savefig('/home/dbrout/bccml/maps/luminosity/lum_hist.png')
return
def test_inf_edges(self):
# Test using +/-inf bin edges works. See #1788.
with np.errstate(invalid='ignore'):
x = np.arange(6).reshape(3, 2)
expected = np.array([[1, 0], [0, 1], [0, 1]])
h, e = np.histogramdd(x, bins=[3, [-np.inf, 2, 10]])
assert_allclose(h, expected)
h, e = np.histogramdd(x, bins=[3, np.array([-1, 2, np.inf])])
assert_allclose(h, expected)
h, e = np.histogramdd(x, bins=[3, [-np.inf, 3, np.inf]])
assert_allclose(h, expected)
def mopso(f, in_dimensions, pop_size, repo_size, grid_partitions=10, w=0.4): population = np.array([[np.random.random() * (upper_bound - lower_bound) + lower_bound for (upper_bound, lower_bound) in zip(upper_bounds, lower_bounds)] for _ in range(pop_size)]) population_velocity = np.zeros((pop_size, in_dimensions)) objectives = np.apply_along_axis(g, 1, population) repository, repository_objectives = get_champions(population, objectives) best_historical_individuals = np.copy(population) best_historical_objectives = np.copy(objectives) w = 0.4 bins = np.array([np.linspace(lower_bounds[i], upper_bounds[i], num=grid_partitions + 1) for i in range(in_dimensions)]) for _ in range(iterations): print "Iteration %d" % _ coarse_map = np.histogramdd(population, bins=bins)[0] population_loneliness = 1.0 / np.array([coarse_map[tuple(np.array([min(max(np.digitize(np.array([coordinate]), bin)[0], 0), grid_partitions) for coordinate, bin in zip(population[i], bins)]) - 1)] for i in range(len(population))]) bests = np.random.choice(pop_size, pop_size, p=population_loneliness/np.sum(population_loneliness)) population_velocity = w * population_velocity + np.random.random(pop_size)[:, np.newaxis] * (best_historical_individuals - population) + np.random.random(pop_size)[:, np.newaxis] * (population[bests] - population) population = np.clip(population + population_velocity, lower_bounds, upper_bounds) objectives = np.apply_along_axis(g, 1, population) champions, champions_objectives = get_champions(population, objectives) repository, repository_objectives = get_champions(repository + champions, repository_objectives + champions_objectives) if len(repository) > repo_size: coarse_map = np.histogramdd(population, bins=bins)[0] population_loneliness = 1.0 / np.array([coarse_map[tuple(np.array([min(max(np.digitize(np.array([coordinate]), bin)[0], 0), grid_partitions) for coordinate, bin in zip(population[i], bins)]) - 1)] for i in range(len(population))]) surviving_indices = np.argsort(population_loneliness)[:repo_size] repository = [repository[i] for i in surviving_indices] repository_objectives = [repository_objectives[i] for i in surviving_indices] for i in range(len(population)): dominance_ = dominance(objectives[i], best_historical_objectives[i]) if dominance_ == 1 or (dominance_ == 0 and np.random.random() > 0.5): best_historical_individuals[i] = population[i] best_historical_objectives[i] = objectives[i] # pl.clf() # pl.plot(np.array(repository_objectives)[:, 0], np.array(repository_objectives)[:, 1], "o") # pl.pause(0.001) return (np.array(repository), np.array(repository_objectives))
def test_reuse_cumul_float(self):
"""
"""
n_bins = np.array(self.n_bins, ndmin=1)
if len(self.sample.shape) == 2:
if len(n_bins) == self.sample.shape[1]:
shp = tuple([x for x in n_bins])
else:
shp = (self.n_bins,) * self.sample.shape[1]
cumul = np.zeros(shp, dtype=np.float32)
else:
shp = (self.n_bins,)
cumul = np.zeros(shp, dtype=np.float32)
result_c_1 = histogramnd(self.sample,
self.histo_range,
self.n_bins,
weights=self.weights,
last_bin_closed=True,
weighted_histo=cumul)
result_np_1 = np.histogramdd(self.sample,
bins=self.n_bins,
range=self.histo_range)
result_np_w_1 = np.histogramdd(self.sample,
bins=self.n_bins,
range=self.histo_range,
weights=self.weights)
# comparing "hits"
hits_cmp = np.array_equal(result_c_1[0],
result_np_1[0])
self.assertTrue(hits_cmp, msg=self.state_msg)
self.assertEqual(result_c_1[1].dtype, np.float32, msg=self.state_msg)
bins_min = [rng[0] for rng in self.histo_range]
bins_max = [rng[1] for rng in self.histo_range]
inrange_idx = _get_in_range_indices(self.sample,
bins_min,
bins_max,
minop=operator.ge,
maxop=operator.le)
weights_sum = \
self.weights[inrange_idx].astype(np.float32).sum(dtype=np.float64)
self.assertTrue(np.allclose(result_c_1[1].sum(dtype=np.float64),
weights_sum), msg=self.state_msg)
self.assertTrue(np.allclose(result_c_1[1].sum(dtype=np.float64),
result_np_w_1[0].sum(dtype=np.float64)),
msg=self.state_msg)
def bhattacharyya_coefficient_discrete(data1, data2, bins=10):
'''
Computing Bhattacharyya coefficient using (multidimensional)
histograms.
'''
hist_range = zip(np.minimum(np.min(data1, axis=0), np.min(data2, axis=0)),
np.maximum(np.max(data1, axis=0), np.max(data2, axis=0)))
bins_total_volume = np.prod([ma-mi for mi, ma in hist_range])
hist1, _ = np.histogramdd(data1, bins=bins, range=hist_range, normed=True)
hist2, _ = np.histogramdd(data2, bins=bins, range=hist_range, normed=True)
return np.mean(np.sqrt(hist1*hist2))*bins_total_volume
def shape_context2d(p, mean_dist=None, r_inner=1./8, r_outer=2., nbins_r=5, nbins_theta=12, outliers=None, sparse=False):
"""
Computes the shape-context log-polar histograms at each point in p -- the point cloud.
p is a Nxd matrix of points.
"""
N, d = p.shape
assert d==2, "shape_context2d requires 2D point data."
pt_nd = p
# compute the coordinates : r,theta, phi
dists_nn = ssd.cdist(pt_nd, pt_nd)
if mean_dist==None:
mean_dist = np.mean(dists_nn)
dists_nn = dists_nn/mean_dist
# theta_nn are in [0,2pi)
dx_nn, dy_nn = pt_nd.T[:,None,:]-pt_nd.T[:,:,None]
theta_nn = np.arctan2(dy_nn, dx_nn)
theta_nn = np.fmod(np.fmod(theta_nn,2*np.pi)+2*np.pi,2*np.pi)
# define histogram edges
r_edges = np.concatenate(([0], loglinspace(r_inner, r_outer, nbins_r)))
theta_edges = np.linspace(0, 2*np.pi, nbins_theta+1)
combined_2nn = np.array([dists_nn, theta_nn])
# compute the bins : 4 dimensional matrix.
# r,t,p are the number of bins of radius, theta, phi
sc_nrt = np.zeros((N, nbins_r, nbins_theta))
for i in xrange(N):
hist, edges = np.histogramdd(combined_2nn[:,i,:].T, bins=[r_edges, theta_edges])
sc_nrt[i,:,:] = hist
sc_nrt = sc_nrt.reshape(N, nbins_r*nbins_theta)
# just r binning:
rbins_nr = np.zeros((N, nbins_r))
for i in xrange(N):
dat = combined_2nn[0,i,:].T
hist, edges = np.histogramdd(dat, bins=[r_edges])
rbins_nr[i,:] = hist
# just theta binning
tbins_nt = np.zeros((N, nbins_theta))
for i in xrange(N):
dat = combined_2nn[1,i,:].T
hist, edges = np.histogramdd(dat, bins=[theta_edges])
tbins_nt[i,:] = hist
return (sc_nrt, mean_dist, dists_nn, theta_nn, rbins_nr, tbins_nt)
def test_inf_edges(self):
"""Test using +/-inf bin edges works. See #1788."""
olderr = np.seterr(invalid='ignore')
try:
x = np.arange(6).reshape(3, 2)
expected = np.array([[1, 0], [0, 1], [0, 1]])
h, e = np.histogramdd(x, bins=[3, [-np.inf, 2, 10]])
assert_allclose(h, expected)
h, e = np.histogramdd(x, bins=[3, np.array([-1, 2, np.inf])])
assert_allclose(h, expected)
h, e = np.histogramdd(x, bins=[3, [-np.inf, 3, np.inf]])
assert_allclose(h, expected)
finally:
np.seterr(**olderr)
def f(data,data_wigglez,label):
n,edges=np.histogramdd(DPM,bins=(bin_x,bin_y,bin_z))
n[n==0]=1
T=np.histogramdd(DPM,bins=(bin_x,bin_y,bin_z),weights=data.reshape(-1))[0]
T_wigglez=np.histogramdd(DPM,bins=(bin_x,bin_y,bin_z),weights=data_wigglez.reshape(-1))[0]
T/=n
T_wigglez/=n
# T*=Nx*Ny*Nz/T.sum()
# T_wigglez*=Nx*Ny*Nz/T_wigglez.sum()
Hx=np.abs(edges[0][1]-edges[0][0])
Hy=np.abs(edges[1][1]-edges[1][0])
Hz=np.abs(edges[2][1]-edges[2][0])
k_x=np.fft.fftfreq(Nx,1./Nx)
k_y=np.fft.fftfreq(Ny,1./Ny)
k_z=np.fft.fftfreq(Nz,1./Nz)
T_k1=np.fft.fftn(T)
T_k2=np.fft.fftn(T_wigglez)
# window_k = np.sinc(1. / Nx * k_x[:,None,None]) * np.sinc(1. / Ny * k_y[None,:,None]) * np.sinc(1. / Nz * k_z[None,None,:])
# T_k1/=window_k
# T_k2/=window_k
Pk_cro=Hx*Hy*Hz/Nx/Ny/Nz*((T_k1.conjugate()*T_k2+T_k2.conjugate()*T_k1)/2).real
Pk_gbt=Hx*Hy*Hz/Nx/Ny/Nz*(np.abs(T_k1)**2)
Pk_wig=Hx*Hy*Hz/Nx/Ny/Nz*(np.abs(T_k2)**2)
k_mag=np.sqrt((2*np.pi/Hx/Nx*k_x[:,None,None])**2+(2*np.pi/Hy/Ny*k_y[None,:,None])**2+(2*np.pi/Hz/Nz*k_z[None,None,:])**2)
Pk_cro*=(k_mag**3.)/(2*np.pi**2) # delta(k)^2
Pk_gbt*=(k_mag**3.)/(2*np.pi**2) # delta(k)^2
Pk_wig*=(k_mag**3.)/(2*np.pi**2) # delta(k)^2
#################################################################################
bin=50
edges=np.linspace(k_mag.min(),k_mag.max(),bin+1,endpoint=True)
n=np.histogram(k_mag,edges)[0]
k_bin=np.histogram(k_mag,edges,weights=k_mag)[0]
pk_cro=np.histogram(k_mag,edges,weights=Pk_cro)[0]
pk_gbt=np.histogram(k_mag,edges,weights=Pk_gbt)[0]
pk_wig=np.histogram(k_mag,edges,weights=Pk_wig)[0]
plt.figure('Pk')
plt.semilogy(k_bin/n,pk_cro/n,'x--',label=label+'_cro')
# plt.semilogy(k_bin/n,pk_gbt/n,'.-',label=label+'_gbt')
# plt.semilogy(k_bin/n,pk_wig/n,'o-',label=label+'_wig')
plt.ylabel('$\Delta ^2(k)$')
plt.xlabel('$k$')
plt.legend()
def fit(self, signal_data, bckgrd_data):
"""
set up classifier ("training")
"""
# some examples of what might be useful:
# 1. signal and background histograms with same binning
_, self.edges = np.histogramdd(np.vstack([signal_data, bckgrd_data]), bins=10)
self.signal_hist, _ = np.histogramdd(signal_data, bins=self.edges)
self.bckgrd_hist, _ = np.histogramdd(bckgrd_data, bins=self.edges)
# 2. mean and covariance matrix
self.signal_mean = np.mean(signal_data, axis=0)
self.signal_cov = np.cov(signal_data.T)
self.bckgrd_mean = np.mean(bckgrd_data, axis=0)
self.bckgrd_cov = np.cov(bckgrd_data.T)
# 3. print stuff
print 'signal_mean = ', self.signal_mean
print 'background_mean = ', self.bckgrd_mean
#self.signal_cov.print()
#self.bckgrd_cov.print()
# 4. create Histos with Gaussian distributions
#for varpair in varpairs:
#c = TCanvas("c","c",800,600)
nbins = 5
minx=min(data[:,0])
maxx=max(data[:,0])
miny=min(data[:,1])
maxy=max(data[:,1])
#if varpair[0]!=varpair[1]:
signal2d = TH2F("signal2d","",nbins,minx,maxx,nbins,miny,maxy)
background2d = TH2F("signal2d","",nbins,minx,maxx,nbins,miny,maxy)
xs, ys = np.random.multivariate_normal(self.signal_mean, self.signal_cov, 5000).T
xb, yb = np.random.multivariate_normal(self.bckgrd_mean, self.bckgrd_cov, 5000).T
for x1,y1,x2,y2 in zip(xs,ys,xb,yb):
signal2d.Fill(x1,y1)
background2d.Fill(x2,y2)
hist2dges = signal2d.Clone()
hist2dges.Add(background2d)
ratio = signal2d.Clone()
ratio.Divide(background2d)
ratio.Draw("colz")
self.clf=ratio
#c.Print("../../Ex_10-2_hist2d_setosa_prob.pdf")
#c.Close()
#c.Print("../../Ex_10-2_hist2d_setosa_prob.pdf]")
self.signal_distribution = np.random.multivariate_normal(self.signal_mean, self.signal_cov, 5000).T
self.bckgrd_distribution = np.random.multivariate_normal(self.bckgrd_mean, self.bckgrd_cov, 5000).T
def bhattacharyya_probability_coefficient_dd(v1,v2,bins,normed=False):
"""
# return the bhattacharyya distance between arbitrary-dimensional
#probabilities, see bhattacharyya_probability_coefficient
Args:
v<1/2>: two arbitrary-dimensional lists to compare
bins: how to bin them
Returns:
bhattacharyya distance, see bhattacharyya_probability_coefficient
"""
histogram_kwargs = dict(bins=bins,weights=None,normed=normed)
v1_hist,v1_edges = np.histogramdd(sample=v1,**histogram_kwargs)
v2_hist,v2_edges = np.histogramdd(sample=v2,**histogram_kwargs)
return bhattacharyya_probability_coefficient(v1_hist,v2_hist)
def kldiv_cs_model(prediction, fm):
"""
Computes Chao-Shen corrected KL-divergence between prediction
and fdm made from fixations in fm.
Parameters :
prediction : np.ndarray
a fixation density map
fm : FixMat object
"""
# compute histogram of fixations needed for ChaoShen corrected kl-div
# image category must exist (>-1) and image_size must be non-empty
assert(len(fm.image_size) == 2 and (fm.image_size[0] > 0) and
(fm.image_size[1] > 0))
assert(-1 not in fm.category)
# check whether fixmat contains fixations
if len(fm.x) == 0:
return np.NaN
(scale_factor, _) = calc_resize_factor(prediction, fm.image_size)
# this specifies left edges of the histogram bins, i.e. fixations between
# ]0 binedge[0]] are included. --> fixations are ceiled
e_y = np.arange(0, np.round(scale_factor*fm.image_size[0]+1))
e_x = np.arange(0, np.round(scale_factor*fm.image_size[1]+1))
samples = np.array(zip((scale_factor*fm.y), (scale_factor*fm.x)))
(fdm, _) = np.histogramdd(samples, (e_y, e_x))
# compute ChaoShen corrected kl-div
q = np.array(prediction, copy = True)
q[q == 0] = np.finfo(q.dtype).eps
q /= np.sum(q)
(H, pa, la) = chao_shen(fdm)
q = q[fdm > 0]
cross_entropy = -np.sum((pa * np.log2(q)) / la)
return (cross_entropy - H)
def fit(self, X, sample_weight=None, **kwargs):
# Checks
X = check_array(X)
if sample_weight is not None and len(sample_weight) != len(X):
raise ValueError
# Compute histogram and edges
h, e = np.histogramdd(X, bins=self.bins, range=self.range,
weights=sample_weight, normed=True)
# Add empty bins for out of bound samples
for j in range(X.shape[1]):
h = np.insert(h, 0, 0., axis=j)
h = np.insert(h, h.shape[j], 0., axis=j)
e[j] = np.insert(e[j], 0, -np.inf)
e[j] = np.insert(e[j], len(e[j]), np.inf)
if X.shape[1] == 1 and self.interpolation:
inputs = e[0][2:-1] - (e[0][2] - e[0][1]) / 2.
inputs[0] = e[0][1]
inputs[-1] = e[0][-2]
outputs = h[1:-1]
self.interpolation_ = interp1d(inputs, outputs,
kind=self.interpolation,
bounds_error=False,
fill_value=0.)
self.histogram_ = h
self.edges_ = e
self.ndim_ = X.shape[1]
return self
def __init__(self,weighter,sim_data,bins=None,range=None,aux_data=None):
self.weighter = weighter
self.sim_data = sim_data
self.aux_data = aux_data
self.ndim = self.sim_data.shape[1]
if bins is None:
bins = [1]*sim_data.shape[1]
self.bin_edges = np.histogramdd(self.sim_data,bins=bins,range=range)[1]
self.bins = [len(b)-1 for b in self.bin_edges]
self.multi_indices = [np.digitize(self.sim_data[:,col],self.bin_edges[col])-1 for col in xrange(self.ndim)]
cut = np.prod([(self.multi_indices[col]<self.bins[col])&(self.multi_indices[col]>-1) for col in xrange(self.ndim)],axis=0).astype(bool)
self.sim_data = self.sim_data[cut]
self.aux_data = self.aux_data[cut]
self.multi_indices = [m[cut] for m in self.multi_indices]
self.indices = np.ravel_multi_index(self.multi_indices,self.bins)
cut = np.argsort(self.indices)
self.sim_data = self.sim_data[cut]
self.aux_data = self.aux_data[cut]
self.multi_indices = [m[cut] for m in self.multi_indices]
self.indices = self.indices[cut]
self.weighter = weighter
def bin_catalog_data(catalog, freq_axis, ra_axis,
dec_axis, debug=False, use_histogramdd=False):
"""
bin catalog data onto a grid in RA, Dec, and frequency
This currently assumes that all of the axes are uniformly spaced
"""
catalog_frequencies = cc.freq_21cm_MHz * 1.e6 / (1 + catalog['z'])
num_catalog = catalog.size
sample = np.zeros((num_catalog, 3))
sample[:, 0] = catalog_frequencies
sample[:, 1] = catalog['RA']
sample[:, 2] = catalog['Dec']
freq_edges = binning.find_edges(freq_axis)
ra_edges = binning.find_edges(ra_axis)
dec_edges = binning.find_edges(dec_axis)
if debug:
binning.print_edges(sample[:, 0], freq_edges, "frequency")
binning.print_edges(sample[:, 1], ra_edges, "RA")
binning.print_edges(sample[:, 2], dec_edges, "Dec")
print sample, freq_edges, ra_edges, dec_edges
if use_histogramdd:
count_cube, edges = np.histogramdd(sample,
bins=[freq_edges,
ra_edges, dec_edges])
print edges
else:
count_cube = binning.histogram3d(sample, freq_edges,
ra_edges, dec_edges)
return count_cube
def learn(self, features, weights=None):
assert len(features) == len(self.edges)
features = np.array(features)
h, _ = np.histogramdd(features.T, bins=self.edges, weights=weights)
self.histogram += h
def mkhist(fname, xmin, xmax, ymin, ymax, deltax, deltay, ihist, jtemp,
k, cx):
xdata = []
udata = []
vdata = []
count = 0
for line in open(fname):
time, x, u, v = map(float, line.strip().split()[:4])
xdata.append(x)
udata.append(u)
vdata.append(v)
if debug and len(xdata) > 10000: break
if n_max and len(xdata) > n_max: break
x = np.array(xdata)
u = np.array(udata)
v = np.array(vdata)
u = u - k * (x - cx)**2 #+ press * v
xbins = [xmin + i * deltax for i in range(nbinx + 1)]
ubins = [umin + i * deltau for i in range(nbinu + 1)]
vbins = [vmin + i * deltav for i in range(nbinv + 1)]
data = np.array((x, u, v)).transpose()
hist[ihist,
jtemp], edges = np.histogramdd(data,
bins=(xbins, ubins, vbins),
range=((xmin, xmax),
(umin, umax), (vmin,
vmax)))
nb_data[ihist, jtemp] = np.sum(hist[ihist, jtemp])
if data_range[0][0] is None or np.min(x) < data_range[0][0]:
data_range[0][0] = np.min(x)
if data_range[0][1] is None or np.max(x) > data_range[0][1]:
data_range[0][1] = np.max(x)
if data_range[1][0] is None or np.min(u) < data_range[1][0]:
data_range[1][0] = np.min(u)
if data_range[1][1] is None or np.max(u) > data_range[1][1]:
data_range[1][1] = np.max(u)
if data_range[2][0] is None or np.min(v) < data_range[2][0]:
data_range[2][0] = np.min(v)
if data_range[2][1] is None or np.max(v) > data_range[2][1]:
data_range[2][1] = np.max(v)
xedges = edges[0]
uedges = edges[1]
print 'statistics for timeseries # ', ihist
print 'minx:', '%8.3f' % np.min(x), 'maxx:', '%8.3f' % np.max(x)
print 'average x', '%8.3f' % np.average(
x), 'rms x', '%8.3f' % np.std(x)
print 'minu:', '%8.3f' % np.min(u), 'maxu:', '%8.3f' % np.max(u)
print 'average u', '%8.3f' % np.average(
u), 'rms u', '%8.3f' % np.std(u)
print 'statistics for histogram # ', ihist
print int(np.sum(hist[ihist, jtemp])), 'points in the histogram x'
print 'average x', '%8.3f' % (np.sum([
hist[ihist, jtemp, i, :] * (xedges[i] + xedges[i + 1]) / 2
for i in range(nbinx)
]) / np.sum(hist[ihist, jtemp]))
print 'average u', '%8.3f' % (np.sum([
hist[ihist, jtemp, :, i] * (uedges[i] + uedges[i + 1]) / 2
for i in range(nbinu)
]) / np.sum(hist[ihist, jtemp]))
print
def create_sampler(self,
mode='hist',
p=None,
transforms=None,
dst='sampler',
**kwargs):
""" Create samplers for every cube and store it in `samplers`
attribute of passed dataset. Also creates one combined sampler
and stores it in `sampler` attribute of passed dataset.
Parameters
----------
mode : str or Sampler
Type of sampler to be created.
If 'hist', then sampler is estimated from given labels.
If 'numpy', then sampler is created with `kwargs` parameters.
If instance of Sampler is provided, it must generate points from unit cube.
p : list
Weights for each mixture in final sampler.
transforms : dict
Mapping from indices to callables. Each callable should define
way to map point from absolute coordinates (X, Y world-wise) to
cube local specific and take array of shape (N, 4) as input.
Notes
-----
Passed `dataset` must have `geometries` and `labels` attributes if you want to create HistoSampler.
"""
#pylint: disable=cell-var-from-loop
lowcut, highcut = [0, 0, 0], [1, 1, 1]
transforms = transforms or dict()
samplers = {}
if not isinstance(mode, dict):
mode = {ix: mode for ix in self.indices}
for ix in self.indices:
if isinstance(mode[ix], Sampler):
sampler = mode[ix]
elif mode[ix] == 'numpy':
sampler = NumpySampler(**kwargs)
elif mode[ix] == 'hist':
point_cloud = getattr(self, 'point_clouds')[ix]
geom = getattr(self, 'geometries')[ix]
offsets = np.array([geom.ilines_offset, geom.xlines_offset, 0])
cube_shape = np.array(geom.cube_shape)
to_cube = lambda points: (points[:, :3] - offsets) / cube_shape
default = lambda points: to_cube(geom.height_correction(points)
)
transform = transforms.get(ix) or default
cube_array = transform(point_cloud)
# Size of ticks along each respective axis
default_bins = cube_shape // np.array([5, 20, 20])
bins = kwargs.get('bins') or default_bins
sampler = HistoSampler(np.histogramdd(cube_array, bins=bins))
else:
sampler = NumpySampler('u', low=0, high=1, dim=3)
sampler = sampler.truncate(low=lowcut, high=highcut)
samplers.update({ix: sampler})
self.samplers = samplers
# One sampler to rule them all
p = p or [1 / len(self) for _ in self.indices]
sampler = 0 & NumpySampler('n', dim=4)
for i, ix in enumerate(self.indices):
sampler_ = (ConstantSampler(ix)
& samplers[ix].apply(lambda d: d.astype(np.object)))
sampler = sampler | (p[i] & sampler_)
setattr(self, dst, sampler)
return self
def grid(mesh,
pos=None,
wts=None,
style='CIC',
ingrid=None,
logscale=False,
dolog=False,
smooth=None,
R=0,
dologsm=False):
mesh.push()
if pos is None:
if ingrid is None:
return 'No input given! Give either position or a grid to operate on.'
else:
mesh.clear()
mesh.real[:] = ingrid[:]
if logscale:
mesh.real[:] = 10**mesh.real[:]
mesh.real[mesh.real == 1] = 0
else:
if wts is None:
wts = numpy.ones(pos.shape[0])
mesh.clear()
if style == 'CIC':
mesh.paint(pos, mass=wts)
elif style == 'NN':
nc = mesh.real.shape[0]
mesh.real[:], dummy = numpy.histogramdd(pos,
bins=(nc, nc, nc),
weights=wts)
toret = mesh.real.copy()
if dolog:
dummy = numpy.empty_like(toret)
dummy[:] = toret[:]
dummy[dummy <= 0] = 1
dummy = log10(dummy)
toret2 = dummy.copy()
if smooth is not None:
mesh.clear()
mesh.real[:] = toret[:]
mesh.r2c()
if smooth == 'Fingauss':
mesh.transfer([tools.pmFingauss(R)])
elif smooth == 'Gauss':
mesh.transfer([tools.pmGauss(R)])
elif smooth == 'Tophat':
mesh.transfer([tools.pmTophat(R)])
else:
return 'Smoothing kernel not defined'
mesh.c2r()
toret3 = mesh.real.copy()
if dologsm:
dummy = numpy.empty_like(toret3)
dummy[:] = toret3[:]
dummy[dummy <= 0] = 1
dummy = log10(dummy)
toret4 = dummy.copy()
mesh.pop()
if dolog:
if smooth:
if dologsm:
return toret, toret2, toret3, toret4
else:
return toret, toret2, toret3
else:
return toret, toret2
elif smooth:
if dologsm:
return toret, toret3, toret4
else:
return toret, toret3
else:
return toret
# Read data
spikes = neurons.getSpikes()
sim.end()
# Calculate x and y coordinates corresponding to each spike
neuron_x = spikes[:, 0] % cost_image.shape[1]
neuron_y = spikes[:, 0] // cost_image.shape[1]
# Get time of last spike
end_time = int(np.amax(spikes[:, 1]))
# Convert spike times to 3D spike matrix
matrix, _ = np.histogramdd(
(neuron_y, neuron_x, spikes[:, 1]),
bins=(range(cost_image.shape[0] + 1), range(cost_image.shape[1] + 1),
range(end_time + 1)))
# Check spike vector only ever has ones and zeros
assert np.amax(matrix) == 1.0
print "End time:%u" % end_time
# Create RGBA image to display path information
path_image = np.zeros((cost_image.shape[0], cost_image.shape[1], 4))
# Add pixels indicating stim and end to image
path_image[stim_y, stim_x] = (0.0, 1.0, 0.0, 1.0)
path_image[target_y, target_x] = (0.0, 0.0, 1.0, 1.0)
# Backtrack to find path
x = target_x
def extract_particle_report(args, particle_type, run_dir, data_file):
xarg = args[0]
yarg = args[1]
nbins = args[2]
opmd = _opmd_time_series(data_file)
data_list = opmd.get_particle(
var_list=[xarg, yarg],
species=particle_type,
iteration=data_file.iteration,
select=None,
output=True,
plot=False,
)
with h5py.File(data_file.filename) as f:
data_list.append(main.read_species_data(f, particle_type, 'w', ()))
select = _particle_selection_args(args)
if select:
with h5py.File(data_file.filename) as f:
main.apply_selection(f, data_list, select, particle_type, ())
xunits = ' [m]' if len(xarg) == 1 else ''
yunits = ' [m]' if len(yarg) == 1 else ''
if len(data_list[0]) < 2:
return {
'x_range': [0, 1e-6, 2],
'y_range': [0, 1e-6, 2],
'x_label': '{}{}'.format(xarg, xunits),
'y_label': '{}{}'.format(yarg, yunits),
'title': 't = {}'.format(_iteration_title(opmd, data_file)),
'z_matrix': [[0, 0], [0, 0]],
'frameCount': data_file.num_frames,
}
if len(xarg) == 1:
data_list[0] /= 1e6
if len(yarg) == 1:
data_list[1] /= 1e6
if xarg == 'z':
data_list = _adjust_z_width(data_list, data_file)
#TODO(pjm): need range checking in type, consolidate with template.elegant
nbins = int(nbins)
if nbins <= 0:
nbins = 1
elif nbins > _HISTOGRAM_BINS_MAX:
nbins = _HISTOGRAM_BINS_MAX
hist, edges = numpy.histogramdd([data_list[0], data_list[1]],
int(nbins),
weights=data_list[2])
return {
'x_range': [float(edges[0][0]),
float(edges[0][-1]),
len(hist)],
'y_range': [float(edges[1][0]),
float(edges[1][-1]),
len(hist[0])],
'x_label': '{}{}'.format(xarg, xunits),
'y_label': '{}{}'.format(yarg, yunits),
'title': 't = {}'.format(_iteration_title(opmd, data_file)),
'z_matrix': hist.T.tolist(),
'frameCount': data_file.num_frames,
}
ax = plt.subplot(111)
weights = mc["ow"] * mc["trueE"]**-3.0
print("True", max(x), min(x), "Reco", max(y), min(y))
print("frac over", end=" ")
mask = x > y
print(np.sum(weights[mask]) / np.sum(weights))
max_err = 1.0
bins = np.linspace(0.0, max_err, 11)
hist_2d, bin_edges = np.histogramdd((x, y),
bins=(bins, bins),
weights=weights)
meds = []
centers = 0.5 * (bin_edges[0][:-1] + bin_edges[0][1:])
for i, row in enumerate(hist_2d.T):
mask = np.logical_and(y > bin_edges[0][i], y < bin_edges[0][i + 1])
row /= np.sum(weights[mask])
mc_cut = x[mask]
# meds.append(np.median(mc_cut))
X, Y = np.meshgrid(bin_edges[0], bin_edges[1])
cbar = ax.pcolormesh(X, Y, hist_2d)
range = [0.0, max_err]
plt.plot(range, range, color="white", linestyle="--")
def compute_color_distributions(
self,
labels="default",
color_rep=["jzazbz", "hsv", "rgb"],
spacing=36,
num_bins=8,
num_channels=3,
Jz_min=0.0,
Jz_max=0.167,
Az_min=-0.1,
Az_max=0.11,
Bz_min=-0.156,
Bz_max=0.115,
h_max=360,
rgb_max=255,
):
"""
Calculates color distributions for each word in a dictionary
Args:
self (class instance): ImageAnalysis class instance
labels (string): if "default" grabs dictionary keys as labels
color_rep(array): colorspaces to calculate distributions in
spacing(int): hue spacing for HSV distribution (in degrees)
num_bins(int): number of bins to calculate 3D distributions in
num_channels(int): number of color channels
*z_min (*z_max) (float): minimum (maximum) of JzAzBz coordinates
h_max (int): maximum hue (in degrees)
rgb_max (int): maximum value in RGB
Returns:
self (class instace): ImageAnalysis class instance containing JzAzBz, HSV, and RGB distributions for each word
"""
dims = self.image_data.dims
if labels == "default":
labels = self.labels_list
labels = labels if isinstance(labels, list) else [labels]
self.jzazbz_dist_dict, self.hsv_dist_dict = {}, {}
self.rgb_ratio_dict, self.rgb_dist_dict = {}, {}
color_rep = [i.lower() for i in color_rep]
if "jzazbz" in color_rep:
self.jzazbz_dist_dict = {}
for key in labels:
if key not in self.image_data.labels_list:
self.log.warning(f"label {key} does not exist")
continue
if key not in self.image_data.jzazbz_dict.keys():
self.image_data.store_jzazbz_from_rgb(key)
jzazbz, dist_array = [], []
imageset = self.jzazbz_dict[key]
for i in range(len(imageset)):
jzazbz.append(imageset[i])
dist = np.ravel(
np.histogramdd(
np.reshape(
imageset[i][:, :, :], (dims[0] * dims[1], num_channels)
),
bins=(
np.linspace(
Jz_min,
Jz_max,
1 + int(num_bins ** (1.0 / num_channels)),
),
np.linspace(
Az_min,
Az_max,
1 + int(num_bins ** (1.0 / num_channels)),
),
np.linspace(
Bz_min,
Bz_max,
1 + int(num_bins ** (1.0 / num_channels)),
),
),
density=True,
)[0]
)
if True in np.isnan(dist):
# Drop any dists that contain NaN
continue
dist_array.append(dist)
self.jzazbz_dist_dict[key] = dist_array
if "hsv" in color_rep:
self.h_dict, self.s_dict, self.v_dict = {}, {}, {}
self.hsv_dist_dict = {}
for key in labels:
if key not in self.image_data.labels_list:
self.log.warning(f"label {key} does not exist")
continue
imageset = self.rgb_ratio_dict[key]
dist_array, h, s, v = [], [], [], []
for i in range(len(imageset)):
hsv_array = mplcolors.rgb_to_hsv(imageset[i] / (1.0 * rgb_max))
dist = np.histogram(
1.0 * h_max * np.ravel(hsv_array[:, :, 0]),
bins=np.arange(0, h_max + spacing, spacing),
density=True,
)[0]
dist_array.append(dist)
h.append(np.mean(np.ravel(hsv_array[:, :, 0])))
s.append(np.mean(np.ravel(hsv_array[:, :, 1])))
v.append(np.mean(np.ravel(hsv_array[:, :, 2])))
self.hsv_dist_dict[key] = dist_array
self.h_dict[key], self.s_dict[key], self.v_dict[key] = h, s, v
if "rgb" in color_rep:
self.rgb_ratio_dict, self.rgb_dist_dict = {}, {}
for key in labels:
if key not in self.image_data.labels_list:
self.log.warning(f"label {key} does not exist")
continue
imageset = self.rgb_dict[key]
rgb = []
dist_array = []
for i in range(len(imageset)):
r = np.sum(np.ravel(imageset[i][:, :, 0]))
g = np.sum(np.ravel(imageset[i][:, :, 1]))
b = np.sum(np.ravel(imageset[i][:, :, 2]))
tot = 1.0 * r + g + b
rgb.append([r / tot, g / tot, b / tot])
dist = np.ravel(
np.histogramdd(
np.reshape(imageset[i], (dims[0] * dims[1], num_channels)),
bins=(
np.linspace(
0,
rgb_max,
1 + int(num_bins ** (1.0 / num_channels)),
),
np.linspace(
0,
rgb_max,
1 + int(num_bins ** (1.0 / num_channels)),
),
np.linspace(
0,
rgb_max,
1 + int(num_bins ** (1.0 / num_channels)),
),
),
density=True,
)[0]
)
dist_array.append(dist)
self.rgb_ratio_dict[key] = rgb
self.rgb_dist_dict[key] = dist_array
def histogram2d(x, y, bins=10, range=None, normed=False, weights=None):
"""
Compute the bi-dimensional histogram of two data samples.
Parameters
----------
x : array_like, shape(N,)
A sequence of values to be histogrammed along the first dimension.
y : array_like, shape(M,)
A sequence of values to be histogrammed along the second dimension.
bins : int or [int, int] or array_like or [array, array], optional
The bin specification:
* If int, the number of bins for the two dimensions (nx=ny=bins).
* If [int, int], the number of bins in each dimension (nx, ny = bins).
* If array_like, the bin edges for the two dimensions (x_edges=y_edges=bins).
* If [array, array], the bin edges in each dimension (x_edges, y_edges = bins).
range : array_like, shape(2,2), optional
The leftmost and rightmost edges of the bins along each dimension
(if not specified explicitly in the `bins` parameters):
``[[xmin, xmax], [ymin, ymax]]``. All values outside of this range
will be considered outliers and not tallied in the histogram.
normed : bool, optional
If False, returns the number of samples in each bin. If True, returns
the bin density, i.e. the bin count divided by the bin area.
weights : array_like, shape(N,), optional
An array of values ``w_i`` weighing each sample ``(x_i, y_i)``. Weights
are normalized to 1 if `normed` is True. If `normed` is False, the
values of the returned histogram are equal to the sum of the weights
belonging to the samples falling into each bin.
Returns
-------
H : ndarray, shape(nx, ny)
The bi-dimensional histogram of samples `x` and `y`. Values in `x`
are histogrammed along the first dimension and values in `y` are
histogrammed along the second dimension.
xedges : ndarray, shape(nx,)
The bin edges along the first dimension.
yedges : ndarray, shape(ny,)
The bin edges along the second dimension.
See Also
--------
histogram: 1D histogram
histogramdd: Multidimensional histogram
Notes
-----
When `normed` is True, then the returned histogram is the sample density,
defined such that:
.. math::
\\sum_{i=0}^{nx-1} \\sum_{j=0}^{ny-1} H_{i,j} \\Delta x_i \\Delta y_j = 1
where `H` is the histogram array and :math:`\\Delta x_i \\Delta y_i`
the area of bin `{i,j}`.
Please note that the histogram does not follow the Cartesian convention
where `x` values are on the abcissa and `y` values on the ordinate axis.
Rather, `x` is histogrammed along the first dimension of the array
(vertical), and `y` along the second dimension of the array (horizontal).
This ensures compatibility with `histogramdd`.
Examples
--------
>>> x, y = np.random.randn(2, 100)
>>> H, xedges, yedges = np.histogram2d(x, y, bins=(5, 8))
>>> H.shape, xedges.shape, yedges.shape
((5, 8), (6,), (9,))
We can now use the Matplotlib to visualize this 2-dimensional histogram:
>>> extent = [yedges[0], yedges[-1], xedges[-1], xedges[0]]
>>> import matplotlib.pyplot as plt
>>> plt.imshow(H, extent=extent, interpolation='nearest')
<matplotlib.image.AxesImage object at ...>
>>> plt.colorbar()
<matplotlib.colorbar.Colorbar instance at ...>
>>> plt.show()
"""
from numpy import histogramdd
try:
N = len(bins)
except TypeError:
N = 1
if N != 1 and N != 2:
xedges = yedges = asarray(bins, float)
bins = [xedges, yedges]
hist, edges = histogramdd([x, y], bins, range, normed, weights)
return hist, edges[0], edges[1]
charges = universe.atoms.charges()
# setting the skip for the traj
universe.trajectory.skip = 10
# appending atomic charges to a particular z-dimension (z-bin)
nframes = 0
for ts in universe.trajectory:
print ts.frame
nframes += 1
# Recenter z coordinates
z = universe.atoms.coordinates()[:, 2]
z -= numpy.average(z)
if charge_dist is None:
charge_dist, edges = numpy.histogramdd(z,
bins=numbins,
range=[(zmin, zmax)],
weights=charges)
else:
newhist, edges = numpy.histogramdd(z,
bins=numbins,
range=[(zmin, zmax)],
weights=charges)
charge_dist += newhist
# Normalize charge distribution
charge_dist /= nframes
# midpoints of bins
edge_1d = edges[0]
zpos = 0.5 * (edge_1d[:-1] + edge_1d[1:])
def get_correlation(self, it, list_corr_task_dic, mask_v_n, multiplier=2.):
edges = []
for setup_dictionary in list_corr_task_dic:
edges.append(self.get_edges(it, setup_dictionary))
edges = tuple(edges)
correlation = np.zeros([len(edge) - 1 for edge in edges])
nlevels = self.get_nlevels(it)
assert nlevels > 0
for rl in range(nlevels):
data = []
# ye_mask = self.get_comp_data(it, rl, plane, "Ye")
# ye_mask[ye_mask < 0.4] = 0
# ye_mask[ye_mask >= 0.4] = 1
mask = self.get_mask(it, rl, mask_v_n)
dens = self.get_comp_data(it, rl, "density")
weights = (dens * mask) * np.prod(self.get_attr(
it, rl, "delta")) * multiplier
for corr_dic in list_corr_task_dic:
tmp = self.get_comp_data(it, rl, corr_dic["v_n"])
# print("\tdata:{} | {} min:{} max:{} "
# .format(tmp.shape, corr_dic["v_n"], tmp.min(), tmp.max()))
data.append(tmp.flatten())
data = tuple(data)
#
#
# mask = self.get_data(it, rl, plane, "rl_mask")
# print("mask", mask.shape)
# dens = self.get_data(it, rl, plane, "density")
# print("dens", dens.shape)
# dens_ = dens * mask
# print("dens[mask]", dens_.shape)
# weights = dens * np.prod(self.get_attr(it, rl, plane, "delta")) * multiplier
# print(weights.shape),
# weights = weights.flatten()
# print(weights.shape)
# # print("rl:{} mass:{} masked:{}".format(rl, np.sum(weights), np.sum(weights[mask])))
# # weights = weights[mask]
# for corr_dic in list_corr_task_dic:
# v_n = corr_dic["v_n"]
# data_ = self.get_data(it, rl, plane, v_n)
# # print(data_.shape)
# data.append(data_.flatten())
# print("data: {} {}".format(data_.shape, data[-1].shape))
# # if v_n == "Q_eff_nua":
# # data[-1] = data[-1][3:-3, 3:-3]
# print("\t{} min:{} max:{} ".format(v_n, data[-1].min(), data[-1].max()))
# data = tuple(data)
try:
tmp, _ = np.histogramdd(data,
bins=edges,
weights=weights.flatten())
except ValueError:
tmp = np.zeros([len(edge) - 1 for edge in edges])
Printcolor.red("ValueError it:{} rl:{} ".format(it, rl))
correlation += tmp
if np.sum(correlation) == 0:
# print("\t")
raise ValueError("sum(corr) = 0")
return edges, correlation
def getCounts(gridResolution):
locations = np.loadtxt('pines.csv', delimiter=',')
counts, _ = np.histogramdd(locations,
bins=(gridResolution, gridResolution),
range=((0, 1.), (0., 1.)))
return counts.reshape(-1, 1)
def time_fine_binning(self):
np.histogramdd(self.d, (10000, 10000), ((0, 100), (0, 100)))
def time_small_coverage(self):
np.histogramdd(self.d, (200, 200), ((50, 51), (50, 51)))
def test_empty(self):
a, b = histogramdd([[], []], bins=([0, 1], [0, 1]))
assert_array_max_ulp(a, np.array([[0.]]))
a, b = np.histogramdd([[], [], []], bins=2)
assert_array_max_ulp(a, np.zeros((2, 2, 2)))
def nr_3d(coords,masses,mcut=False,mhigh=1,mlow=1):
if mcut == True:
arr = zip(coords,masses)
arr2 = []
for a,b in zip(mlow,mhigh):
arr3 = [i for i in arr if a < i[1] < b]
arr2.append(arr3)
x = []
y = []
z = []
for i in arr2:
pos,mass = zip(*i)
xx,yy,zz = zip(*pos)
x.append(xx)
y.append(yy)
z.append(zz)
#print np.shape(x),np.shape(y),np.shape(z)
else:
x,y,z = zip(*coords)
pix_num = 50
pix_size = Rng/pix_num
H = []
xe = []
ye = []
ze = []
Rmax = []
for i,j,k in zip(x,y,z):
i = np.asarray(i)
j = np.asarray(j)
k = np.asarray(k)
r2 = np.sqrt(i**2+j**2+k**2)
Rmax.append(max(r2))
counts,[xedges,yedges,zedges] = np.histogramdd((i,j,k),bins=(pix_num,pix_num,pix_num))
H.append(counts)
xe.append(xedges)
ye.append(yedges)
ze.append(zedges)
xe2 = []
for i in xe:
xedges = [j+0.5*pix_size for j in i[:-1]]
xe2.append(xedges)
ye2 = []
for i in ye:
yedges = [j+0.5*pix_size for j in i[:-1]]
ye2.append(yedges)
ze2 = []
for i in ze:
zedges = [j+0.5*pix_size for j in i[:-1]]
ze2.append(zedges)
nr_tot = []
R_tot = []
for i,j,k,l,q in zip(H,xe2,ye2,ze2,Rmax):
im = np.array(i)
xx,yy,zz = np.meshgrid(j,k,l)
r = np.sqrt(xx**2+yy**2+zz**2)
rmax = q
dr = pix_size
R = np.arange(rmax/dr)*dr
R_tot.append(R)
nr = []
for u in range(len(R)):
rmin = u*dr
rmax = rmin + dr
index = (r>=rmin) * (r<=rmax)
nr.append(im[index].mean())
nr_tot.append(nr)
return R_tot,nr_tot
def rad_plot(X_2,
pairwise_distances,
samples,
labels,
file_id,
color_dict,
gridsize=50,
dg_radius=0.2,
axes_radius=0.4,
figsize=8,
log_scale=False,
linkage_method='complete',
plot_type='hexbin'):
n_s = len(np.unique(samples))
clim = np.array([0, .1])
d_max = np.max(X_2, axis=0)
d_min = np.min(X_2, axis=0)
c_center = (d_max + d_min) / 2
c_radius = np.max(
np.sqrt(np.sum(np.power(X_2 - c_center[np.newaxis, :], 2),
axis=1))) * 1.1
c_pos = pol2cart(np.linspace(0, 2 * np.pi, 200),
c_radius) + c_center[np.newaxis, :]
x_edges = np.linspace(d_min[0], d_max[0], gridsize)
y_edges = np.linspace(d_min[1], d_max[1], gridsize)
Y, X = np.meshgrid(x_edges[:-1] + (np.diff(x_edges) / 2),
y_edges[:-1] + (np.diff(y_edges) / 2))
Z = optimal_leaf_ordering(
linkage(pairwise_distances, method=linkage_method), pairwise_distances)
dg_order = leaves_list(Z)
fig = plt.figure(figsize=[figsize, figsize])
axes_pos = pol2cart(np.linspace(0, 2 * np.pi, n_s + 1),
rho=axes_radius) + 0.5
axes_size = axes_radius * np.sin(0.5 * (2 * np.pi / n_s))
ax = [None] * n_s
for i in range(n_s):
ax[i] = fig.add_axes([
axes_pos[i, 0] - axes_size, axes_pos[i, 1] - axes_size,
2 * axes_size, 2 * axes_size
])
ax[i].plot(c_pos[:, 0],
c_pos[:, 1],
'-',
linewidth=5.,
color=color_dict[labels[dg_order[i]]])
smp_d = X_2[file_id == samples[dg_order[i]], :]
if plot_type is 'hexbin':
ax[i].hexbin(smp_d[:, 0], smp_d[:, 1], gridsize=gridsize, mincnt=1)
elif plot_type is '2dhist':
h, _ = np.histogramdd(smp_d, [x_edges, y_edges])
ax[i].pcolormesh(X,
Y,
h / np.sum(h),
shading='gouraud',
vmin=clim[0],
vmax=clim[1],
cmap='GnBu')
else:
ax[i].plot(smp_d, '.', markersize=1, alpha=0.5)
ax[i].set(xticks=[], yticks=[], frame_on=False)
#ax[i].set_title(samples[i])
dg = dendrogram(Z, no_plot=True)
polar_dendrogram(dg, fig, ax_radius=dg_radius, log_scale=log_scale)
def fit(self, theta, x, fill_empty_bins=False):
n_samples = x.shape[0]
self.n_parameters = theta.shape[1]
self.n_observables = x.shape[1]
assert theta.shape[0] == n_samples
logger.debug("Filling histogram with settings:")
logger.debug(" Samples: %s", n_samples)
logger.debug(" Parameters: %s with means %s", self.n_parameters,
np.mean(theta, axis=0))
logger.debug(" Observables: %s with means %s", self.n_observables,
np.mean(x, axis=0))
logger.debug(" No empty bins: %s", fill_empty_bins)
# Find bins
logger.debug("Calculating binning")
self.n_bins = []
self.edges = []
ranges = []
if self.separate_1d_x_histos:
for observable in range(self.n_observables):
histo_n_bins, histo_edges, histo_ranges = self._calculate_binning(
theta, x, [observable])
self.n_bins.append(histo_n_bins)
self.edges.append(histo_edges)
ranges.append(histo_ranges)
else:
histo_n_bins, histo_edges, histo_ranges = self._calculate_binning(
theta, x)
self.n_bins.append(histo_n_bins)
self.edges.append(histo_edges)
ranges.append(histo_ranges)
for h, (histo_n_bins, histo_edges,
histo_ranges) in enumerate(zip(self.n_bins, self.edges,
ranges)):
logger.debug("Histogram %s: bin edges", h + 1)
for i, (axis_bins, axis_edges, axis_range) in enumerate(
zip(histo_n_bins, histo_edges, histo_ranges)):
if i < theta.shape[1]:
logger.debug(" theta %s: %s bins, range %s, edges %s",
i + 1, axis_bins, axis_range, axis_edges)
else:
logger.debug(
" x %s: %s bins, range %s, edges %s",
i + 1 - theta.shape[1],
axis_bins,
axis_range,
axis_edges,
)
# Fill histograms
logger.debug("Filling histograms")
self.histos = []
theta_x = np.hstack([theta, x])
for histo_edges, histo_ranges, histo_n_bins in zip(
self.edges, ranges, self.n_bins):
histo, _ = np.histogramdd(theta_x,
bins=histo_edges,
range=histo_ranges,
normed=False,
weights=None)
# Avoid empty bins
if fill_empty_bins:
histo[histo <= 1.0] = 1.0
# Calculate cell volumes
original_shape = tuple(histo_n_bins)
flat_shape = tuple([-1] + list(histo_n_bins[self.n_parameters:]))
# Fix edges for bvolume calculation (to avoid larger volumes for more training data)
modified_histo_edges = []
for i in range(x.shape[1]):
axis_edges = histo_edges[self.n_parameters + i]
axis_edges[0] = min(np.percentile(x[:, i], 5.0),
axis_edges[1] - 0.01)
axis_edges[-1] = max(np.percentile(x[:, i], 95.0),
axis_edges[-2] + 0.01)
modified_histo_edges.append(axis_edges)
bin_widths = [
axis_edges[1:] - axis_edges[:-1]
for axis_edges in modified_histo_edges
]
volumes = np.ones(flat_shape[1:])
for obs in range(self.n_observables):
# Broadcast bin widths to array with shape like volumes
bin_widths_broadcasted = np.ones(flat_shape[1:])
for indices in np.ndindex(flat_shape[1:]):
bin_widths_broadcasted[indices] = bin_widths[obs][
indices[obs]]
volumes[:] *= bin_widths_broadcasted
# Normalize histograms (for each theta bin)
histo = histo.reshape(flat_shape)
for i in range(histo.shape[0]):
histo[i] /= np.sum(histo[i])
histo[i] /= volumes
histo = histo.reshape(original_shape)
# Avoid NaNs
histo[np.invert(np.isfinite(histo))] = 0.0
self.histos.append(histo)
def carve(xfix=None, zfix=None, use_opencl=True):
global gridinds, inds, grid
gridmin = np.zeros((4, ), 'f')
gridmax = np.zeros((4, ), 'f')
gridmin[:3] = config.bounds[0]
gridmax[:3] = config.bounds[1]
global occ, vac
if xfix is None or zfix is None:
import lattice
xfix, zfix = lattice.meanx, lattice.meanz
if use_opencl:
opencl.compute_gridinds(xfix, zfix, config.LW, config.LH, gridmin,
gridmax)
gridinds = opencl.get_gridinds()
if 0:
inds = gridinds[gridinds[:, 0, 3] != 0, :, :3]
if len(inds) == 0:
return None
bins = [
np.arange(0, gridmax[i] - gridmin[i] + 1) for i in range(3)
]
global occ, vac
occH, _ = np.histogramdd(inds[:, 0, :], bins)
vacH, _ = np.histogramdd(inds[:, 1, :], bins)
vac = vacH > 30
occ = occH > 30
return occ, vac
else:
shape = [gridmax[i] - gridmin[i] for i in range(3)]
occ = np.zeros(shape, 'u1')
vac = np.zeros(shape, 'u1')
speedup_cy.occvac(gridinds, occ, vac, gridmin.astype('i'),
gridmax.astype('i'))
return occ.astype('bool'), vac.astype('bool')
else:
global X, Y, Z, XYZ
X, Y, Z, face = np.rollaxis(opencl.get_modelxyz(), 1)
XYZ = np.array((X, Y, Z)).transpose()
fix = np.array((xfix, 0, zfix))
cxyz = np.frombuffer(np.array(face).data, dtype='i1').reshape(-1,
4)[:, :3]
global cx, cy, cz
cx, cy, cz = np.rollaxis(cxyz, 1)
f1 = cxyz * 0.5
mod = np.array([config.LW, config.LH, config.LW])
gi = np.floor(-gridmin[:3] + (XYZ - fix) / mod + f1)
gi = np.array((gi, gi - cxyz))
gi = np.rollaxis(gi, 1)
def get_gridinds():
(L, T), (R, B) = opencl.rect
length = opencl.length
return (gridinds[:length, :, :].reshape(T - B, R - L, 2, 3),
gridinds[length:, :, :].reshape(T - B, R - L, 2, 3))
gridinds = gi
grid = get_gridinds()
inds = gridinds[np.any(cxyz != 0, 1), :, :]
if len(inds) == 0:
return None
bins = [np.arange(0, gridmax[i] - gridmin[i] + 1) for i in range(3)]
occH, _ = np.histogramdd(inds[:, 0, :], bins)
vacH, _ = np.histogramdd(inds[:, 1, :], bins)
vac = vacH > 30
occ = occH > 30
return occ, vac
def calculate_microstate_volumes(clusters, originalCoordinates, bins, d):
"""
Estimate the clusters volumes using a cubic discretization of volumes
"""
numberOfClusters = clusters.shape[0]
print("Number of clusters", numberOfClusters)
histogram = np.array([])
histograms = []
microstateVolume = np.zeros(numberOfClusters)
# dtrajs = clusteringObject.assign(originalCoordinates)
dtrajs = assignNewTrajectories(originalCoordinates, clusters)
for i in range(numberOfClusters):
allCoords = []
for trajOriginalCoordinates, dtraj in zip(originalCoordinates, dtrajs):
assert dtraj.shape[0] == trajOriginalCoordinates.shape[0]
belongingFrames = np.argwhere(dtraj == i)
trajCoords = trajOriginalCoordinates[belongingFrames, :3]
trajCoords = trajCoords.flatten().tolist()
allCoords.extend(trajCoords)
allCoords = np.reshape(allCoords, (-1, 3))
current_hist, _ = np.histogramdd(allCoords, bins=bins)
histograms.append(current_hist)
if histogram.size == 0:
histogram = np.copy(current_hist)
else:
histogram += current_hist
nRows, nCols, nDepth = histogram.shape
pseudo = False
for i in range(numberOfClusters):
histogramCluster = histograms[i]
if pseudo:
# Add "pseudocounts" to try to fill the holes that lead to volume
# underestimation compared to Matlab script for free energies
histogramTotal = histogram.copy()
for x, y, z in zip(*np.where(histogramCluster)):
upBound = max(x - 1, 0)
lowBound = min(x + 2, nRows)
leftBound = max(0, y - 1)
rightBound = min(y + 2, nCols)
topBound = max(z - 1, 0)
botBound = min(z + 2, nDepth)
signsCluster = np.sign(histogramCluster[upBound:lowBound,
leftBound:rightBound,
topBound:botBound])
# signs = np.sign(histogramTotal[upBound:lowBound, leftBound:rightBound, topBound:botBound])
histogramCluster[
upBound:lowBound, leftBound:rightBound,
topBound:botBound] += (
1 - signsCluster) * d / 8 # + signsCluster*d/2
histogramTotal[upBound:lowBound, leftBound:rightBound,
topBound:botBound] += (1 - signsCluster
) # + signs * d/2
histogramTotal = histogramTotal[histogramCluster > 0]
else:
histogramTotal = histogram[histogramCluster > 0]
histogramCluster = histogramCluster[histogramCluster > 0]
microstateVolume[i] = (histogramCluster / histogramTotal).sum() * d**3
return microstateVolume
def histogram1d(*args,**kwargs):
_range = kwargs.pop('range',None)
if not _range is None:
kwargs['range'] = (_range,) # needs to be a sequence
h,e = numpy.histogramdd(*args,**kwargs)
return h,e[0]
def convert_geometry_hex1d_to_rect2d(geom, signal, key=None, add_rot=0):
"""converts the geometry object of a camera with a hexagonal grid into
a square grid by slanting and stretching the 1D arrays of pixel x
and y positions and signal intensities are converted to 2D
arrays. If the signal array contains a time-dimension it is
conserved.
Parameters
----------
geom : CameraGeometry object
geometry object of hexagonal cameras
signal : ndarray
1D (no timing) or 2D (with timing) array of the pmt signals
key : (default: None)
arbitrary key to store the transformed geometry in a buffer
add_rot : int/float (default: 0)
parameter to apply an additional rotation of `add_rot` times 60°
Returns
-------
new_geom : CameraGeometry object
geometry object of the slanted picture now with a rectangular
grid and a 2D grid for the pixel positions. contains now a 2D
masking array signifying which of the pixels came from the
original geometry and which are simply fillers from the
rectangular grid
rot_img : ndarray 2D (no timing) or 3D (with timing)
the rectangular signal image
"""
if key in rot_buffer:
# if the conversion with this key was done before and stored,
# just read it in
(geom, new_geom, hex_to_rect_map) = rot_buffer[key]
else:
# otherwise, we have to do the conversion first now,
# skew all the coordinates of the original geometry
# extra_rot is the angle to get back to aligned hexagons with flat
# tops. Note that the pixel rotation angle brings the camera so that
# hexagons have a point at the top, so need to go 30deg back to
# make them flat
extra_rot = geom.pix_rotation - 30 * u.deg
# total rotation angle:
rot_angle = (add_rot * 60 * u.deg) - extra_rot
logger.debug("geom={}".format(geom))
logger.debug("rot={}, extra={}".format(rot_angle, extra_rot))
rot_x, rot_y = unskew_hex_pixel_grid(geom.pix_x,
geom.pix_y,
cam_angle=rot_angle)
# with all the coordinate points, we can define the bin edges
# of a 2D histogram
x_edges, y_edges, x_scale = get_orthogonal_grid_edges(rot_x, rot_y)
# this histogram will introduce bins that do not correspond to
# any pixel from the original geometry. so we create a mask to
# remember the true camera pixels by simply throwing all pixel
# positions into numpy.histogramdd: proper pixels contain the
# value 1, false pixels the value 0.
square_mask = np.histogramdd([rot_y, rot_x],
bins=(y_edges, x_edges))[0].astype(bool)
# to be consistent with the pixel intensity, instead of saving
# only the rotated positions of the true pixels (rot_x and
# rot_y), create 2D arrays of all x and y positions (also the
# false ones).
grid_x, grid_y = np.meshgrid((x_edges[:-1] + x_edges[1:]) / 2.,
(y_edges[:-1] + y_edges[1:]) / 2.)
ids = []
# instead of blindly enumerating all pixels, let's instead
# store a list of all valid -- i.e. picked by the mask -- 2D
# indices
for i, row in enumerate(square_mask):
for j, val in enumerate(row):
if val is True:
ids.append((i, j))
# the area of the pixels (note that this is still a deformed
# image)
pix_area = np.ones_like(grid_x) \
* (x_edges[1] - x_edges[0]) * (y_edges[1] - y_edges[0])
# creating a new geometry object with the attributes we just determined
new_geom = CameraGeometry(
cam_id=geom.cam_id + "_rect",
pix_id=ids, # this is a list of all the valid coordinate pairs now
pix_x=grid_x * u.m,
pix_y=grid_y * u.m,
pix_area=pix_area * u.m**2,
neighbors=geom.neighbors,
pix_type='rectangular',
apply_derotation=False)
# storing the pixel mask for later use
new_geom.mask = square_mask
# create a transfer map by enumerating all pixel positions in a 2D histogram
hex_to_rect_map = np.histogramdd([rot_y, rot_x],
bins=(y_edges, x_edges),
weights=np.arange(
len(signal)))[0].astype(int)
# bins that do not correspond to the original image get an entry of `-1`
hex_to_rect_map[~square_mask] = -1
if signal.ndim > 1:
long_map = []
for i in range(signal.shape[-1]):
tmp_map = hex_to_rect_map + i * (np.max(hex_to_rect_map) + 1)
tmp_map[~square_mask] = -1
long_map.append(tmp_map)
hex_to_rect_map = np.array(long_map)
if key is not None:
# if a key is given, store the essential objects in a buffer
rot_buffer[key] = (geom, new_geom, hex_to_rect_map)
# done `if key in rot_buffer`
# create the rotated rectangular image by applying `hex_to_rect_map` to the flat,
# extended input image
# `input_img_ext` is the flattened input image extended by one entry that contains NaN
# since `hex_to_rect_map` contains `-1` for "fake" pixels, it maps this extra NaN
# value at the last array position to any bin that does not correspond to a pixel of
# the original image
input_img_ext = np.full(np.prod(signal.shape) + 1, np.nan)
# the way the map is produced, it has the time dimension as axis=0;
# but `signal` has it as axis=-1, so we need to roll the axes back and forth a bit.
# if there is no time dimension, `signal` is a 1d array and `rollaxis` has no effect.
input_img_ext[:-1] = np.rollaxis(signal, axis=-1, start=0).ravel()
# now apply the transfer map
rot_img = input_img_ext[hex_to_rect_map]
# if there is a time dimension, roll the time axis back to the last position
try:
rot_img = np.rollaxis(rot_img, 0, 3)
except ValueError:
pass
return new_geom, rot_img
def run(
cells_file,
output_filename,
target_size,
raw_image_shape,
raw_image_bin_sizes,
transformation_matrix,
atlas_scale,
smoothing=10,
mask=True,
atlas=None,
cells_only=True,
convert_16bit=True,
):
"""
:param cells_file: Cellfinder output cells file.
:param output_filename: File to save heatmap into
:param target_size: Size of the final heatmap
:param raw_image_shape: Size of the raw data (coordinate space of the
cells)
:param raw_image_bin_sizes: List/tuple of the sizes of the bins in the
raw data space
:param transformation_matrix: Transformation matrix so that the resulting
nifti can be processed using other tools.
:param atlas_scale: Image scaling so that the resulting nifti can be
processed using other tools.
:param smoothing: Smoothing kernel size, in the target image space
:param mask: Whether or not to mask the heatmap based on an atlas file
:param atlas: Atlas file to mask the heatmap
:param cells_only: Only use "cells", not artefacts
:param convert_16bit: Convert final image to 16 bit
"""
# TODO: compare the smoothing effects of gaussian filtering, and upsampling
target_size = convert_shape_dict_to_array_shape(target_size, type="fiji")
raw_image_shape = convert_shape_dict_to_array_shape(raw_image_shape,
type="fiji")
cells_array = get_cell_location_array(cells_file, cells_only=cells_only)
bins = get_bins(raw_image_shape, raw_image_bin_sizes)
logging.debug("Generating heatmap (3D histogram)")
heatmap_array, _ = np.histogramdd(cells_array, bins=bins)
# otherwise resized array is too big to fit into RAM
heatmap_array = heatmap_array.astype(np.uint16)
logging.debug("Resizing heatmap to the size of the target image")
heatmap_array = resize_array(heatmap_array, target_size)
if smoothing is not None:
logging.debug("Applying Gaussian smoothing with a kernel sigma of: "
"{}".format(smoothing))
heatmap_array = gaussian(heatmap_array, sigma=smoothing)
if mask:
logging.debug("Masking image based on registered atlas")
# copy, otherwise it's modified, which affects later figure generation
atlas_for_mask = np.copy(atlas)
heatmap_array = mask_image_threshold(heatmap_array, atlas_for_mask)
if convert_16bit:
logging.debug("Converting to 16 bit")
heatmap_array = scale_and_convert_to_16_bits(heatmap_array)
logging.debug("Ensuring output directory exists")
ensure_directory_exists(Path(output_filename).parent)
logging.debug("Saving heatmap image")
brainio.to_nii(
heatmap_array,
output_filename,
scale=atlas_scale,
affine_transform=transformation_matrix,
)
def on_off_chi_squared_single(samples,
pix,
pix_center,
on_region='disc',
size=np.deg2rad(10),
off_region='allsky',
nside=64,
bins=None,
hist_func=None):
# Construct on region mask
in_on_region = on_region_func(on_region)(pix,
pix_center,
size=size,
nside=nside)
# Construct off region mask
in_off_region = off_region_func(off_region)(pix,
pix_center,
in_on_region,
size,
nside=nside)
# Value distributions for on and off regions
if bins is None:
bins = np.linspace(samples.min(), samples.max(), 20)
if isinstance(bins, np.ndarray) and bins.ndim == 1:
bins = [bins]
if samples.ndim == 1:
samples = samples.reshape(-1, 1)
counts_on, _ = np.histogramdd(samples[in_on_region], bins=bins)
counts_off, _ = np.histogramdd(samples[in_off_region], bins=bins)
if hist_func is not None:
counts_on = hist_func(counts_on)
counts_off = hist_func(counts_off)
if 0 in counts_on:
raise ValueError('Binned distribution for on region centered at pixel '
'{} has zero counts in a bin'.format(pix_center))
if 0 in counts_off:
raise ValueError(
'Binned distribution for off region centered at pixel '
'{} has zero counts in a bin'.format(pix_center))
# Want to make sure off region histogram is scaled to the on region histogram
alpha = np.sum(counts_on) / np.sum(counts_off)
scaled_counts_off = alpha * counts_off
# Calculate chi-squared, p-value, and significance
chi_squared = counts_chi_squared(counts_on, scaled_counts_off)
ndof = counts_on.shape[0]
pval = stats.chi2.sf(chi_squared, ndof)
sig = erfcinv(2 * pval) * np.sqrt(2)
result = {
'pix_center': pix_center,
'alpha': alpha,
'num_on': np.sum(counts_on),
'chi2': chi_squared,
'pval': pval,
'sig': sig,
'ndof': ndof,
}
return result
def _get_obs(self):
"""Get the observations."""
## Birdeye rendering
self.birdeye_render.vehicle_polygons = self.vehicle_polygons
self.birdeye_render.walker_polygons = self.walker_polygons
self.birdeye_render.waypoints = self.waypoints
# birdeye view with roadmap and actors
birdeye_render_types = ['roadmap', 'actors']
if self.display_route:
birdeye_render_types.append('waypoints')
self.birdeye_render.render(self.display, birdeye_render_types)
birdeye = pygame.surfarray.array3d(self.display)
birdeye = birdeye[0:self.display_size, :, :]
birdeye = display_to_rgb(birdeye, self.obs_size)
# Roadmap
if self.pixor:
roadmap_render_types = ['roadmap']
if self.display_route:
roadmap_render_types.append('waypoints')
self.birdeye_render.render(self.display, roadmap_render_types)
roadmap = pygame.surfarray.array3d(self.display)
roadmap = roadmap[0:self.display_size, :, :]
roadmap = display_to_rgb(roadmap, self.obs_size)
# Add ego vehicle
for i in range(self.obs_size):
for j in range(self.obs_size):
if abs(birdeye[i, j, 0] - 255)<20 and abs(birdeye[i, j, 1] - 0)<20 and abs(birdeye[i, j, 0] - 255)<20:
roadmap[i, j, :] = birdeye[i, j, :]
# Display birdeye image
birdeye_surface = rgb_to_display_surface(birdeye, self.display_size)
self.display.blit(birdeye_surface, (0, 0))
## Lidar image generation
point_cloud = []
# Get point cloud data
for location in self.lidar_data:
point_cloud.append([location.x, location.y, -location.z])
point_cloud = np.array(point_cloud)
# Separate the 3D space to bins for point cloud, x and y is set according to self.lidar_bin,
# and z is set to be two bins.
y_bins = np.arange(-(self.obs_range - self.d_behind), self.d_behind+self.lidar_bin, self.lidar_bin)
x_bins = np.arange(-self.obs_range/2, self.obs_range/2+self.lidar_bin, self.lidar_bin)
z_bins = [-self.lidar_height-1, -self.lidar_height+0.25, 1]
# Get lidar image according to the bins
lidar, _ = np.histogramdd(point_cloud, bins=(x_bins, y_bins, z_bins))
lidar[:,:,0] = np.array(lidar[:,:,0]>0, dtype=np.uint8)
lidar[:,:,1] = np.array(lidar[:,:,1]>0, dtype=np.uint8)
# Add the waypoints to lidar image
if self.display_route:
wayptimg = (birdeye[:,:,0] <= 10) * (birdeye[:,:,1] <= 10) * (birdeye[:,:,2] >= 240)
else:
wayptimg = birdeye[:,:,0] < 0 # Equal to a zero matrix
wayptimg = np.expand_dims(wayptimg, axis=2)
wayptimg = np.fliplr(np.rot90(wayptimg, 3))
# Get the final lidar image
lidar = np.concatenate((lidar, wayptimg), axis=2)
lidar = np.flip(lidar, axis=1)
lidar = np.rot90(lidar, 1)
lidar = lidar * 255
# Display lidar image
lidar_surface = rgb_to_display_surface(lidar, self.display_size)
self.display.blit(lidar_surface, (self.display_size, 0))
## Display camera image
camera = resize(self.camera_img, (self.obs_size, self.obs_size)) * 255
camera_surface = rgb_to_display_surface(camera, self.display_size)
self.display.blit(camera_surface, (self.display_size * 2, 0))
# Display on pygame
pygame.display.flip()
# State observation
ego_trans = self.ego.get_transform()
ego_x = ego_trans.location.x
ego_y = ego_trans.location.y
ego_yaw = ego_trans.rotation.yaw/180*np.pi
lateral_dis, w = get_preview_lane_dis(self.waypoints, ego_x, ego_y)
delta_yaw = np.arcsin(np.cross(w,
np.array(np.array([np.cos(ego_yaw), np.sin(ego_yaw)]))))
v = self.ego.get_velocity()
speed = np.sqrt(v.x**2 + v.y**2)
state = np.array([lateral_dis, - delta_yaw, speed, self.vehicle_front])
if self.pixor:
## Vehicle classification and regression maps (requires further normalization)
vh_clas = np.zeros((self.pixor_size, self.pixor_size))
vh_regr = np.zeros((self.pixor_size, self.pixor_size, 6))
# Generate the PIXOR image. Note in CARLA it is using left-hand coordinate
# Get the 6-dim geom parametrization in PIXOR, here we use pixel coordinate
for actor in self.world.get_actors().filter('vehicle.*'):
x, y, yaw, l, w = get_info(actor)
x_local, y_local, yaw_local = get_local_pose((x, y, yaw), (ego_x, ego_y, ego_yaw))
if actor.id != self.ego.id:
if abs(y_local)<self.obs_range/2+1 and x_local<self.obs_range-self.d_behind+1 and x_local>-self.d_behind-1:
x_pixel, y_pixel, yaw_pixel, l_pixel, w_pixel = get_pixel_info(
local_info=(x_local, y_local, yaw_local, l, w),
d_behind=self.d_behind, obs_range=self.obs_range, image_size=self.pixor_size)
cos_t = np.cos(yaw_pixel)
sin_t = np.sin(yaw_pixel)
logw = np.log(w_pixel)
logl = np.log(l_pixel)
pixels = get_pixels_inside_vehicle(
pixel_info=(x_pixel, y_pixel, yaw_pixel, l_pixel, w_pixel),
pixel_grid=self.pixel_grid)
for pixel in pixels:
vh_clas[pixel[0], pixel[1]] = 1
dx = x_pixel - pixel[0]
dy = y_pixel - pixel[1]
vh_regr[pixel[0], pixel[1], :] = np.array(
[cos_t, sin_t, dx, dy, logw, logl])
# Flip the image matrix so that the origin is at the left-bottom
vh_clas = np.flip(vh_clas, axis=0)
vh_regr = np.flip(vh_regr, axis=0)
# Pixor state, [x, y, cos(yaw), sin(yaw), speed]
pixor_state = [ego_x, ego_y, np.cos(ego_yaw), np.sin(ego_yaw), speed]
obs = {
'camera':camera.astype(np.uint8),
'lidar':lidar.astype(np.uint8),
'birdeye':birdeye.astype(np.uint8),
'state': state,
}
if self.pixor:
obs.update({
'roadmap':roadmap.astype(np.uint8),
'vh_clas':np.expand_dims(vh_clas, -1).astype(np.float32),
'vh_regr':vh_regr.astype(np.float32),
'pixor_state': pixor_state,
})
return obs
class icc(Stage):
"""
Data loader stage
Paramaters
----------
params : ParamSet
atm_muon_scale: float
scale factor to be apllied to outputs
icc_bg_file : string
path pointing to the hdf5 file containing the events
use_def1 : bool
whether ICC definition 1 is used
sim_ver: string
indicating the sim version, wither 4digit, 5digit or dima
livetime : time quantity
livetime scale factor
bdt_cut : float
further cut applied to events for the atm. muon rejections BDT
alt_icc_bg_file : string
path pointing to an hdf5 file containing the events for an
kde_hist : bool
fixed_scale_factor : float
scale fixed errors
Notes
-----
The current version of this code is a port from pisa v2 nutau branch.
It clearly needs to be cleaned up properly at some point.
"""
def __init__(self, params, output_binning, disk_cache=None,
memcache_deepcopy=True, error_method=None,
outputs_cache_depth=20, debug_mode=None):
expected_params = (
'atm_muon_scale',
'icc_bg_file',
'use_def1',
'sim_ver',
'livetime',
'bdt_cut',
'alt_icc_bg_file',
'kde_hist',
'fixed_scale_factor'
)
output_names = ('total')
super(self.__class__, self).__init__(
use_transforms=False,
params=params,
expected_params=expected_params,
output_names=output_names,
error_method=error_method,
disk_cache=disk_cache,
memcache_deepcopy=memcache_deepcopy,
outputs_cache_depth=outputs_cache_depth,
output_binning=output_binning,
debug_mode=debug_mode
)
if self.params.kde_hist.value:
from pisa.utils.kde_hist import kde_histogramdd
self.kde_histogramdd = kde_histogramdd
def _compute_nominal_outputs(self):
'''
load events, perform sanity check and put them into histograms,
if alt_bg file is specified, also put these events into separate histograms,
that are normalized to the nominal ones (we are only interested in the shape difference)
'''
# get params
icc_bg_file = self.params.icc_bg_file.value
if 'shape' in self.error_method:
alt_icc_bg_file = self.params.alt_icc_bg_file.value
else:
alt_icc_bg_file = None
sim_ver = self.params.sim_ver.value
use_def1 = self.params.use_def1.value
bdt_cut = self.params.bdt_cut.m_as('dimensionless')
self.bin_names = self.output_binning.names
self.bin_edges = []
for name in self.bin_names:
if 'energy' in name:
bin_edges = self.output_binning[name].bin_edges.to('GeV').magnitude
else:
bin_edges = self.output_binning[name].bin_edges.magnitude
self.bin_edges.append(bin_edges)
# the rest of this function is PISA v2 legacy code...
logging.info('Initializing BackgroundServiceICC...')
logging.info('Opening file: %s'%(icc_bg_file))
try:
bg_file = h5py.File(find_resource(icc_bg_file),'r')
if alt_icc_bg_file is not None:
alt_bg_file = h5py.File(find_resource(alt_icc_bg_file),'r')
except IOError,e:
logging.error("Unable to open icc_bg_file %s"%icc_bg_file)
logging.error(e)
sys.exit(1)
# sanity check
santa_doms = bg_file['IC86_Dunkman_L6_SANTA_DirectDOMs']['value']
l3 = bg_file['IC86_Dunkman_L3']['value']
l4 = bg_file['IC86_Dunkman_L4']['result']
l5 = bg_file['IC86_Dunkman_L5']['bdt_score']
l6 = bg_file['IC86_Dunkman_L6']
if use_def1:
l4_pass = np.all(l4==1)
else:
if sim_ver in ['5digit', 'dima']:
l4_invVICH = bg_file['IC86_Dunkman_L4']['result_invertedVICH']
l4_pass = np.all(np.logical_or(l4==1, l4_invVICH==1))
else:
logging.info(
'For the old simulation, def.2 background not done yet,'
' so still use def1 for it.'
)
l4_pass = np.all(l4==1)
assert (np.all(santa_doms>=3) and np.all(l3 == 1) and l4_pass and
np.all(l5 >= 0.1))
corridor_doms_over_threshold = l6['corridor_doms_over_threshold']
inverted_corridor_cut = corridor_doms_over_threshold > 1
assert (np.all(inverted_corridor_cut) and
np.all(l6['santa_direct_doms'] >= 3) and
np.all(l6['mn_start_contained'] == 1.) and
np.all(l6['mn_stop_contained'] == 1.))
#load events
if sim_ver == '4digit':
variable ='IC86_Dunkman_L6_MultiNest8D_PDG_Neutrino'
elif sim_ver in ['5digit', 'dima']:
variable = 'IC86_Dunkman_L6_PegLeg_MultiNest8D_NumuCC'
else:
raise ValueError('Only allow sim_ver 4digit, 5 digit or dima!')
reco_energy_all = np.array(bg_file[variable]['energy'])
reco_coszen_all = np.array(np.cos(bg_file[variable]['zenith']))
pid_all = np.array(bg_file['IC86_Dunkman_L6']['delta_LLH'])
if alt_icc_bg_file is not None:
alt_reco_energy_all = np.array(alt_bg_file[variable]['energy'])
alt_reco_coszen_all = np.array(np.cos(alt_bg_file[variable]['zenith']))
alt_pid_all = np.array(alt_bg_file['IC86_Dunkman_L6']['delta_LLH'])
alt_l5 = alt_bg_file['IC86_Dunkman_L5']['bdt_score']
# Cut: Only keep bdt score >= 0.2 (from MSU latest result, make data/MC
# agree much better)
cut_events = {}
cut = l5>=bdt_cut
cut_events['reco_energy'] = reco_energy_all[cut]
cut_events['reco_coszen'] = reco_coszen_all[cut]
cut_events['pid'] = pid_all[cut]
if alt_icc_bg_file is not None:
# Cut: Only keep bdt score >= 0.2 (from MSU latest result, make
# data/MC agree much better)
alt_cut_events = {}
alt_cut = alt_l5>=bdt_cut
alt_cut_events['reco_energy'] = alt_reco_energy_all[alt_cut]
alt_cut_events['reco_coszen'] = alt_reco_coszen_all[alt_cut]
alt_cut_events['pid'] = alt_pid_all[alt_cut]
logging.info("Creating a ICC background hists...")
# make histo
if self.params.kde_hist.value:
self.icc_bg_hist = self.kde_histogramdd(
np.array([cut_events[bin_name] for bin_name in self.bin_names]).T,
binning=self.output_binning,
coszen_name='reco_coszen',
use_cuda=True,
bw_method='silverman',
alpha=0.3,
oversample=10,
coszen_reflection=0.5,
adaptive=True
)
else:
self.icc_bg_hist,_ = np.histogramdd(sample = np.array([cut_events[bin_name] for bin_name in self.bin_names]).T, bins=self.bin_edges)
conversion = self.params.atm_muon_scale.value.m_as('dimensionless') / ureg('common_year').to('seconds').m
logging.info('nominal ICC rate at %.6E Hz'%(self.icc_bg_hist.sum()*conversion))
if alt_icc_bg_file is not None:
if self.params.kde_hist.value:
self.alt_icc_bg_hist = self.kde_histogramdd(
np.array([alt_cut_events[bin_name] for bin_name in self.bin_names]).T,
binning=self.output_binning,
coszen_name='reco_coszen',
use_cuda=True,
bw_method='silverman',
alpha=0.3,
oversample=10,
coszen_reflection=0.5,
adaptive=True
)
else:
self.alt_icc_bg_hist,_ = np.histogramdd(sample = np.array([alt_cut_events[bin_name] for bin_name in self.bin_names]).T, bins=self.bin_edges)
# only interested in shape difference, not rate
scale = self.icc_bg_hist.sum()/self.alt_icc_bg_hist.sum()
self.alt_icc_bg_hist *= scale
#!/usr/bin/env python
# coding=utf-8
import numpy as np
from scipy.interpolate import griddata
L = 2.
N = 2.
H = L / N
grid = np.zeros((N, N, N))
sample = np.array([[0.5, 0.5, 0.5], [0.5, 1.5, 1.5], [0.5, 0.5, 1.5],
[0.5, 0.5, 0.6]])
edge = np.array([0, 1, 2])
x = sample[:, 0]
y = sample[:, 1]
z = sample[:, 2]
zz = np.array([1, 2, 3, 4])
#for i,j,k in sample/H :
# print i,j,k
# nx=int(i)
# ny=int(j)
# nz=int(k)
# grid[nx,ny,nz]+=1
#print grid
grid_r = np.arange(N)
################################################################################
#zi=griddata(sample,zz,(grid_r[:,None,None],grid_r[None,:,None],grid_r[None,None,:]),method='nearest')
s, edges = np.histogramdd(sample=sample, bins=(edge, edge, edge), weights=zz)
print s
def histogram2d(x, y, bins=10, range=None, normed=None, weights=None,
density=None):
"""
Compute the bi-dimensional histogram of two data samples.
Parameters
----------
x : array_like, shape (N,)
An array containing the x coordinates of the points to be
histogrammed.
y : array_like, shape (N,)
An array containing the y coordinates of the points to be
histogrammed.
bins : int or array_like or [int, int] or [array, array], optional
The bin specification:
* If int, the number of bins for the two dimensions (nx=ny=bins).
* If array_like, the bin edges for the two dimensions
(x_edges=y_edges=bins).
* If [int, int], the number of bins in each dimension
(nx, ny = bins).
* If [array, array], the bin edges in each dimension
(x_edges, y_edges = bins).
* A combination [int, array] or [array, int], where int
is the number of bins and array is the bin edges.
range : array_like, shape(2,2), optional
The leftmost and rightmost edges of the bins along each dimension
(if not specified explicitly in the `bins` parameters):
``[[xmin, xmax], [ymin, ymax]]``. All values outside of this range
will be considered outliers and not tallied in the histogram.
density : bool, optional
If False, the default, returns the number of samples in each bin.
If True, returns the probability *density* function at the bin,
``bin_count / sample_count / bin_area``.
normed : bool, optional
An alias for the density argument that behaves identically. To avoid
confusion with the broken normed argument to `histogram`, `density`
should be preferred.
weights : array_like, shape(N,), optional
An array of values ``w_i`` weighing each sample ``(x_i, y_i)``.
Weights are normalized to 1 if `normed` is True. If `normed` is
False, the values of the returned histogram are equal to the sum of
the weights belonging to the samples falling into each bin.
Returns
-------
H : ndarray, shape(nx, ny)
The bi-dimensional histogram of samples `x` and `y`. Values in `x`
are histogrammed along the first dimension and values in `y` are
histogrammed along the second dimension.
xedges : ndarray, shape(nx+1,)
The bin edges along the first dimension.
yedges : ndarray, shape(ny+1,)
The bin edges along the second dimension.
See Also
--------
histogram : 1D histogram
histogramdd : Multidimensional histogram
Notes
-----
When `normed` is True, then the returned histogram is the sample
density, defined such that the sum over bins of the product
``bin_value * bin_area`` is 1.
Please note that the histogram does not follow the Cartesian convention
where `x` values are on the abscissa and `y` values on the ordinate
axis. Rather, `x` is histogrammed along the first dimension of the
array (vertical), and `y` along the second dimension of the array
(horizontal). This ensures compatibility with `histogramdd`.
Examples
--------
>>> from matplotlib.image import NonUniformImage
>>> import matplotlib.pyplot as plt
Construct a 2-D histogram with variable bin width. First define the bin
edges:
>>> xedges = [0, 1, 3, 5]
>>> yedges = [0, 2, 3, 4, 6]
Next we create a histogram H with random bin content:
>>> x = np.random.normal(2, 1, 100)
>>> y = np.random.normal(1, 1, 100)
>>> H, xedges, yedges = np.histogram2d(x, y, bins=(xedges, yedges))
>>> H = H.T # Let each row list bins with common y range.
:func:`imshow <matplotlib.pyplot.imshow>` can only display square bins:
>>> fig = plt.figure(figsize=(7, 3))
>>> ax = fig.add_subplot(131, title='imshow: square bins')
>>> plt.imshow(H, interpolation='nearest', origin='lower',
... extent=[xedges[0], xedges[-1], yedges[0], yedges[-1]])
<matplotlib.image.AxesImage object at 0x...>
:func:`pcolormesh <matplotlib.pyplot.pcolormesh>` can display actual edges:
>>> ax = fig.add_subplot(132, title='pcolormesh: actual edges',
... aspect='equal')
>>> X, Y = np.meshgrid(xedges, yedges)
>>> ax.pcolormesh(X, Y, H)
<matplotlib.collections.QuadMesh object at 0x...>
:class:`NonUniformImage <matplotlib.image.NonUniformImage>` can be used to
display actual bin edges with interpolation:
>>> ax = fig.add_subplot(133, title='NonUniformImage: interpolated',
... aspect='equal', xlim=xedges[[0, -1]], ylim=yedges[[0, -1]])
>>> im = NonUniformImage(ax, interpolation='bilinear')
>>> xcenters = (xedges[:-1] + xedges[1:]) / 2
>>> ycenters = (yedges[:-1] + yedges[1:]) / 2
>>> im.set_data(xcenters, ycenters, H)
>>> ax.images.append(im)
>>> plt.show()
"""
from numpy import histogramdd
try:
N = len(bins)
except TypeError:
N = 1
if N != 1 and N != 2:
xedges = yedges = asarray(bins)
bins = [xedges, yedges]
hist, edges = histogramdd([x, y], bins, range, normed, weights, density)
return hist, edges[0], edges[1]
def rgb_img_to_3d_histo(img, bin_count): img = img.reshape((img.shape[0]*img.shape[1],3)) return np.histogramdd(img, bins = [bin_count,bin_count,bin_count], range=((0,256),(0,256),(0,256)))
def plot_magdist(self,type='sf',cut=True,splitcolors=False,overplot=False):
"""
NAME:
plot_magdist
PURPOSE:
Plot the distribution of counts in magnitude
INPUT:
type= ('sf') Plot 'sf': selection function
'tgas': TGAS counts
'2mass': 2MASS counts
cut= (True) cut to the 'good' part of the sky
splitcolors= (False) if True, plot the 3 color bins separately
OUTPUT:
Plot to output device
HISTORY:
2017-01-17 - Written - Bovy (UofT/CCA)
"""
jtbins= (numpy.amax(_2mc[0])-numpy.amin(_2mc[0]))/0.1+1
nstar2mass, edges= numpy.histogramdd(\
_2mc[:3].T,bins=[jtbins,3,_BASE_NPIX],
range=[[numpy.amin(_2mc[0])-0.05,numpy.amax(_2mc[0])+0.05],
[-0.05,1.0],[-0.5,_BASE_NPIX-0.5]],weights=_2mc[3])
findx= (self._full_jk > -0.05)*(self._full_jk < 1.0)\
*(self._full_twomass['j_mag'] < 13.5)
nstartgas, edges= numpy.histogramdd(\
numpy.array([self._full_jt[findx],self._full_jk[findx],\
(self._full_tgas['source_id'][findx]\
/2**(35.+2*(12.-numpy.log2(_BASE_NSIDE))))\
.astype('int')]).T,
bins=[jtbins,3,_BASE_NPIX],
range=[[numpy.amin(_2mc[0])-0.05,numpy.amax(_2mc[0])+0.05],
[-0.05,1.0],[-0.5,_BASE_NPIX-0.5]])
if cut:
nstar2mass[:,:,self._exclude_mask_skyonly]= numpy.nan
nstartgas[:,:,self._exclude_mask_skyonly]= numpy.nan
nstar2mass= numpy.nansum(nstar2mass,axis=-1)
nstartgas= numpy.nansum(nstartgas,axis=-1)
exs= 0.5*(numpy.roll(edges[0],1)+edges[0])[1:]
for ii in range(3):
if type == 'sf':
if splitcolors:
pt= nstartgas[:,ii]/nstar2mass[:,ii]
else:
pt= numpy.nansum(nstartgas,axis=-1)\
/numpy.nansum(nstar2mass,axis=-1)
vmin= 0.
vmax= 1.
ylabel=r'$\mathrm{completeness}$'
semilogy= False
elif type == 'tgas' or type == '2mass':
vmin= 1.
vmax= 10**6.
ylabel= r'$\log_{10}\mathrm{number\ counts}$'
semilogy= True
if type == 'tgas':
if splitcolors:
pt= nstartgas[:,ii]
else:
pt= numpy.nansum(nstartgas,-1)
elif type == '2mass':
if splitcolors:
pt= nstar2mass[:,ii]
else:
pt= numpy.nansum(nstar2mass,-1)
bovy_plot.bovy_plot(exs,pt,ls='steps-mid',
xrange=[2.,14.],
yrange=[vmin,vmax],
semilogy=semilogy,
xlabel=r'$J+\Delta J$',
ylabel=ylabel,
overplot=(ii>0)+overplot)
if not splitcolors: break
return None
def time_full_coverage(self):
np.histogramdd(self.d, (200, 200), ((0, 100), (0, 100)))
