def testDistanceBetweenCurves(self):
l1 = {'save_xd': array([[0.5,1,0], [1.5,1,0]]), 'lamb':array([0.0, 1.0])}
l2 = {'save_xd': array([[0,0,0], [1,0,0], [2,0,0]])}
x = arange(-1,1,0.005)
line = array(zip(x,x,x)) #not actually needed for calcualtion, but dummy argument to residuals_cal for now
residuals_calc = LPCResiduals(line, tube_radius = 0.2)
dist = residuals_calc._distanceBetweenCurves(l1,l2)
def column_stack(tup):
""" Stack 1D arrays as columns into a 2D array
Description:
Take a sequence of 1D arrays and stack them as columns
to make a single 2D array. All arrays in the sequence
must have the same first dimension. 2D arrays are
stacked as-is, just like with hstack. 1D arrays are turned
into 2D columns first.
Arguments:
tup -- sequence of 1D or 2D arrays. All arrays must have the same
first dimension.
Examples:
>>> import numpy
>>> a = array((1,2,3))
>>> b = array((2,3,4))
>>> numpy.column_stack((a,b))
array([[1, 2],
[2, 3],
[3, 4]])
"""
arrays = []
for v in tup:
arr = array(v,copy=False,subok=True)
if arr.ndim < 2:
arr = array(arr,copy=False,subok=True,ndmin=2).T
arrays.append(arr)
return _nx.concatenate(arrays,1)
def _replace_zero_by_x_arrays(sub_arys):
for i in range(len(sub_arys)):
if len(_nx.shape(sub_arys[i])) == 0:
sub_arys[i] = _nx.array([])
elif _nx.sometrue(_nx.equal(_nx.shape(sub_arys[i]),0)):
sub_arys[i] = _nx.array([])
return sub_arys
def test_matrix_product(self):
A = array( [[1,1],
[0,1]] )
B = array( [[2,0],
[3,4]] )
C = dot(A,B)
numpy.testing.assert_array_equal(C,array([[5, 4],
[3, 4]]))
def test_elementwise_product(self):
A = array( [[1,1],
[0,1]] )
B = array( [[2,0],
[3,4]] )
C = A*B # elementwise product
numpy.testing.assert_array_equal(C, array([[2, 0],
[0, 4]]))
def loadClassifierNormNum():
# 加载数据
datingDataMat, datingLabels = kNN.filedata2matrix('datingTestSet2.txt')
normMat, ranges, minVals = kNN.autoNorm(datingDataMat)
# 打印数据
print normMat
print ranges
print minVals
# 图形化显示数据
fig = plt.figure()
ax = fig.add_subplot(111)
ax.scatter(normMat[:, 0], normMat[:, 1], 15.0 * array(datingLabels), 15.0 * array(datingLabels))
plt.show()
def _removeNonTracklikeClusterCenters(self):
'''NOTE : Much of this code is copied from LPCMImpl.followXSingleDirection (factor out?)
'''
labels = self._meanShift.labels_
labels_unique = unique(labels)
cluster_centers = self._meanShift.cluster_centers_
rsp = lpcRandomStartPoints()
cluster_representatives = []
for k in range(len(labels_unique)):
cluster_members = labels == k
cluster_center = cluster_centers[k]
cluster = self._Xi[cluster_members,:]
mean_sub = cluster - cluster_center
cov_x = dot(transpose(mean_sub), mean_sub)
eigen_cov = eigh(cov_x)
sorted_eigen_cov = zip(eigen_cov[0],map(ravel,vsplit(eigen_cov[1].transpose(),len(eigen_cov[1]))))
sorted_eigen_cov.sort(key = lambda elt: elt[0], reverse = True)
rho = sorted_eigen_cov[1][0] / sorted_eigen_cov[0][0] #Ratio of two largest eigenvalues
if rho < self._lpcParameters['rho_threshold']:
cluster_representatives.append(cluster_center)
else: #append a random element of the cluster
random_cluster_element = rsp(cluster, 1)[0]
cluster_representatives.append(random_cluster_element)
return array(cluster_representatives)
def calculatePurity(self, curves, data_range, voxel_to_pdg_dictionary):
'''NB - self._residuals_runner should have had calculateResiduals method called with calc_residuals = True beofre calling this method
'''
hit_tuples = voxel_to_pdg_dictionary.keys()
if data_range is None:
data_range = 1.0
#rescales the truth data if necessary
hits = array([[h[0], h[1], h[2]] for h in hit_tuples]) / data_range
self._residuals_runner.setDataPoints(hits)
self._residuals_runner.setLpcCurves(curves)
self._residuals_runner.calculateResiduals(True, False, False)
residuals = self._residuals_runner.getResiduals()
tau_range = self._residuals_runner.getTauRange()
purity = {}
for tau in tau_range:
pdg_code_frequencies = []
for i in range(len(curves)):
d = defaultdict(int)
hit_labels = [voxel_to_pdg_dictionary[hit_tuples[i]] for i in residuals['curve_residuals'][i]['coverage_indices'][tau]]
flattened_hit_labels = [pdg_code for pdg_code_list in hit_labels for pdg_code in pdg_code_list]
for pdg_code in flattened_hit_labels:
d[pdg_code] += 1
pdg_code_frequencies.append(d)
purity[tau] = pdg_code_frequencies
return purity
def kron(a,b):
"""kronecker product of a and b
Kronecker product of two arrays is block array
[[ a[ 0 ,0]*b, a[ 0 ,1]*b, ... , a[ 0 ,n-1]*b ],
[ ... ... ],
[ a[m-1,0]*b, a[m-1,1]*b, ... , a[m-1,n-1]*b ]]
"""
wrapper = get_array_wrap(a, b)
b = asanyarray(b)
a = array(a,copy=False,subok=True,ndmin=b.ndim)
ndb, nda = b.ndim, a.ndim
if (nda == 0 or ndb == 0):
return _nx.multiply(a,b)
as_ = a.shape
bs = b.shape
if not a.flags.contiguous:
a = reshape(a, as_)
if not b.flags.contiguous:
b = reshape(b, bs)
nd = ndb
if (ndb != nda):
if (ndb > nda):
as_ = (1,)*(ndb-nda) + as_
else:
bs = (1,)*(nda-ndb) + bs
nd = nda
result = outer(a,b).reshape(as_+bs)
axis = nd-1
for _ in xrange(nd):
result = concatenate(result, axis=axis)
if wrapper is not None:
result = wrapper(result)
return result
def apply_over_axes(func, a, axes):
"""Apply a function repeatedly over multiple axes, keeping the same shape
for the resulting array.
func is called as res = func(a, axis). The result is assumed
to be either the same shape as a or have one less dimension.
This call is repeated for each axis in the axes sequence.
"""
val = asarray(a)
N = a.ndim
if array(axes).ndim == 0:
axes = (axes,)
for axis in axes:
if axis < 0: axis = N + axis
args = (val, axis)
res = func(*args)
if res.ndim == val.ndim:
val = res
else:
res = expand_dims(res,axis)
if res.ndim == val.ndim:
val = res
else:
raise ValueError, "function is not returning"\
" an array of correct shape"
return val
def getCoverageGraph(self, curves, tau_range): '''Return a 2*len(tau_range) array of the proportion of self._X points within tau (in tau_range, an array) of the curve segments, where 'curves' is a either a list of curve dictionaries as returned by LPCImpl.lpc, or an element thereof This should give graphs similar to the output of BOakley ''' coverage = [1.0*len(self.calculateCoverageIndices(curves,tau))/len(self._X) for tau in tau_range] return array([tau_range,coverage])
def nanargmax(a, axis=None):
"""Find the maximum over the given axis ignoring NaNs.
"""
y = array(a,subok=True)
if not issubclass(y.dtype.type, _nx.integer):
y[isnan(a)] = -_nx.inf
return y.argmax(axis)
def nanmin(a, axis=None):
"""Find the minimium over the given axis, ignoring NaNs.
"""
y = array(a,subok=True)
if not issubclass(y.dtype.type, _nx.integer):
y[isnan(a)] = _nx.inf
return y.min(axis)
def twoDisjointLinesWithMSClustering():
t = arange(-1,1,0.002)
x = map(lambda x: x + gauss(0,0.02)*(1-x*x), t)
y = map(lambda x: x + gauss(0,0.02)*(1-x*x), t)
z = map(lambda x: x + gauss(0,0.02)*(1-x*x), t)
line1 = array(zip(x,y,z))
line = vstack((line1, line1 + 3))
lpc = LPCImpl(start_points_generator = lpcMeanShift(ms_h = 1), h = 0.05, mult = None, it = 200, cross = False, scaled = False, convergence_at = 0.001)
lpc_curve = lpc.lpc(X=line)
#Plot results
fig = plt.figure()
ax = Axes3D(fig)
labels = lpc._startPointsGenerator._meanShift.labels_
labels_unique = unique(labels)
cluster_centers = lpc._startPointsGenerator._meanShift.cluster_centers_
n_clusters = len(labels_unique)
colors = cycle('bgrcmyk')
for k, col in zip(range(n_clusters), colors):
cluster_members = labels == k
cluster_center = cluster_centers[k]
ax.scatter(line[cluster_members, 0], line[cluster_members, 1], line[cluster_members, 2], c = col, alpha = 0.1)
ax.scatter([cluster_center[0]], [cluster_center[1]], [cluster_center[2]], c = 'b', marker= '^')
curve = lpc_curve[k]['save_xd']
ax.plot(curve[:,0],curve[:,1],curve[:,2], c = col, linewidth = 3)
plt.show()
def lpc(self, x0 = None, X=None, weights = None):
''' Will return the scaled curve if self._lpcParameters['scaled'] = True, to return the curve on the same scale as the originally input data, call getCurve with unscale = True
Arguments
---------
x0 : 2-dim numpy.array containing #rows equal to number of explicitly defined start points
and #columns equal to dimension of the feature space points; seeds for the start points algorithm
X : 2-dim numpy.array containing #rows equal to number of data points and #columns equal to dimension
of the feature space points
weights : see self._followxSingleDirection docs
'''
if X is None:
if self.Xi is None:
raise ValueError, 'Data points have not yet been set in this LPCImpl instance. Either supply as X parameter to this function or call setDataPoints'
else:
self.setDataPoints(X)
N = self.Xi.shape[0]
if self._lpcParameters['binary'] or weights is None:
self._weights = ones(N, dtype = float)
else:
self._weights = array(weights, dtype = float)
if self._weights.shape != (N):
raise ValueError, 'Weights must be one dimensional of vector of weights with size equal to the sample size'
self._selectStartPoints(x0)
#TODO add initialization relevant for other branches
m = self.x0.shape[0] #how many starting points were actually generated
way = self._lpcParameters['way']
self._curve = [self._followx(self.x0[j], way = way, weights = self._weights) for j in range(m)]
return self._curve
def __init__(self,filepath):
p = Path(filepath).resolve()
self._filepath = p
self._imgPtr = Image.open(str(p))
self._imgArr = [array(self._imgPtr)] # cached copies of image levels
self._levels = [LevelInfo(self._imgArr[0].shape[0],self._imgArr[0].shape[1],1,1,1)]
def __init__(self, c_or_r, r=False, variable=None):
if isinstance(c_or_r, poly1d):
self._variable = c_or_r._variable
self._coeffs = c_or_r._coeffs
if set(c_or_r.__dict__) - set(self.__dict__):
msg = ("In the future extra properties will not be copied "
"across when constructing one poly1d from another")
warnings.warn(msg, FutureWarning, stacklevel=2)
self.__dict__.update(c_or_r.__dict__)
if variable is not None:
self._variable = variable
return
if r:
c_or_r = poly(c_or_r)
c_or_r = atleast_1d(c_or_r)
if c_or_r.ndim > 1:
raise ValueError("Polynomial must be 1d only.")
c_or_r = trim_zeros(c_or_r, trim='f')
if len(c_or_r) == 0:
c_or_r = NX.array([0.])
self._coeffs = c_or_r
if variable is None:
variable = 'x'
self._variable = variable
def testNoisyLine2(self): x = map(lambda x: x + gauss(0,0.005), arange(-1,1,0.005)) y = map(lambda x: x + gauss(0,0.005), arange(-1,1,0.005)) z = map(lambda x: x + gauss(0,0.005), arange(-1,1,0.005)) line = array(zip(x,y,z)) lpc = LPCImpl(h = 0.2, convergence_at = 0.001, mult = 2) lpc_curve = lpc.lpc(X = line)
def testNoisyLine1(self): x = map(lambda x: x + gauss(0,0.002), arange(-1,1,0.001)) y = map(lambda x: x + gauss(0,0.002), arange(-1,1,0.001)) z = map(lambda x: x + gauss(0,0.02), arange(-1,1,0.001)) line = array(zip(x,y,z)) lpc = LPCImpl(h = 0.2, mult = 2) lpc_curve = lpc.lpc(X = line)
def findCoords(gs, candidates=None):
if candidates == None:
candidates=[]
# List all the possible z-level (heights)
zRange = list(takewhile(lambda x : x < gs.boardSize[2], \
sort(unique(flatten(gs.heightMap())))))
if zRange==[]:
print "Board is full, cannot find legal coordinates !"
return None
else:
zRange = sort(unique(map(third,candidates)))
# Do we have a choice on the z-level ?
if len(zRange)==1:
z = zRange[0]
else:
print "\n",gs.boardToASCII(markedCubes=candidates)
# Discard the z height max
if zRange[-1]==gs.boardSize[2]:
zRange = zRange[:-1]
z = -1+input("Which z-level ? (%d-%d)\n> " \
% (zRange[0]+1,zRange[-1]+1))
candidates = filter(lambda c: c[2]==z, candidates)
if len(candidates)>1:
# Display the z-level with xy coordinates as letter-number
print ' '+''.join(chr(97+x) for x in xrange(gs.boardSize[0]))
print ' +'+'-'*gs.boardSize[0]
lines = gs.boardToASCII(zRange=[z],markedCubes=candidates)\
.split('\n')
for y in xrange(gs.boardSize[1]):
print '%s |%s' % (str(y+1).zfill(2),lines[y])
print "\n"
xy = raw_input("Which xy coordinates ?\n> ")
return array([ord(xy[0])-97,int(xy[1:])-1,z])
else:
return candidates[0]
def test_multi_itr_array(self):
c = array( [ [[ 0, 1, 2],
[ 10, 12, 13]],
[[100,101,102],
[110,112,113]] ] )
def nansum(a, axis=None):
"""Sum the array over the given axis, treating NaNs as 0.
"""
y = array(a,subok=True)
if not issubclass(y.dtype.type, _nx.integer):
y[isnan(a)] = 0
return y.sum(axis)
def helixHeteroscedasticCrossingDemo():
#Parameterise a helix (no noise)
fig5 = plt.figure()
t = arange(-1,1,0.001)
x = map(lambda x: x + gauss(0,0.01 + 0.05*sin(8*pi*x)), (1 - t*t)*sin(4*pi*t))
y = map(lambda x: x + gauss(0,0.01 + 0.05*sin(8*pi*x)), (1 - t*t)*cos(4*pi*t))
z = map(lambda x: x + gauss(0,0.01 + 0.05*sin(8*pi*x)), t)
line = array(zip(x,y,z))
lpc = LPCImpl(h = 0.15, t0 = 0.1, mult = 2, it = 500, scaled = False)
lpc_curve = lpc.lpc(line)
ax = Axes3D(fig5)
ax.set_title('helixHeteroscedasticWithCrossing')
curve = lpc_curve[0]['save_xd']
ax.scatter(x,y,z, c = 'red')
ax.plot(curve[:,0],curve[:,1],curve[:,2])
saveToPdf(fig5, '/tmp/helixHeteroscedasticWithCrossing.pdf')
lpc.set_in_dict('cross', False, '_lpcParameters')
fig6 = plt.figure()
lpc_curve = lpc.lpc(X=line)
ax = Axes3D(fig6)
ax.set_title('helixHeteroscedasticWithoutCrossing')
curve = lpc_curve[0]['save_xd']
ax.scatter(x,y,z, c = 'red')
ax.plot(curve[:,0],curve[:,1],curve[:,2])
saveToPdf(fig6, '/tmp/helixHeteroscedasticWithoutCrossing.pdf')
def helixHeteroscedasticDiags():
#Parameterise a helix (no noise)
fig5 = plt.figure()
t = arange(-1,1,0.0005)
x = map(lambda x: x + gauss(0,0.001 + 0.001*sin(2*pi*x)**2), (1 - t*t)*sin(4*pi*t))
y = map(lambda x: x + gauss(0,0.001 + 0.001*sin(2*pi*x)**2), (1 - t*t)*cos(4*pi*t))
z = map(lambda x: x + gauss(0,0.001 + 0.001*sin(2*pi*x)**2), t)
line = array(zip(x,y,z))
lpc = LPCImpl(h = 0.1, t0 = 0.1, mult = 1, it = 500, scaled = False, cross = False)
lpc_curve = lpc.lpc(X=line)
ax = Axes3D(fig5)
ax.set_title('helixHeteroscedastic')
curve = lpc_curve[0]['save_xd']
ax.scatter(x,y,z, c = 'red')
ax.plot(curve[:,0],curve[:,1],curve[:,2])
saveToPdf(fig5, '/tmp/helixHeteroscedastic.pdf')
residuals_calc = LPCResiduals(line, tube_radius = 0.2, k = 20)
residual_diags = residuals_calc.getPathResidualDiags(lpc_curve[0])
fig6 = plt.figure()
#plt.plot(lpc_curve[0]['lamb'][1:], residual_diags['line_seg_num_NN'], drawstyle = 'step', linestyle = '--')
plt.plot(lpc_curve[0]['lamb'][1:], residual_diags['line_seg_mean_NN'])
plt.plot(lpc_curve[0]['lamb'][1:], residual_diags['line_seg_std_NN'])
saveToPdf(fig6, '/tmp/helixHeteroscedasticPathResiduals.pdf')
coverage_graph = residuals_calc.getCoverageGraph(lpc_curve[0], arange(0.01, .052, 0.01))
fig7 = plt.figure()
plt.plot(coverage_graph[0],coverage_graph[1])
saveToPdf(fig7, '/tmp/helixHeteroscedasticCoverage.pdf')
residual_graph = residuals_calc.getGlobalResiduals(lpc_curve[0])
fig8 = plt.figure()
plt.plot(residual_graph[0], residual_graph[1])
saveToPdf(fig8, '/tmp/helixHeteroscedasticResiduals.pdf')
fig9 = plt.figure()
plt.plot(range(len(lpc_curve[0]['lamb'])), lpc_curve[0]['lamb'])
saveToPdf(fig9, '/tmp/helixHeteroscedasticPathLength.pdf')
def plot2():
fig5 = plt.figure()
x = map(lambda x: x + gauss(0,0.02)*(1-x*x), arange(-1,1,0.001))
y = map(lambda x: x + gauss(0,0.02)*(1-x*x), arange(-1,1,0.001))
z = map(lambda x: x + gauss(0,0.02)*(1-x*x), arange(-1,1,0.001))
line = array(zip(x,y,z))
lpc = LPCImpl(h = 0.05, mult = 2, it = 200, cross = False, scaled = False, convergence_at = 0.001)
lpc_curve = lpc.lpc(X=line)
ax = Axes3D(fig5)
ax.set_title('testNoisyLine2')
curve = lpc_curve[0]['save_xd']
ax.scatter(x,y,z, c = 'red')
ax.plot(curve[:,0],curve[:,1],curve[:,2])
saveToPdf(fig5, '/tmp/testNoisyLine2.pdf')
residuals_calc = LPCResiduals(line, tube_radius = 0.05, k = 10)
residual_diags = residuals_calc.getPathResidualDiags(lpc_curve[0])
fig6 = plt.figure()
#plt.plot(lpc_curve[0]['lamb'][1:], residual_diags['line_seg_num_NN'], drawstyle = 'step', linestyle = '--')
plt.plot(lpc_curve[0]['lamb'][1:], residual_diags['line_seg_mean_NN'])
plt.plot(lpc_curve[0]['lamb'][1:], residual_diags['line_seg_std_NN'])
saveToPdf(fig6, '/tmp/testNoisyLine2PathResiduals.pdf')
coverage_graph = residuals_calc.getCoverageGraph(lpc_curve[0], arange(0.001, .102, 0.005))
fig7 = plt.figure()
plt.plot(coverage_graph[0],coverage_graph[1])
saveToPdf(fig7, '/tmp/testNoisyLine2Coverage.pdf')
residual_graph = residuals_calc.getGlobalResiduals(lpc_curve[0])
fig8 = plt.figure()
plt.plot(residual_graph[0], residual_graph[1])
saveToPdf(fig8, '/tmp/testNoisyLine2Residuals.pdf')
fig9 = plt.figure()
plt.plot(range(len(lpc_curve[0]['lamb'])), lpc_curve[0]['lamb'])
saveToPdf(fig9, '/tmp/testNoisyLine2PathLength.pdf')
def test_linkage_to_d3_4_observations(self):
Z = array([[ 1. , 3. , 0.45015331, 2. ], # arr[0], cluster4
[ 0. , 2. , 1.29504919, 2. ], # arr[1], cluster5
[ 4. , 5. , 1.55180264, 4. ]]) # arr[2], cluster6
expected = {
"name": "cluster6",
"children": [
{
"name": "cluster4",
"children": [
{"name": "cluster1", "size": 10},
{"name": "cluster3", "size": 10},
]
},
{
"name": "cluster5",
"children": [
{"name": "cluster0", "size": 10},
{"name": "cluster2", "size": 10},
],
},
]
}
# n = len(Z)+1
# d3_dict = _do_linkage_to_d3(n, len(Z)+n-1, Z)
d3_dict = linkage_to_d3(Z)
self.assertDictEqual(expected, d3_dict)
def calculatePurity(self, curves, data_range, voxel_to_pdg_dictionary):
'''NB - self._residuals_runner should have had calculateResiduals method called with calc_residuals = True before calling this method
'''
hit_tuples = voxel_to_pdg_dictionary.keys()
if data_range is None:
data_range = 1.0
#rescales the truth data if necessary
hits = array([[h[0], h[1], h[2]] for h in hit_tuples]) / data_range
self._residuals_runner.setDataPoints(hits)
self._residuals_runner.setLpcCurves(curves)
self._residuals_runner.calculateResiduals(True, False, False)
residuals = self._residuals_runner.getResiduals()
tau_range = self._residuals_runner.getTauRange()
purity = {}
for tau in tau_range:
pdg_code_energy_deposition = []
for i in range(len(curves)):
d = defaultdict(float)
#NB This relies upon the the keys of voxel_to_pdg_dictionary (hit_tuple) being stored in the same order as the original data points in
#lpc.Xi (as read in from file in lpcAnalyser), so the coverage_indices correctly index identify the hit.
hit_labels = [voxel_to_pdg_dictionary[hit_tuples[i]] for i in residuals['curve_residuals'][i]['coverage_indices'][tau]]
flattened_hit_labels = [pdg_code_weight for pdg_code_weight_list in hit_labels for pdg_code_weight in pdg_code_weight_list]
for pdg_code_weight in flattened_hit_labels:
d[pdg_code_weight[0]] += pdg_code_weight[1]
pdg_code_energy_deposition.append(d)
purity[tau] = pdg_code_energy_deposition
return purity
def cov(m, y=None, rowvar=1, bias=0):
"""Estimate the covariance matrix.
If m is a vector, return the variance. For matrices return the
covariance matrix.
If y is given it is treated as an additional (set of)
variable(s).
Normalization is by (N-1) where N is the number of observations
(unbiased estimate). If bias is 1 then normalization is by N.
If rowvar is non-zero (default), then each row is a variable with
observations in the columns, otherwise each column
is a variable and the observations are in the rows.
"""
X = array(m, ndmin=2, dtype=float)
if X.shape[0] == 1:
rowvar = 1
if rowvar:
axis = 0
tup = (slice(None),newaxis)
else:
axis = 1
tup = (newaxis, slice(None))
if y is not None:
y = array(y, copy=False, ndmin=2, dtype=float)
X = concatenate((X,y),axis)
X -= X.mean(axis=1-axis)[tup]
if rowvar:
N = X.shape[1]
else:
N = X.shape[0]
if bias:
fact = N*1.0
else:
fact = N-1.0
if not rowvar:
return (dot(X.T, X.conj()) / fact).squeeze()
else:
return (dot(X, X.T.conj()) / fact).squeeze()
def testCoverage(self): x = map(lambda x: x + gauss(0,0.005 + 0.3*x*x), arange(-1,1,0.005)) y = map(lambda x: x + gauss(0,0.005 + 0.3*x*x), arange(-1,1,0.005)) z = map(lambda x: x + gauss(0,0.005 + 0.3*x*x), arange(-1,1,0.005)) line = array(zip(x,y,z)) lpc = LPCImpl(h = 0.05, convergence_at = 0.0001, it = 100, mult = 2) lpc_curve = lpc.lpc(X=line) residuals_calc = LPCResiduals(line, tube_radius = 1)
def array_split(ary,indices_or_sections,axis = 0):
""" Divide an array into a list of sub-arrays.
Description:
Divide ary into a list of sub-arrays along the
specified axis. If indices_or_sections is an integer,
ary is divided into that many equally sized arrays.
If it is impossible to make an equal split, each of the
leading arrays in the list have one additional member. If
indices_or_sections is a list of sorted integers, its
entries define the indexes where ary is split.
Arguments:
ary -- N-D array.
Array to be divided into sub-arrays.
indices_or_sections -- integer or 1D array.
If integer, defines the number of (close to) equal sized
sub-arrays. If it is a 1D array of sorted indices, it
defines the indexes at which ary is divided. Any empty
list results in a single sub-array equal to the original
array.
axis -- integer. default=0.
Specifies the axis along which to split ary.
Caveats:
Currently, the default for axis is 0. This
means a 2D array is divided into multiple groups
of rows. This seems like the appropriate default,
"""
try:
Ntotal = ary.shape[axis]
except AttributeError:
Ntotal = len(ary)
try: # handle scalar case.
Nsections = len(indices_or_sections) + 1
div_points = [0] + list(indices_or_sections) + [Ntotal]
except TypeError: #indices_or_sections is a scalar, not an array.
Nsections = int(indices_or_sections)
if Nsections <= 0:
raise ValueError, 'number sections must be larger than 0.'
Neach_section,extras = divmod(Ntotal,Nsections)
section_sizes = [0] + \
extras * [Neach_section+1] + \
(Nsections-extras) * [Neach_section]
div_points = _nx.array(section_sizes).cumsum()
sub_arys = []
sary = _nx.swapaxes(ary,axis,0)
for i in range(Nsections):
st = div_points[i]; end = div_points[i+1]
sub_arys.append(_nx.swapaxes(sary[st:end],axis,0))
# there is a wierd issue with array slicing that allows
# 0x10 arrays and other such things. The following cluge is needed
# to get around this issue.
sub_arys = _replace_zero_by_x_arrays(sub_arys)
# end cluge.
return sub_arys
def fromrecords(recList,
dtype=None,
shape=None,
formats=None,
names=None,
titles=None,
aligned=False,
byteorder=None):
""" create a recarray from a list of records in text form
The data in the same field can be heterogeneous, they will be promoted
to the highest data type. This method is intended for creating
smaller record arrays. If used to create large array without formats
defined
r=fromrecords([(2,3.,'abc')]*100000)
it can be slow.
If formats is None, then this will auto-detect formats. Use list of
tuples rather than list of lists for faster processing.
>>> r=fromrecords([(456,'dbe',1.2),(2,'de',1.3)],names='col1,col2,col3')
>>> print r[0]
(456, 'dbe', 1.2)
>>> r.col1
array([456, 2])
>>> r.col2
chararray(['dbe', 'de'],
dtype='|S3')
>>> import cPickle
>>> print cPickle.loads(cPickle.dumps(r))
[(456, 'dbe', 1.2) (2, 'de', 1.3)]
"""
nfields = len(recList[0])
if formats is None and dtype is None: # slower
obj = sb.array(recList, dtype=object)
arrlist = [sb.array(obj[..., i].tolist()) for i in xrange(nfields)]
return fromarrays(arrlist,
formats=formats,
shape=shape,
names=names,
titles=titles,
aligned=aligned,
byteorder=byteorder)
if dtype is not None:
descr = sb.dtype(dtype)
else:
descr = format_parser(formats, names, titles, aligned,
byteorder)._descr
try:
retval = sb.array(recList, dtype=descr)
except TypeError: # list of lists instead of list of tuples
if (shape is None or shape == 0):
shape = len(recList)
if isinstance(shape, (int, long)):
shape = (shape, )
if len(shape) > 1:
raise ValueError, "Can only deal with 1-d array."
_array = recarray(shape, descr)
for k in xrange(_array.size):
_array[k] = tuple(recList[k])
return _array
else:
if shape is not None and retval.shape != shape:
retval.shape = shape
res = retval.view(recarray)
res.dtype = sb.dtype((record, res.dtype))
return res
def cov(m, y=None, rowvar=1, bias=0, ddof=None):
"""
Estimate a covariance matrix, given data.
Covariance indicates the level to which two variables vary together.
If we examine N-dimensional samples, :math:`X = [x_1, x_2, ... x_N]^T`,
then the covariance matrix element :math:`C_{ij}` is the covariance of
:math:`x_i` and :math:`x_j`. The element :math:`C_{ii}` is the variance
of :math:`x_i`.
Parameters
----------
m : array_like
A 1-D or 2-D array containing multiple variables and observations.
Each row of `m` represents a variable, and each column a single
observation of all those variables. Also see `rowvar` below.
y : array_like, optional
An additional set of variables and observations. `y` has the same
form as that of `m`.
rowvar : int, optional
If `rowvar` is non-zero (default), then each row represents a
variable, with observations in the columns. Otherwise, the relationship
is transposed: each column represents a variable, while the rows
contain observations.
bias : int, optional
Default normalization is by ``(N - 1)``, where ``N`` is the number of
observations given (unbiased estimate). If `bias` is 1, then
normalization is by ``N``. These values can be overridden by using
the keyword ``ddof`` in numpy versions >= 1.5.
ddof : int, optional
.. versionadded:: 1.5
If not ``None`` normalization is by ``(N - ddof)``, where ``N`` is
the number of observations; this overrides the value implied by
``bias``. The default value is ``None``.
Returns
-------
out : ndarray
The covariance matrix of the variables. The data type of `out` is np.complex128 if either `m` or `y` is complex, otherwise np.float64.
See Also
--------
corrcoef : Normalized covariance matrix
Examples
--------
Consider two variables, :math:`x_0` and :math:`x_1`, which
correlate perfectly, but in opposite directions:
>>> x = np.array([[0, 2], [1, 1], [2, 0]]).T
>>> x
array([[0, 1, 2],
[2, 1, 0]])
Note how :math:`x_0` increases while :math:`x_1` decreases. The covariance
matrix shows this clearly:
>>> np.cov(x)
array([[ 1., -1.],
[-1., 1.]])
Note that element :math:`C_{0,1}`, which shows the correlation between
:math:`x_0` and :math:`x_1`, is negative.
>>> x = np.array([[0, 2], [1, 1], [2, 0]], dtype=np.complex128).T
>>> x
array([[ 0.+0.j, 1.+0.j, 2.+0.j],
[ 2.+0.j, 1.+0.j, 0.+0.j]])
>>> npcov.cov(x)
array([[ 1.+0.j, -1.+0.j],
[-1.+0.j, 1.+0.j]])
Further, note how `x` and `y` are combined:
>>> x = [-2.1, -1, 4.3]
>>> y = [3, 1.1, 0.12]
>>> X = np.vstack((x,y))
>>> print np.cov(X)
[[ 11.71 -4.286 ]
[ -4.286 2.14413333]]
>>> print np.cov(x, y)
[[ 11.71 -4.286 ]
[ -4.286 2.14413333]]
>>> print np.cov(x)
11.71
"""
# Check inputs
if ddof is not None and ddof != int(ddof):
raise ValueError("ddof must be integer")
# Handles complex arrays too
m = np.asarray(m)
if y is None:
dtype = np.result_type(m, np.float64)
else:
y = np.asarray(y)
dtype = np.result_type(m, y, np.float64)
X = array(m, ndmin=2, dtype=dtype)
if X.shape[0] == 1:
rowvar = 1
if rowvar:
N = X.shape[1]
axis = 0
else:
N = X.shape[0]
axis = 1
# check ddof
if ddof is None:
if bias == 0:
ddof = 1
else:
ddof = 0
fact = float(N - ddof)
if fact <= 0:
warnings.warn("Degrees of freedom <= 0 for slice", RuntimeWarning)
fact = 0.0
if y is not None:
y = array(y, copy=False, ndmin=2, dtype=dtype)
X = concatenate((X, y), axis)
X -= X.mean(axis=1 - axis, keepdims=True)
if not rowvar:
return (dot(X.T, X.conj()) / fact).squeeze()
else:
return (dot(X, X.T.conj()) / fact).squeeze()
def roots(p):
"""
Return the roots of a polynomial with coefficients given in p.
The values in the rank-1 array `p` are coefficients of a polynomial.
If the length of `p` is n+1 then the polynomial is described by::
p[0] * x**n + p[1] * x**(n-1) + ... + p[n-1]*x + p[n]
Parameters
----------
p : array_like
Rank-1 array of polynomial coefficients.
Returns
-------
out : ndarray
An array containing the complex roots of the polynomial.
Raises
------
ValueError
When `p` cannot be converted to a rank-1 array.
See also
--------
poly : Find the coefficients of a polynomial with a given sequence
of roots.
polyval : Compute polynomial values.
polyfit : Least squares polynomial fit.
poly1d : A one-dimensional polynomial class.
Notes
-----
The algorithm relies on computing the eigenvalues of the
companion matrix [1]_.
References
----------
.. [1] R. A. Horn & C. R. Johnson, *Matrix Analysis*. Cambridge, UK:
Cambridge University Press, 1999, pp. 146-7.
Examples
--------
>>> coeff = [3.2, 2, 1]
>>> np.roots(coeff)
array([-0.3125+0.46351241j, -0.3125-0.46351241j])
"""
# If input is scalar, this makes it an array
p = atleast_1d(p)
if len(p.shape) != 1:
raise ValueError("Input must be a rank-1 array.")
# find non-zero array entries
non_zero = NX.nonzero(NX.ravel(p))[0]
# Return an empty array if polynomial is all zeros
if len(non_zero) == 0:
return NX.array([])
# find the number of trailing zeros -- this is the number of roots at 0.
trailing_zeros = len(p) - non_zero[-1] - 1
# strip leading and trailing zeros
p = p[int(non_zero[0]):int(non_zero[-1]) + 1]
# casting: if incoming array isn't floating point, make it floating point.
if not issubclass(p.dtype.type, (NX.floating, NX.complexfloating)):
p = p.astype(float)
N = len(p)
if N > 1:
# build companion matrix and find its eigenvalues (the roots)
A = diag(NX.ones((N - 2, ), p.dtype), -1)
A[0, :] = -p[1:] / p[0]
roots = eigvals(A)
else:
roots = NX.array([])
# tack any zeros onto the back of the array
roots = hstack((roots, NX.zeros(trailing_zeros, roots.dtype)))
return roots
def tile(A, reps):
"""
Construct an array by repeating A the number of times given by reps.
If `reps` has length ``d``, the result will have dimension of
``max(d, A.ndim)``.
If ``A.ndim < d``, `A` is promoted to be d-dimensional by prepending new
axes. So a shape (3,) array is promoted to (1, 3) for 2-D replication,
or shape (1, 1, 3) for 3-D replication. If this is not the desired
behavior, promote `A` to d-dimensions manually before calling this
function.
If ``A.ndim > d``, `reps` is promoted to `A`.ndim by pre-pending 1's to it.
Thus for an `A` of shape (2, 3, 4, 5), a `reps` of (2, 2) is treated as
(1, 1, 2, 2).
Note : Although tile may be used for broadcasting, it is strongly
recommended to use numpy's broadcasting operations and functions.
Parameters
----------
A : array_like
The input array.
reps : array_like
The number of repetitions of `A` along each axis.
Returns
-------
c : ndarray
The tiled output array.
See Also
--------
repeat : Repeat elements of an array.
broadcast_to : Broadcast an array to a new shape
Examples
--------
>>> a = np.array([0, 1, 2])
>>> np.tile(a, 2)
array([0, 1, 2, 0, 1, 2])
>>> np.tile(a, (2, 2))
array([[0, 1, 2, 0, 1, 2],
[0, 1, 2, 0, 1, 2]])
>>> np.tile(a, (2, 1, 2))
array([[[0, 1, 2, 0, 1, 2]],
[[0, 1, 2, 0, 1, 2]]])
>>> b = np.array([[1, 2], [3, 4]])
>>> np.tile(b, 2)
array([[1, 2, 1, 2],
[3, 4, 3, 4]])
>>> np.tile(b, (2, 1))
array([[1, 2],
[3, 4],
[1, 2],
[3, 4]])
>>> c = np.array([1,2,3,4])
>>> np.tile(c,(4,1))
array([[1, 2, 3, 4],
[1, 2, 3, 4],
[1, 2, 3, 4],
[1, 2, 3, 4]])
"""
try:
tup = tuple(reps)
except TypeError:
tup = (reps,)
d = len(tup)
if all(x == 1 for x in tup) and isinstance(A, _nx.ndarray):
# Fixes the problem that the function does not make a copy if A is a
# numpy array and the repetitions are 1 in all dimensions
return _nx.array(A, copy=True, subok=True, ndmin=d)
else:
# Note that no copy of zero-sized arrays is made. However since they
# have no data there is no risk of an inadvertent overwrite.
c = _nx.array(A, copy=False, subok=True, ndmin=d)
if (d < c.ndim):
tup = (1,)*(c.ndim-d) + tup
shape_out = tuple(s*t for s, t in zip(c.shape, tup))
n = c.size
if n > 0:
for dim_in, nrep in zip(c.shape, tup):
if nrep != 1:
c = c.reshape(-1, n).repeat(nrep, 0)
n //= dim_in
return c.reshape(shape_out)
def boxplotVolatilities(np,vol, lbdMean):
'''
DEFINITION: makes a boxplot of the mean (within an episode) learning rates for different probabilities and a given volatility
INPUTS:
prob: probabilities array
nv: given volatility (you have to know what the indexes in lbdMean mean in terms of volatility e.g. 1 would mean volatility .005 if the simulation was done with vol=[.001 .005])
TODO - think of a way of how not to depend on knowing beforehand of how the simulation was done
lbdMean: array with the mean chosen learning rate in an episode. lbdMean is indexed [p,v,e]
'''
l=[]
for v in xrange(len(vol)):
l.append(lbdMean[np,v,:])
figure()
boxplot(l)
pylab.xticks(range(1,len(vol)+1), vol)
xlabel('Volatility')
ylabel('Average learning rate lambda')
title('Average learning rates distributions for p='+str(prob[np]))
prob = array([.55, .65, .75, .85, .95])
vol = array([0,.001,.005,.01,.05])
path = 'data/tabuHigh/statistics/'
lbdMean = loadVar(path,'lbdMean')
#for i in xrange(len(prob)):
# boxplotVolatilities(i, vol, lbdMean)
#for i in xrange(len(vol)):
# boxplotProbabilities(prob, 3, lbdMean)
boxplotVolatilities(3, vol, lbdMean)
show()
def __execOperation__(self):
global nullValue, imagem_media, imagem_sd, imagem_cv, imagem_soma, imagem_min, imagem_max
global imagem_mediana, imagem_amplitude, images, n_linhas, n_colunas, threads_ready, n_threadings
print("executando operação")
images_super = self.paramentrosIN_carregados["images"]
print("Numero de imagens para ler: " + str(len(images_super)))
nullValue = np.double(images_super[0].getRasterInformation()["NoData"])
statistics = self.paramentrosIN_carregados["statistics"]
print("Estatisticas a fazer: ", statistics)
do = dict()
do["Media"] = "media" in statistics
do["CV"] = "cv" in statistics
do["SD"] = "sd" in statistics
do["Soma"] = "soma" in statistics
do["Min"] = "min" in statistics
do["Max"] = "max" in statistics
do["Mediana"] = "mediana" in statistics
do["Amplitude"] = "amplitude" in statistics
images = images_super.loadListRasterData()
print("Numero de imagens lidas: " + str(len(images)))
n_linhas = len(images[0])
n_colunas = len(images[0][0])
for img in images:
if len(img) != n_linhas or len(img[0]) != n_colunas:
raise IndexError(
"Erro - As imagens precisam ter o mesmo número de linhas e colunas"
)
print("numero de colunas e linhas: " + str(n_linhas) + " : " +
str(n_colunas))
#imagem_referencia = [[0 for x in range(n_colunas)] for x in range(n_linhas)]
imagem_referencia = np.zeros((n_linhas, n_colunas))
imagem_out = dict
if do["Media"]:
imagem_out["media"] = array(
imagem_referencia) #.astype(dtype="int16")
if do["CV"]:
imagem_out["cv"] = array(
imagem_referencia) #.astype(dtype="int16")
if do["SD"]:
imagem_out["sd"] = array(
imagem_referencia) #.astype(dtype="int16")
if do["Soma"]:
imagem_out["soma"] = array(
imagem_referencia) #.astype(dtype="int16")
if do["Min"]:
imagem_out["min"] = array(
imagem_referencia) #.astype(dtype="int16")
if do["Max"]:
imagem_out["max"] = array(
imagem_referencia) #.astype(dtype="int16")
if do["Mediana"]:
imagem_out["mediana"] = array(
imagem_referencia) #.astype(dtype="int16")
if do["Amplitude"]:
imagem_out["amplitude"] = array(
imagem_referencia) #.astype(dtype="int16")
print("processando:")
numero_de_nucleos = GeneralTools.available_cpu_count()
n_threadings = int(numero_de_nucleos - 2)
print("Numero de threads", n_threadings)
threads_ready = 0
pool = Pool()
#pool = multiprocessing.Pool(processes=n_threadings)
for i in range(0, n_threadings):
#t = threading.Thread(target=thread_process, args=(n_linhas/n_threadings*i, n_linhas/n_threadings*(i+1)))
#t.start()
linha_inicial = n_linhas / n_threadings * i
linha_final = n_linhas / n_threadings * (i + 1)
p = Process(target=thread_process,
args=(linha_inicial, linha_final))
p.start()
#pool.map(thread_process(n_linhas/n_threadings*i, n_linhas/n_threadings*(i+1)))
#pool.close()
while (threads_ready < n_threadings):
pass
print("Arrumando imagens de saida")
saida = SerialFile()
saida.metadata = self.paramentrosIN_carregados["images"][0].metadata
if do["Media"]:
imagem_media = RasterFile(data=imagem_media)
imagem_media.metadata = saida.metadata
imagem_media.file_name = "imagem_media"
saida.append(imagem_media)
if do["CV"]:
imagem_cv = RasterFile(data=imagem_cv)
imagem_cv.metadata = saida.metadata
imagem_cv.file_name = "imagem_coeficiente_variacao"
saida.append(imagem_cv)
if do["SD"]:
imagem_sd = RasterFile(data=imagem_sd)
imagem_sd.metadata = saida.metadata
imagem_sd.file_name = "imagem_desvio_padrao"
saida.append(imagem_sd)
if do["Soma"]:
imagem_soma = RasterFile(data=imagem_soma)
imagem_soma.metadata = saida.metadata
imagem_soma.file_name = "imagem_soma"
saida.append(imagem_soma)
if do["Min"]:
imagem_min = RasterFile(data=imagem_min)
imagem_min.metadata = saida.metadata
imagem_min.file_name = "imagem_minimo"
saida.append(imagem_min)
if do["Max"]:
imagem_max = RasterFile(data=imagem_max)
imagem_max.metadata = saida.metadata
imagem_max.file_name = "imagem_maximo"
saida.append(imagem_max)
if do["Mediana"]:
imagem_mediana = RasterFile(data=imagem_mediana)
imagem_mediana.metadata = saida.metadata
imagem_mediana.file_name = "imagem_mediana"
saida.append(imagem_mediana)
if do["Amplitude"]:
imagem_amplitude = RasterFile(data=imagem_amplitude)
imagem_amplitude.metadata = saida.metadata
imagem_amplitude.file_name = "imagem_amplitude"
saida.append(imagem_amplitude)
print("imagens prontas para gravar, statistical stractor completo")
return saida
def __getitem__(self, key):
trans1d = self.trans1d
ndmin = self.ndmin
if isinstance(key, str):
frame = sys._getframe().f_back
mymat = matrix.bmat(key, frame.f_globals, frame.f_locals)
return mymat
if not isinstance(key, tuple):
key = (key, )
objs = []
scalars = []
arraytypes = []
scalartypes = []
for k in range(len(key)):
scalar = False
if isinstance(key[k], slice):
step = key[k].step
start = key[k].start
stop = key[k].stop
if start is None:
start = 0
if step is None:
step = 1
if isinstance(step, complex):
size = int(abs(step))
newobj = function_base.linspace(start, stop, num=size)
else:
newobj = _nx.arange(start, stop, step)
if ndmin > 1:
newobj = array(newobj, copy=False, ndmin=ndmin)
if trans1d != -1:
newobj = newobj.swapaxes(-1, trans1d)
elif isinstance(key[k], str):
if k != 0:
raise ValueError("special directives must be the "
"first entry.")
key0 = key[0]
if key0 in 'rc':
self.matrix = True
self.col = (key0 == 'c')
continue
if ',' in key0:
vec = key0.split(',')
try:
self.axis, ndmin = \
[int(x) for x in vec[:2]]
if len(vec) == 3:
trans1d = int(vec[2])
continue
except:
raise ValueError("unknown special directive")
try:
self.axis = int(key[k])
continue
except (ValueError, TypeError):
raise ValueError("unknown special directive")
elif type(key[k]) in ScalarType:
newobj = array(key[k], ndmin=ndmin)
scalars.append(k)
scalar = True
scalartypes.append(newobj.dtype)
else:
newobj = key[k]
if ndmin > 1:
tempobj = array(newobj, copy=False, subok=True)
newobj = array(newobj, copy=False, subok=True, ndmin=ndmin)
if trans1d != -1 and tempobj.ndim < ndmin:
k2 = ndmin - tempobj.ndim
if (trans1d < 0):
trans1d += k2 + 1
defaxes = list(range(ndmin))
k1 = trans1d
axes = defaxes[:k1] + defaxes[k2:] + \
defaxes[k1:k2]
newobj = newobj.transpose(axes)
del tempobj
objs.append(newobj)
if not scalar and isinstance(newobj, _nx.ndarray):
arraytypes.append(newobj.dtype)
# Esure that scalars won't up-cast unless warranted
final_dtype = find_common_type(arraytypes, scalartypes)
if final_dtype is not None:
for k in scalars:
objs[k] = objs[k].astype(final_dtype)
res = _nx.concatenate(tuple(objs), axis=self.axis)
return self._retval(res)
def nan_to_num(x, copy=True):
"""
Replace NaN with zero and infinity with large finite numbers.
If `x` is inexact, NaN is replaced by zero, and infinity and -infinity
replaced by the respectively largest and most negative finite floating
point values representable by ``x.dtype``.
For complex dtypes, the above is applied to each of the real and
imaginary components of `x` separately.
If `x` is not inexact, then no replacements are made.
Parameters
----------
x : scalar or array_like
Input data.
copy : bool, optional
Whether to create a copy of `x` (True) or to replace values
in-place (False). The in-place operation only occurs if
casting to an array does not require a copy.
Default is True.
.. versionadded:: 1.13
Returns
-------
out : ndarray
`x`, with the non-finite values replaced. If `copy` is False, this may
be `x` itself.
See Also
--------
isinf : Shows which elements are positive or negative infinity.
isneginf : Shows which elements are negative infinity.
isposinf : Shows which elements are positive infinity.
isnan : Shows which elements are Not a Number (NaN).
isfinite : Shows which elements are finite (not NaN, not infinity)
Notes
-----
NumPy uses the IEEE Standard for Binary Floating-Point for Arithmetic
(IEEE 754). This means that Not a Number is not equivalent to infinity.
Examples
--------
>>> np.nan_to_num(np.inf)
1.7976931348623157e+308
>>> np.nan_to_num(-np.inf)
-1.7976931348623157e+308
>>> np.nan_to_num(np.nan)
0.0
>>> x = np.array([np.inf, -np.inf, np.nan, -128, 128])
>>> np.nan_to_num(x)
array([ 1.79769313e+308, -1.79769313e+308, 0.00000000e+000, # may vary
-1.28000000e+002, 1.28000000e+002])
>>> y = np.array([complex(np.inf, np.nan), np.nan, complex(np.nan, np.inf)])
array([ 1.79769313e+308, -1.79769313e+308, 0.00000000e+000, # may vary
-1.28000000e+002, 1.28000000e+002])
>>> np.nan_to_num(y)
array([ 1.79769313e+308 +0.00000000e+000j, # may vary
0.00000000e+000 +0.00000000e+000j,
0.00000000e+000 +1.79769313e+308j])
"""
x = _nx.array(x, subok=True, copy=copy)
xtype = x.dtype.type
isscalar = (x.ndim == 0)
if not issubclass(xtype, _nx.inexact):
return x[()] if isscalar else x
iscomplex = issubclass(xtype, _nx.complexfloating)
dest = (x.real, x.imag) if iscomplex else (x, )
maxf, minf = _getmaxmin(x.real.dtype)
for d in dest:
_nx.copyto(d, 0.0, where=isnan(d))
_nx.copyto(d, maxf, where=isposinf(d))
_nx.copyto(d, minf, where=isneginf(d))
return x[()] if isscalar else x
def __getitem__(self, key):
# handle matrix builder syntax
if isinstance(key, str):
frame = sys._getframe().f_back
mymat = matrixlib.bmat(key, frame.f_globals, frame.f_locals)
return mymat
if not isinstance(key, tuple):
key = (key, )
# copy attributes, since they can be overridden in the first argument
trans1d = self.trans1d
ndmin = self.ndmin
matrix = self.matrix
axis = self.axis
objs = []
scalars = []
arraytypes = []
scalartypes = []
for k, item in enumerate(key):
scalar = False
if isinstance(item, slice):
step = item.step
start = item.start
stop = item.stop
if start is None:
start = 0
if step is None:
step = 1
if isinstance(step, complex):
size = int(abs(step))
newobj = linspace(start, stop, num=size)
else:
newobj = _nx.arange(start, stop, step)
if ndmin > 1:
newobj = array(newobj, copy=False, ndmin=ndmin)
if trans1d != -1:
newobj = newobj.swapaxes(-1, trans1d)
elif isinstance(item, str):
if k != 0:
raise ValueError("special directives must be the "
"first entry.")
if item in ('r', 'c'):
matrix = True
col = (item == 'c')
continue
if ',' in item:
vec = item.split(',')
try:
axis, ndmin = [int(x) for x in vec[:2]]
if len(vec) == 3:
trans1d = int(vec[2])
continue
except Exception:
raise ValueError("unknown special directive")
try:
axis = int(item)
continue
except (ValueError, TypeError):
raise ValueError("unknown special directive")
elif type(item) in ScalarType:
newobj = array(item, ndmin=ndmin)
scalars.append(len(objs))
scalar = True
scalartypes.append(newobj.dtype)
else:
item_ndim = ndim(item)
newobj = array(item, copy=False, subok=True, ndmin=ndmin)
if trans1d != -1 and item_ndim < ndmin:
k2 = ndmin - item_ndim
k1 = trans1d
if k1 < 0:
k1 += k2 + 1
defaxes = list(range(ndmin))
axes = defaxes[:k1] + defaxes[k2:] + defaxes[k1:k2]
newobj = newobj.transpose(axes)
objs.append(newobj)
if not scalar and isinstance(newobj, _nx.ndarray):
arraytypes.append(newobj.dtype)
# Ensure that scalars won't up-cast unless warranted
final_dtype = find_common_type(arraytypes, scalartypes)
if final_dtype is not None:
for k in scalars:
objs[k] = objs[k].astype(final_dtype)
res = self.concatenate(tuple(objs), axis=axis)
if matrix:
oldndim = res.ndim
res = self.makemat(res)
if oldndim == 1 and col:
res = res.T
return res
def tile(A, reps):
"""
Construct an array by repeating A the number of times given by reps.
If `reps` has length ``d``, the result will have dimension of
``max(d, A.ndim)``.
If ``A.ndim < d``, `A` is promoted to be d-dimensional by prepending new
axes. So a shape (3,) array is promoted to (1, 3) for 2-D replication,
or shape (1, 1, 3) for 3-D replication. If this is not the desired
behavior, promote `A` to d-dimensions manually before calling this
function.
If ``A.ndim > d``, `reps` is promoted to `A`.ndim by pre-pending 1's to it.
Thus for an `A` of shape (2, 3, 4, 5), a `reps` of (2, 2) is treated as
(1, 1, 2, 2).
Parameters
----------
A : array_like
The input array.
reps : array_like
The number of repetitions of `A` along each axis.
Returns
-------
c : ndarray
The tiled output array.
See Also
--------
repeat : Repeat elements of an array.
Examples
--------
>>> a = np.array([0, 1, 2])
>>> np.tile(a, 2)
array([0, 1, 2, 0, 1, 2])
>>> np.tile(a, (2, 2))
array([[0, 1, 2, 0, 1, 2],
[0, 1, 2, 0, 1, 2]])
>>> np.tile(a, (2, 1, 2))
array([[[0, 1, 2, 0, 1, 2]],
[[0, 1, 2, 0, 1, 2]]])
>>> b = np.array([[1, 2], [3, 4]])
>>> np.tile(b, 2)
array([[1, 2, 1, 2],
[3, 4, 3, 4]])
>>> np.tile(b, (2, 1))
array([[1, 2],
[3, 4],
[1, 2],
[3, 4]])
"""
try:
tup = tuple(reps)
except TypeError:
tup = (reps, )
d = len(tup)
c = _nx.array(A, copy=False, subok=True, ndmin=d)
shape = list(c.shape)
n = max(c.size, 1)
if (d < c.ndim):
tup = (1, ) * (c.ndim - d) + tup
for i, nrep in enumerate(tup):
if nrep != 1:
c = c.reshape(-1, n).repeat(nrep, 0)
dim_in = shape[i]
dim_out = dim_in * nrep
shape[i] = dim_out
n //= max(dim_in, 1)
return c.reshape(shape)
def array_split(ary, indices_or_sections, axis=0, two_dimensional=False):
"""
Split an array into multiple sub-arrays.
Please refer to the ``split`` documentation. The only difference
between these functions is that ``array_split`` allows
`indices_or_sections` to be an integer that does *not* equally
divide the axis. For an array of length l that should be split
into n sections, it returns l % n sub-arrays of size l//n + 1
and the rest of size l//n. In the case where two_dimensional is set
to True, this holds for both elements of `indices_or_sections`.
See Also
--------
split : Split array into multiple sub-arrays of equal size.
Examples
--------
>>> x = np.arange(8.0)
>>> np.array_split(x, 3)
[array([0., 1., 2.]), array([3., 4., 5.]), array([6., 7.])]
>>> x = np.arange(7.0)
>>> np.array_split(x, 3)
[array([0., 1., 2.]), array([3., 4.]), array([5., 6.])]
>>> x = np.reshape(np.arange(16), (4, 4))
>>> np.array_split(x, [3, 3], 0, True)
[array([[0, 1]),
array([4, 5]]),
array([[2], [6]]),
array([[3], [7]]),
array([[8, 9]]),
array([[10]]),
array([[11]]),
array([[12, 13]]),
array([[14]]),
array([[15]])]
"""
if two_dimensional:
try:
indices_or_sections[1]
except (IndexError, TypeError):
raise ValueError('indices_or_sections must be an array of length 2.')
subarrays = array_split(ary, indices_or_sections[0], axis=0, two_dimensional=False)
res = []
for subarray in subarrays:
res.extend(array_split(subarray, indices_or_sections[1], axis=1, two_dimensional=False))
return res
try:
Ntotal = ary.shape[axis]
except AttributeError:
Ntotal = len(ary)
try:
# handle array case.
Nsections = len(indices_or_sections) + 1
div_points = [0] + list(indices_or_sections) + [Ntotal]
except TypeError:
# indices_or_sections is a scalar, not an array.
Nsections = int(indices_or_sections)
if Nsections <= 0:
raise ValueError('number sections must be larger than 0.')
Neach_section, extras = divmod(Ntotal, Nsections)
section_sizes = ([0] +
extras * [Neach_section+1] +
(Nsections-extras) * [Neach_section])
div_points = _nx.array(section_sizes, dtype=_nx.intp).cumsum()
sub_arys = []
sary = _nx.swapaxes(ary, axis, 0)
for i in range(Nsections):
st = div_points[i]
end = div_points[i + 1]
sub_arys.append(_nx.swapaxes(sary[st:end], axis, 0))
return sub_arys
def nan_to_num(x, copy=True, nan=0.0, posinf=None, neginf=None):
"""
Replace NaN with zero and infinity with large finite numbers (default
behaviour) or with the numbers defined by the user using the `nan`,
`posinf` and/or `neginf` keywords.
If `x` is inexact, NaN is replaced by zero or by the user defined value in
`nan` keyword, infinity is replaced by the largest finite floating point
values representable by ``x.dtype`` or by the user defined value in
`posinf` keyword and -infinity is replaced by the most negative finite
floating point values representable by ``x.dtype`` or by the user defined
value in `neginf` keyword.
For complex dtypes, the above is applied to each of the real and
imaginary components of `x` separately.
If `x` is not inexact, then no replacements are made.
Parameters
----------
x : scalar or array_like
Input data.
copy : bool, optional
Whether to create a copy of `x` (True) or to replace values
in-place (False). The in-place operation only occurs if
casting to an array does not require a copy.
Default is True.
nan : int, float, optional
Value to be used to fill NaN values. If no value is passed
then NaN values will be replaced with 0.0.
posinf : int, float, optional
Value to be used to fill positive infinity values. If no value is
passed then positive infinity values will be replaced with a very
large number.
neginf : int, float, optional
Value to be used to fill negative infinity values. If no value is
passed then negative infinity values will be replaced with a very
small (or negative) number.
.. versionadded:: 1.13
Returns
-------
out : ndarray
`x`, with the non-finite values replaced. If `copy` is False, this may
be `x` itself.
See Also
--------
isinf : Shows which elements are positive or negative infinity.
isneginf : Shows which elements are negative infinity.
isposinf : Shows which elements are positive infinity.
isnan : Shows which elements are Not a Number (NaN).
isfinite : Shows which elements are finite (not NaN, not infinity)
Notes
-----
NumPy uses the IEEE Standard for Binary Floating-Point for Arithmetic
(IEEE 754). This means that Not a Number is not equivalent to infinity.
Examples
--------
>>> np.nan_to_num(np.inf)
1.7976931348623157e+308
>>> np.nan_to_num(-np.inf)
-1.7976931348623157e+308
>>> np.nan_to_num(np.nan)
0.0
>>> x = np.array([np.inf, -np.inf, np.nan, -128, 128])
>>> np.nan_to_num(x)
array([ 1.79769313e+308, -1.79769313e+308, 0.00000000e+000, # may vary
-1.28000000e+002, 1.28000000e+002])
>>> np.nan_to_num(x, nan=-9999, posinf=33333333, neginf=33333333)
array([ 3.3333333e+07, 3.3333333e+07, -9.9990000e+03,
-1.2800000e+02, 1.2800000e+02])
>>> y = np.array([complex(np.inf, np.nan), np.nan, complex(np.nan, np.inf)])
array([ 1.79769313e+308, -1.79769313e+308, 0.00000000e+000, # may vary
-1.28000000e+002, 1.28000000e+002])
>>> np.nan_to_num(y)
array([ 1.79769313e+308 +0.00000000e+000j, # may vary
0.00000000e+000 +0.00000000e+000j,
0.00000000e+000 +1.79769313e+308j])
>>> np.nan_to_num(y, nan=111111, posinf=222222)
array([222222.+111111.j, 111111. +0.j, 111111.+222222.j])
"""
x = _nx.array(x, subok=True, copy=copy)
xtype = x.dtype.type
isscalar = (x.ndim == 0)
if not issubclass(xtype, _nx.inexact):
return x[()] if isscalar else x
iscomplex = issubclass(xtype, _nx.complexfloating)
dest = (x.real, x.imag) if iscomplex else (x, )
maxf, minf = _getmaxmin(x.real.dtype)
if posinf is not None:
maxf = posinf
if neginf is not None:
minf = neginf
for d in dest:
idx_nan = isnan(d)
idx_posinf = isposinf(d)
idx_neginf = isneginf(d)
_nx.copyto(d, nan, where=idx_nan)
_nx.copyto(d, maxf, where=idx_posinf)
_nx.copyto(d, minf, where=idx_neginf)
return x[()] if isscalar else x
def nan_to_num(x, copy=True):
"""
Replace nan with zero and inf with finite numbers.
Returns an array or scalar replacing Not a Number (NaN) with zero,
(positive) infinity with a very large number and negative infinity
with a very small (or negative) number.
Parameters
----------
x : array_like
Input data.
copy : bool, optional
Whether to create a copy of `x` (True) or to replace values
in-place (False). The in-place operation only occurs if
casting to an array does not require a copy.
Default is True.
.. versionadded:: 1.13
Returns
-------
out : ndarray
New Array with the same shape as `x` and dtype of the element in
`x` with the greatest precision. If `x` is inexact, then NaN is
replaced by zero, and infinity (-infinity) is replaced by the
largest (smallest or most negative) floating point value that fits
in the output dtype. If `x` is not inexact, then a copy of `x` is
returned.
See Also
--------
isinf : Shows which elements are positive or negative infinity.
isneginf : Shows which elements are negative infinity.
isposinf : Shows which elements are positive infinity.
isnan : Shows which elements are Not a Number (NaN).
isfinite : Shows which elements are finite (not NaN, not infinity)
Notes
-----
NumPy uses the IEEE Standard for Binary Floating-Point for Arithmetic
(IEEE 754). This means that Not a Number is not equivalent to infinity.
Examples
--------
>>> np.set_printoptions(precision=8)
>>> x = np.array([np.inf, -np.inf, np.nan, -128, 128])
>>> np.nan_to_num(x)
array([ 1.79769313e+308, -1.79769313e+308, 0.00000000e+000,
-1.28000000e+002, 1.28000000e+002])
"""
x = _nx.array(x, subok=True, copy=copy)
xtype = x.dtype.type
if not issubclass(xtype, _nx.inexact):
return x
iscomplex = issubclass(xtype, _nx.complexfloating)
isscalar = (x.ndim == 0)
x = x[None] if isscalar else x
dest = (x.real, x.imag) if iscomplex else (x, )
maxf, minf = _getmaxmin(x.real.dtype)
for d in dest:
_nx.copyto(d, 0.0, where=isnan(d))
_nx.copyto(d, maxf, where=isposinf(d))
_nx.copyto(d, minf, where=isneginf(d))
return x[0] if isscalar else x
def nan_to_num(x):
"""
Replace nan with zero and inf with finite numbers.
Returns an array or scalar replacing Not a Number (NaN) with zero,
(positive) infinity with a very large number and negative infinity
with a very small (or negative) number.
Parameters
----------
x : array_like
Input data.
Returns
-------
out : ndarray, float
Array with the same shape as `x` and dtype of the element in `x` with
the greatest precision. NaN is replaced by zero, and infinity
(-infinity) is replaced by the largest (smallest or most negative)
floating point value that fits in the output dtype. All finite numbers
are upcast to the output dtype (default float64).
See Also
--------
isinf : Shows which elements are negative or negative infinity.
isneginf : Shows which elements are negative infinity.
isposinf : Shows which elements are positive infinity.
isnan : Shows which elements are Not a Number (NaN).
isfinite : Shows which elements are finite (not NaN, not infinity)
Notes
-----
Numpy uses the IEEE Standard for Binary Floating-Point for Arithmetic
(IEEE 754). This means that Not a Number is not equivalent to infinity.
Examples
--------
>>> x = np.array([np.inf, -np.inf, np.nan, -128, 128])
>>> np.nan_to_num(x)
array([ 1.79769313e+308, -1.79769313e+308, 0.00000000e+000,
-1.28000000e+002, 1.28000000e+002])
"""
try:
t = x.dtype.type
except AttributeError:
t = obj2sctype(type(x))
if issubclass(t, _nx.complexfloating):
return nan_to_num(x.real) + 1j * nan_to_num(x.imag)
else:
try:
y = x.copy()
except AttributeError:
y = array(x)
if not issubclass(t, _nx.integer):
if not y.shape:
y = array([x])
scalar = True
else:
scalar = False
are_inf = isposinf(y)
are_neg_inf = isneginf(y)
are_nan = isnan(y)
maxf, minf = _getmaxmin(y.dtype.type)
y[are_nan] = 0
y[are_inf] = maxf
y[are_neg_inf] = minf
if scalar:
y = y[0]
return y
def array(obj,
dtype=None,
shape=None,
offset=0,
strides=None,
formats=None,
names=None,
titles=None,
aligned=False,
byteorder=None,
copy=True):
"""Construct a record array from a wide-variety of objects.
"""
if isinstance(obj, (type(None), str, file)) and (formats is None) \
and (dtype is None):
raise ValueError("Must define formats (or dtype) if object is "\
"None, string, or an open file")
kwds = {}
if dtype is not None:
dtype = sb.dtype(dtype)
elif formats is not None:
dtype = format_parser(formats, names, titles, aligned,
byteorder)._descr
else:
kwds = {
'formats': formats,
'names': names,
'titles': titles,
'aligned': aligned,
'byteorder': byteorder
}
if obj is None:
if shape is None:
raise ValueError("Must define a shape if obj is None")
return recarray(shape, dtype, buf=obj, offset=offset, strides=strides)
elif isinstance(obj, str):
return fromstring(obj, dtype, shape=shape, offset=offset, **kwds)
elif isinstance(obj, (list, tuple)):
if isinstance(obj[0], (tuple, list)):
return fromrecords(obj, dtype=dtype, shape=shape, **kwds)
else:
return fromarrays(obj, dtype=dtype, shape=shape, **kwds)
elif isinstance(obj, recarray):
if dtype is not None and (obj.dtype != dtype):
new = obj.view(dtype)
else:
new = obj
if copy:
new = new.copy()
return new
elif isinstance(obj, file):
return fromfile(obj, dtype=dtype, shape=shape, offset=offset)
elif isinstance(obj, ndarray):
if dtype is not None and (obj.dtype != dtype):
new = obj.view(dtype)
else:
new = obj
if copy:
new = new.copy()
res = new.view(recarray)
if issubclass(res.dtype.type, nt.void):
res.dtype = sb.dtype((record, res.dtype))
return res
else:
interface = getattr(obj, "__array_interface__", None)
if interface is None or not isinstance(interface, dict):
raise ValueError("Unknown input type")
obj = sb.array(obj)
if dtype is not None and (obj.dtype != dtype):
obj = obj.view(dtype)
res = obj.view(recarray)
if issubclass(res.dtype.type, nt.void):
res.dtype = sb.dtype((record, res.dtype))
return res
def average(a, axis=None, weights=None, returned=False):
"""average(a, axis=None weights=None, returned=False)
Average the array over the given axis. If the axis is None,
average over all dimensions of the array. Equivalent to
a.mean(axis) and to
a.sum(axis) / size(a, axis)
If weights are given, result is:
sum(a * weights,axis) / sum(weights,axis),
where the weights must have a's shape or be 1D with length the
size of a in the given axis. Integer weights are converted to
Float. Not specifying weights is equivalent to specifying
weights that are all 1.
If 'returned' is True, return a tuple: the result and the sum of
the weights or count of values. The shape of these two results
will be the same.
Raises ZeroDivisionError if appropriate. (The version in MA does
not -- it returns masked values).
"""
if axis is None:
a = array(a).ravel()
if weights is None:
n = add.reduce(a)
d = len(a) * 1.0
else:
w = array(weights).ravel() * 1.0
n = add.reduce(multiply(a, w))
d = add.reduce(w)
else:
a = array(a)
ash = a.shape
if ash == ():
a.shape = (1,)
if weights is None:
n = add.reduce(a, axis)
d = ash[axis] * 1.0
if returned:
d = ones(n.shape) * d
else:
w = array(weights, copy=False) * 1.0
wsh = w.shape
if wsh == ():
wsh = (1,)
if wsh == ash:
n = add.reduce(a*w, axis)
d = add.reduce(w, axis)
elif wsh == (ash[axis],):
ni = ash[axis]
r = [newaxis]*ni
r[axis] = slice(None, None, 1)
w1 = eval("w["+repr(tuple(r))+"]*ones(ash, float)")
n = add.reduce(a*w1, axis)
d = add.reduce(w1, axis)
else:
raise ValueError, 'averaging weights have wrong shape'
if not isinstance(d, ndarray):
if d == 0.0:
raise ZeroDivisionError, 'zero denominator in average()'
if returned:
return n/d, d
else:
return n/d
def fromfile(fd,
dtype=None,
shape=None,
offset=0,
formats=None,
names=None,
titles=None,
aligned=False,
byteorder=None):
"""Create an array from binary file data
If file is a string then that file is opened, else it is assumed
to be a file object.
>>> from tempfile import TemporaryFile
>>> a = N.empty(10,dtype='f8,i4,a5')
>>> a[5] = (0.5,10,'abcde')
>>>
>>> fd=TemporaryFile()
>>> a = a.newbyteorder('<')
>>> a.tofile(fd)
>>>
>>> fd.seek(0)
>>> r=fromfile(fd, formats='f8,i4,a5', shape=10, byteorder='<')
>>> print r[5]
(0.5, 10, 'abcde')
>>> r.shape
(10,)
"""
if (shape is None or shape == 0):
shape = (-1, )
elif isinstance(shape, (int, long)):
shape = (shape, )
name = 0
if isinstance(fd, str):
name = 1
fd = open(fd, 'rb')
if (offset > 0):
fd.seek(offset, 1)
size = get_remaining_size(fd)
if dtype is not None:
descr = sb.dtype(dtype)
else:
descr = format_parser(formats, names, titles, aligned,
byteorder)._descr
itemsize = descr.itemsize
shapeprod = sb.array(shape).prod()
shapesize = shapeprod * itemsize
if shapesize < 0:
shape = list(shape)
shape[shape.index(-1)] = size / -shapesize
shape = tuple(shape)
shapeprod = sb.array(shape).prod()
nbytes = shapeprod * itemsize
if nbytes > size:
raise ValueError(
"Not enough bytes left in file for specified shape and type")
# create the array
_array = recarray(shape, descr)
nbytesread = fd.readinto(_array.data)
if nbytesread != nbytes:
raise IOError("Didn't read as many bytes as expected")
if name:
fd.close()
return _array
def copy(a):
"""Return an array copy of the given object.
"""
return array(a, copy=True)
def delete(arr, obj, axis=None):
"""Return a new array with sub-arrays along an axis deleted.
Return a new array with the sub-arrays (i.e. rows or columns)
deleted along the given axis as specified by obj
obj may be a slice_object (s_[3:5:2]) or an integer
or an array of integers indicated which sub-arrays to
remove.
If axis is None, then ravel the array first.
Example:
>>> arr = [[3,4,5],
... [1,2,3],
... [6,7,8]]
>>> delete(arr, 1, 1)
array([[3, 5],
[1, 3],
[6, 8]])
>>> delete(arr, 1, 0)
array([[3, 4, 5],
[6, 7, 8]])
"""
wrap = None
if type(arr) is not ndarray:
try:
wrap = arr.__array_wrap__
except AttributeError:
pass
arr = asarray(arr)
ndim = arr.ndim
if axis is None:
if ndim != 1:
arr = arr.ravel()
ndim = arr.ndim;
axis = ndim-1;
if ndim == 0:
if wrap:
return wrap(arr)
else:
return arr.copy()
slobj = [slice(None)]*ndim
N = arr.shape[axis]
newshape = list(arr.shape)
if isinstance(obj, (int, long, integer)):
if (obj < 0): obj += N
if (obj < 0 or obj >=N):
raise ValueError, "invalid entry"
newshape[axis]-=1;
new = empty(newshape, arr.dtype, arr.flags.fnc)
slobj[axis] = slice(None, obj)
new[slobj] = arr[slobj]
slobj[axis] = slice(obj,None)
slobj2 = [slice(None)]*ndim
slobj2[axis] = slice(obj+1,None)
new[slobj] = arr[slobj2]
elif isinstance(obj, slice):
start, stop, step = obj.indices(N)
numtodel = len(xrange(start, stop, step))
if numtodel <= 0:
if wrap:
return wrap(new)
else:
return arr.copy()
newshape[axis] -= numtodel
new = empty(newshape, arr.dtype, arr.flags.fnc)
# copy initial chunk
if start == 0:
pass
else:
slobj[axis] = slice(None, start)
new[slobj] = arr[slobj]
# copy end chunck
if stop == N:
pass
else:
slobj[axis] = slice(stop-numtodel,None)
slobj2 = [slice(None)]*ndim
slobj2[axis] = slice(stop, None)
new[slobj] = arr[slobj2]
# copy middle pieces
if step == 1:
pass
else: # use array indexing.
obj = arange(start, stop, step, dtype=intp)
all = arange(start, stop, dtype=intp)
obj = setdiff1d(all, obj)
slobj[axis] = slice(start, stop-numtodel)
slobj2 = [slice(None)]*ndim
slobj2[axis] = obj
new[slobj] = arr[slobj2]
else: # default behavior
obj = array(obj, dtype=intp, copy=0, ndmin=1)
all = arange(N, dtype=intp)
obj = setdiff1d(all, obj)
slobj[axis] = obj
new = arr[slobj]
if wrap:
return wrap(new)
else:
return new
def histogramdd(sample, bins=10, range=None, normed=False, weights=None):
"""histogramdd(sample, bins=10, range=None, normed=False, weights=None)
Return the N-dimensional histogram of the sample.
Parameters:
sample : sequence or array
A sequence containing N arrays or an NxM array. Input data.
bins : sequence or scalar
A sequence of edge arrays, a sequence of bin counts, or a scalar
which is the bin count for all dimensions. Default is 10.
range : sequence
A sequence of lower and upper bin edges. Default is [min, max].
normed : boolean
If False, return the number of samples in each bin, if True,
returns the density.
weights : array
Array of weights. The weights are normed only if normed is True.
Should the sum of the weights not equal N, the total bin count will
not be equal to the number of samples.
Returns:
hist : array
Histogram array.
edges : list
List of arrays defining the lower bin edges.
SeeAlso:
histogram
Example
>>> x = random.randn(100,3)
>>> hist3d, edges = histogramdd(x, bins = (5, 6, 7))
"""
try:
# Sample is an ND-array.
N, D = sample.shape
except (AttributeError, ValueError):
# Sample is a sequence of 1D arrays.
sample = atleast_2d(sample).T
N, D = sample.shape
nbin = empty(D, int)
edges = D*[None]
dedges = D*[None]
if weights is not None:
weights = asarray(weights)
try:
M = len(bins)
if M != D:
raise AttributeError, 'The dimension of bins must be a equal to the dimension of the sample x.'
except TypeError:
bins = D*[bins]
# Select range for each dimension
# Used only if number of bins is given.
if range is None:
smin = atleast_1d(array(sample.min(0), float))
smax = atleast_1d(array(sample.max(0), float))
else:
smin = zeros(D)
smax = zeros(D)
for i in arange(D):
smin[i], smax[i] = range[i]
# Make sure the bins have a finite width.
for i in arange(len(smin)):
if smin[i] == smax[i]:
smin[i] = smin[i] - .5
smax[i] = smax[i] + .5
# Create edge arrays
for i in arange(D):
if isscalar(bins[i]):
nbin[i] = bins[i] + 2 # +2 for outlier bins
edges[i] = linspace(smin[i], smax[i], nbin[i]-1)
else:
edges[i] = asarray(bins[i], float)
nbin[i] = len(edges[i])+1 # +1 for outlier bins
dedges[i] = diff(edges[i])
nbin = asarray(nbin)
# Compute the bin number each sample falls into.
Ncount = {}
for i in arange(D):
Ncount[i] = digitize(sample[:,i], edges[i])
# Using digitize, values that fall on an edge are put in the right bin.
# For the rightmost bin, we want values equal to the right
# edge to be counted in the last bin, and not as an outlier.
outliers = zeros(N, int)
for i in arange(D):
# Rounding precision
decimal = int(-log10(dedges[i].min())) +6
# Find which points are on the rightmost edge.
on_edge = where(around(sample[:,i], decimal) == around(edges[i][-1], decimal))[0]
# Shift these points one bin to the left.
Ncount[i][on_edge] -= 1
# Flattened histogram matrix (1D)
hist = zeros(nbin.prod(), float)
# Compute the sample indices in the flattened histogram matrix.
ni = nbin.argsort()
shape = []
xy = zeros(N, int)
for i in arange(0, D-1):
xy += Ncount[ni[i]] * nbin[ni[i+1:]].prod()
xy += Ncount[ni[-1]]
# Compute the number of repetitions in xy and assign it to the flattened histmat.
if len(xy) == 0:
return zeros(nbin-2, int), edges
flatcount = bincount(xy, weights)
a = arange(len(flatcount))
hist[a] = flatcount
# Shape into a proper matrix
hist = hist.reshape(sort(nbin))
for i in arange(nbin.size):
j = ni[i]
hist = hist.swapaxes(i,j)
ni[i],ni[j] = ni[j],ni[i]
# Remove outliers (indices 0 and -1 for each dimension).
core = D*[slice(1,-1)]
hist = hist[core]
# Normalize if normed is True
if normed:
s = hist.sum()
for i in arange(D):
shape = ones(D, int)
shape[i] = nbin[i]-2
hist = hist / dedges[i].reshape(shape)
hist /= s
return hist, edges
def apply_over_axes(func, a, axes):
"""
Apply a function repeatedly over multiple axes.
`func` is called as `res = func(a, axis)`, where `axis` is the first
element of `axes`. The result `res` of the function call must have
either the same dimensions as `a` or one less dimension. If `res`
has one less dimension than `a`, a dimension is inserted before
`axis`. The call to `func` is then repeated for each axis in `axes`,
with `res` as the first argument.
Parameters
----------
func : function
This function must take two arguments, `func(a, axis)`.
a : array_like
Input array.
axes : array_like
Axes over which `func` is applied; the elements must be integers.
Returns
-------
apply_over_axis : ndarray
The output array. The number of dimensions is the same as `a`,
but the shape can be different. This depends on whether `func`
changes the shape of its output with respect to its input.
See Also
--------
apply_along_axis :
Apply a function to 1-D slices of an array along the given axis.
Examples
--------
>>> a = np.arange(24).reshape(2,3,4)
>>> a
array([[[ 0, 1, 2, 3],
[ 4, 5, 6, 7],
[ 8, 9, 10, 11]],
[[12, 13, 14, 15],
[16, 17, 18, 19],
[20, 21, 22, 23]]])
Sum over axes 0 and 2. The result has same number of dimensions
as the original array:
>>> np.apply_over_axes(np.sum, a, [0,2])
array([[[ 60],
[ 92],
[124]]])
"""
val = asarray(a)
N = a.ndim
if array(axes).ndim == 0:
axes = (axes, )
for axis in axes:
if axis < 0: axis = N + axis
args = (val, axis)
res = func(*args)
if res.ndim == val.ndim:
val = res
else:
res = expand_dims(res, axis)
if res.ndim == val.ndim:
val = res
else:
raise ValueError("function is not returning "
"an array of the correct shape")
return val
def msort(a):
b = array(a,subok=True,copy=True)
b.sort(0)
return b
def kron(a, b):
"""
Kronecker product of two arrays.
Computes the Kronecker product, a composite array made of blocks of the
second array scaled by the first.
Parameters
----------
a, b : array_like
Returns
-------
out : ndarray
See Also
--------
outer : The outer product
Notes
-----
The function assumes that the number of dimenensions of `a` and `b`
are the same, if necessary prepending the smallest with ones.
If `a.shape = (r0,r1,..,rN)` and `b.shape = (s0,s1,...,sN)`,
the Kronecker product has shape `(r0*s0, r1*s1, ..., rN*SN)`.
The elements are products of elements from `a` and `b`, organized
explicitly by::
kron(a,b)[k0,k1,...,kN] = a[i0,i1,...,iN] * b[j0,j1,...,jN]
where::
kt = it * st + jt, t = 0,...,N
In the common 2-D case (N=1), the block structure can be visualized::
[[ a[0,0]*b, a[0,1]*b, ... , a[0,-1]*b ],
[ ... ... ],
[ a[-1,0]*b, a[-1,1]*b, ... , a[-1,-1]*b ]]
Examples
--------
>>> np.kron([1,10,100], [5,6,7])
array([ 5, 6, 7, 50, 60, 70, 500, 600, 700])
>>> np.kron([5,6,7], [1,10,100])
array([ 5, 50, 500, 6, 60, 600, 7, 70, 700])
>>> np.kron(np.eye(2), np.ones((2,2)))
array([[ 1., 1., 0., 0.],
[ 1., 1., 0., 0.],
[ 0., 0., 1., 1.],
[ 0., 0., 1., 1.]])
>>> a = np.arange(100).reshape((2,5,2,5))
>>> b = np.arange(24).reshape((2,3,4))
>>> c = np.kron(a,b)
>>> c.shape
(2, 10, 6, 20)
>>> I = (1,3,0,2)
>>> J = (0,2,1)
>>> J1 = (0,) + J # extend to ndim=4
>>> S1 = (1,) + b.shape
>>> K = tuple(np.array(I) * np.array(S1) + np.array(J1))
>>> c[K] == a[I]*b[J]
True
"""
b = asanyarray(b)
a = array(a, copy=False, subok=True, ndmin=b.ndim)
ndb, nda = b.ndim, a.ndim
if (nda == 0 or ndb == 0):
return _nx.multiply(a, b)
as_ = a.shape
bs = b.shape
if not a.flags.contiguous:
a = reshape(a, as_)
if not b.flags.contiguous:
b = reshape(b, bs)
nd = ndb
if (ndb != nda):
if (ndb > nda):
as_ = (1, ) * (ndb - nda) + as_
else:
bs = (1, ) * (nda - ndb) + bs
nd = nda
result = outer(a, b).reshape(as_ + bs)
axis = nd - 1
for _ in range(nd):
result = concatenate(result, axis=axis)
wrapper = get_array_prepare(a, b)
if wrapper is not None:
result = wrapper(result)
wrapper = get_array_wrap(a, b)
if wrapper is not None:
result = wrapper(result)
return result
def split(layers, indices_or_sections=None, axis=0):
if not isinstance(layers, list):
layers = [layers]
split_layers = []
for layer in layers:
try:
layer = layer.output
except:
pass
dims = K.ndim(layer)
Ntotal = K.int_shape(layer)[axis + 1]
try:
# handle scalar case.
Nsections = len(indices_or_sections) + 1
div_points = [0] + list(indices_or_sections) + [Ntotal]
except TypeError:
# indices_or_sections is a scalar, not an array.
Nsections = int(indices_or_sections)
if Nsections <= 0:
raise ValueError('number sections must be larger than 0.')
Neach_section, extras = divmod(Ntotal, Nsections)
section_sizes = ([0] + extras * [Neach_section + 1] +
(Nsections - extras) * [Neach_section])
div_points = _nx.array(section_sizes).cumsum()
for i in range(Nsections):
def split_func(array, st, end, sub_array_axis, sub_array_dims):
split_dims = np.arange(sub_array_dims)
split_dims[sub_array_axis + 1] = 1
split_dims[1] = sub_array_axis + 1
trans_array = tf.transpose(array, split_dims)
if sub_array_dims == 2:
split_array = trans_array[:, st:end]
elif sub_array_dims == 3:
split_array = trans_array[:, st:end, :]
else:
split_array = trans_array[:] # TODO
sub_array = tf.transpose(split_array, split_dims)
return sub_array
split_lambda = Lambda(split_func,
arguments={
'st': div_points[i],
'end': div_points[i + 1],
'sub_array_axis': axis,
'sub_array_dims': dims
})
split_lambda(layer)
split_layers.append(split_lambda)
return split_layers