def plot_sphere(radius=1, fig=None, ax=None):
import numpy as np
"""Plots a sphere given a radius
Args:
radius: The radius of the sphere.
fig: The figure onto which the axis is plotted. If None is passed a new
figure is made.
ax: The axis onto which the sphere is plotted. If None is passed a new
axis is made.
Returns:
None
"""
if fig is None:
fig = plt.figure()
if ax is None:
ax = fig.add_subplot(111, projection='3d')
# Source: http://matplotlib.org/examples/mplot3d/surface3d_demo2.html
phi = np.linspace(0, 2 * dnp.pi, 100)
theta = np.linspace(0, dnp.pi, 100)
x = radius * np.outer(np.cos(phi), np.sin(theta))
y = radius * np.outer(np.sin(phi), np.sin(theta))
z = radius * np.outer(np.ones(np.size(phi)), np.cos(theta))
ax.plot_wireframe(x, y, z, rstride=5, cstride=5, color='y')
def updateParameters(self, articlePicked, click, userID): self.counter +=1 self.Wlong = vectorize(self.W) featureDimension = len(articlePicked.featureVector) T_X = vectorize(np.outer(articlePicked.featureVector, self.W.T[userID])) self.A += np.outer(T_X, T_X) self.b += click*T_X self.AInv = np.linalg.inv(self.A) self.UserTheta = matrixize(np.dot(self.AInv, self.b), len(articlePicked.featureVector)) Xi_Matirx = np.zeros(shape = (featureDimension, self.userNum)) Xi_Matirx.T[userID] = articlePicked.featureVector W_X = vectorize( np.dot(np.transpose(self.UserTheta), Xi_Matirx)) self.batchGradient +=evaluateGradient(W_X, click, self.Wlong, self.lambda_, self.regu ) if self.counter%self.windowSize ==0: self.Wlong -= 1/(float(self.counter/self.windowSize)+1)*self.batchGradient self.W = matrixize(self.Wlong, self.userNum) self.W = normalize(self.W, axis=0, norm='l1') #print 'SVD', self.W self.batchGradient = np.zeros(self.userNum*self.userNum) # Use Ridge regression to fit W ''' plt.pcolor(self.W_b) plt.colorbar plt.show() ''' if self.W.T[userID].any() <0 or self.W.T[userID].any()>1: print self.W.T[userID] self.CoTheta = np.dot(self.UserTheta, self.W) self.BigW = np.kron(np.transpose(self.W), np.identity(n=len(articlePicked.featureVector))) self.CCA = np.dot(np.dot(self.BigW , self.AInv), np.transpose(self.BigW)) self.BigTheta = np.kron(np.identity(n=self.userNum) , self.UserTheta)
def setup(self):
surfaces = self.options['surfaces']
system_size = 0
for surface in self.options['surfaces']:
mesh = surface['mesh']
nx = mesh.shape[0]
ny = mesh.shape[1]
system_size += (nx - 1) * (ny - 1)
self.system_size = system_size
self.add_input('mtx', shape=(system_size, system_size), units='1/m')
self.add_input('rhs', shape=system_size, units='m/s')
self.add_output('circulations', shape=system_size, units='m**2/s')
self.declare_partials('circulations', 'circulations',
rows=np.outer(np.arange(system_size), np.ones(system_size, int)).flatten(),
cols=np.outer(np.ones(system_size, int), np.arange(system_size)).flatten(),
)
self.declare_partials('circulations', 'mtx',
rows=np.outer(np.arange(system_size), np.ones(system_size, int)).flatten(),
cols=np.arange(system_size ** 2),
)
self.declare_partials('circulations', 'rhs', val=-1.,
rows=np.arange(system_size),
cols=np.arange(system_size),
)
def get_response_content(fs):
# make the laplacian matrix for the graph
weight = 1 / fs.edge_length
n = fs.nvertices
L = np.zeros((n,n))
# set the diagonal
for i in range(n):
L[i,i] = 2 * weight
L[0,0] = weight
L[-1,-1] = weight
# complete the tridiagonal
for i in range(n-1):
L[i+1,i] = -weight
L[i,i+1] = -weight
# define other matrices
L_pinv = np.linalg.pinv(L)
HDH = -2*L_pinv
v = np.diag(HDH)
e = np.ones(n)
D = HDH - (np.outer(v, e) + np.outer(e, v))/2
# show some matrices
out = StringIO()
np.set_printoptions(linewidth=300)
print >> out, 'Laplacian matrix:'
print >> out, L
print >> out
print >> out, 'HDH:'
print >> out, HDH
print >> out
print >> out, 'EDM:'
print >> out, D
print >> out
return out.getvalue()
def plot_surface(image, center=None, size=15, output=False, ds9_indexing=False,
**kwargs):
"""
Create a surface plot from image.
By default, the whole image is plotted. The 'center' and 'size' attributes
allow to crop the image.
Parameters
----------
image : numpy.array
The image as a numpy.array.
center : tuple of 2 int (optional, default=None)
If None, the whole image will be plotted. Otherwise, it grabs a square
subimage at the 'center' (Y,X) from the image.
size : int (optional, default=15)
It corresponds to the size of a square in the image.
output : {False, True}, bool optional
Whether to output the grids and intensities or not.
ds9_indexing : {False, True}, bool optional
If True the coordinates are in X,Y convention and in 1-indexed format.
kwargs:
Additional attributes are passed to the matplotlib figure() and
plot_surface() method.
Returns
-------
out : tuple of 3 numpy.array
x and y for the grid, and the intensity
"""
if not center:
size = image.shape[0]
x = np.outer(np.arange(0,size,1), np.ones(size))
y = x.copy().T
z = image
else:
if ds9_indexing:
center = (center[0]-1,center[1]-1)
cx, cy = center
else:
cy, cx = center
if size % 2: # if size is odd
x = np.outer(np.arange(0,size,1), np.ones(size))
else: # otherwise, size is even
x = np.outer(np.arange(0,size+1,1), np.ones(size+1))
y = x.copy().T
z = image[cy-size//2:cy+size//2+1,cx-size//2:cx+size//2+1]
figure(figsize=kwargs.pop('figsize',(5,5)))
ax = axes(projection='3d')
ax.plot_surface(x, y, z, rstride=1, cstride=1, linewidth=0, **kwargs)
ax.set_xlabel('$x$')
ax.set_ylabel('$y$')
ax.set_zlabel('$I(x,y)$')
ax.set_title('Data')
show()
if output:
return (x,y,z)
def backProp(self,node,error=None):
# Clear nodes
node.fprop = False
# y_hat - y
delta5 = node.probs
delta5[node.label] -= 1.0
self.dbs += delta5
# dU
self.dWs += np.outer(delta5, node.hActs2)
delta4 = np.dot(self.Ws.T, delta5)
delta3 = delta4 * (node.hActs2 > 0)
self.db2 += delta3
self.dW2 += np.outer(delta3, node.hActs1)
delta2 = np.dot(self.W2.T, delta3)
if error is not None:
delta2 += error
if node.isLeaf:
self.dL[node.word] += delta2
else:
delta1 = delta2 * (node.hActs1 > 0)
self.db1 += delta1
self.dW1 += np.outer(delta1, np.hstack([node.left.hActs1, node.right.hActs1]))
delta0 = np.dot(self.W1.T, delta1)
self.backProp(node.left, delta0[:self.wvecDim])
self.backProp(node.right, delta0[self.wvecDim:])
def updateA(self, featureVector, decay=None, current_time=None): if decay: assert decay <= 1 and decay >=0 self.DD = self.decayAverage(decay, self.DD, np.outer(featureVector, featureVector), current_time) self.A = self.DD + self.identityMatrix else: self.A += np.outer(featureVector, featureVector)
def backpropagation(self, input, output):
'''
Compute gradientes, with the back-propagation method
inputs:
x: vector with the (embedding) indicies of the words of a
sentence
outputs: vector with the indicies of the tags for each word of
the sentence outputs:
gradient_parameters: vector with parameters gradientes
'''
# Get parameters and sizes
W_e, W_x, W_h, W_y = self.parameters
nr_steps = input.shape[0]
log_p_y, y, h, z_e, x = self.log_forward(input)
p_y = np.exp(log_p_y)
# Initialize gradients with zero entrances
gradient_W_e = np.zeros(W_e.shape)
gradient_W_x = np.zeros(W_x.shape)
gradient_W_h = np.zeros(W_h.shape)
gradient_W_y = np.zeros(W_y.shape)
# ----------
# Solution to Exercise 6.1
# Gradient of the cost with respect to the last linear model
I = index2onehot(output, W_y.shape[0])
error = (p_y - I) / nr_steps
# backward pass, with gradient computation
error_h_next = np.zeros_like(h[0, :])
for t in reversed(range(nr_steps)):
# Output linear
error_h = np.dot(W_y.T, error[t, :]) + error_h_next
# Non-linear
error_raw = h[t+1, :] * (1. - h[t+1, :]) * error_h
# Hidden-linear
error_h_next = np.dot(W_h.T, error_raw)
# Weight gradients
gradient_W_y += np.outer(error[t, :], h[t+1, :])
gradient_W_h += np.outer(error_raw, h[t, :])
gradient_W_x += np.outer(error_raw, z_e[t, :])
gradient_W_e[x[t], :] += W_x.T.dot(error_raw)
# End of Solution to Exercise 6.1
# ----------
# Normalize over sentence length
gradient_parameters = [
gradient_W_e, gradient_W_x, gradient_W_h, gradient_W_y
]
return gradient_parameters
def Cramer(var1, var2): """ Compute Cramer's V statistic for two Pandas series Parameters: ---------- var1, var2: Pandas series Returns: -------- v : float The Cramer's V statistic of two categorical-variable series Status: ------- Cramer's V Implementation Author: Jesse Lund, [email protected] Date: 9/12/2015 ##Round 1## Comments: Thomas Roderick, [email protected] Date: 9/13/2015 """ table = crosstab(var1,var2) #For Pandas: must have an index, can't just feed in two lists. This could be a sticking point. Might be better to do a check or roll our own crosstab implementation l,w = table.shape #save on a (small) function call here--reads in both outputs df = min(l-1, w-1) colsum, rowsum = table.sum(0), table.sum(1) n = float(l*w) expectmat = outer(rowsum,colsum)/n outmat = outer(table.sum(0),table.sum(1))/n #this works if same size return sqrt((((table - expectmat)**2)/(expectmat*n*df)).sum().sum())
def unscented_func (f, sigmaPoints, meanWeight, covWeight, angleMask=None, **kwargs):
n = sigmaPoints.shape[1]
y = f (sigmaPoints[:,0], **kwargs)
m = len (y)
y = zeros ((m, n))
for i in range (n):
y[:,i] = f (sigmaPoints[:,i], **kwargs)
muPrime = sum (y * meanWeight, axis=1)
if angleMask is not None:
for i, mask in enumerate (angleMask):
if mask:
muPrime[i] = circularMean (y[i,:], weights=meanWeight)
muPrime[i] = minimizedAngle (muPrime[i])
SigmaPrime = zeros ((m, m))
for i in range (n):
if angleMask is None:
SigmaPrime += covWeight[i] * outer (y[:,i] - muPrime, (y[:,i] - muPrime))
else:
SigmaPrime += covWeight[i] * outer (minimizedAngle (y[:,i] - muPrime, angleMask),
minimizedAngle (y[:,i] - muPrime, angleMask))
return muPrime, SigmaPrime
def unscented_obs_model (f, sigmaPoints, meanWeight, covWeight, state, stateAngleMask, obsModelAngleMask, **kwargs):
n = sigmaPoints.shape[1]
stateLen = len (state)
y = f (sigmaPoints[0:stateLen,0], **kwargs) + sigmaPoints[stateLen:,0]
m = len (y)
y = zeros ((m, n))
for i in range (n):
y[:,i] = f (sigmaPoints[0:stateLen,i], **kwargs) + sigmaPoints[stateLen:,i]
muPrime = sum (y * meanWeight, axis=1)
for i, mask in enumerate (obsModelAngleMask):
if mask:
muPrime[i] = circularMean (y[i,:], weights=meanWeight)
muPrime[i] = minimizedAngle (muPrime[i])
SigmaPrime = zeros ((m, m))
for i in range (n):
SigmaPrime += covWeight[i] * outer (minimizedAngle (y[:,i] - muPrime, obsModelAngleMask),
minimizedAngle (y[:,i] - muPrime, obsModelAngleMask))
crossCov = zeros ((stateLen, m))
for i in range (n):
diffState = minimizedAngle (sigmaPoints[0:stateLen,i] - state, stateAngleMask)
diffMeas = minimizedAngle (y[:,i] - muPrime, obsModelAngleMask)
crossCov += covWeight[i] * outer (diffState, diffMeas)
return muPrime, SigmaPrime, crossCov
def backProp(self,node,error=None):
# Clear nodes
node.fprop = False
errorCur = node.probs - make_onehot(node.label,len(self.bs))
self.dWs += np.outer(errorCur, node.hActs1)
self.dbs += errorCur
errorCur = errorCur.dot(self.Ws)
if error is not None:
errorCur += error
if node.isLeaf == True:
self.dL[node.word] += errorCur
return
errorCur = errorCur*self.df(node.hActs1)
LR = np.hstack([node.left.hActs1, node.right.hActs1])
self.dW += np.outer(errorCur,LR)
self.db += errorCur
S = np.zeros(len(LR))
for i in range(len(self.V)):
self.dV[i] += errorCur[i]*np.outer(LR,LR)
S += (self.V[i]+self.V[i].T).dot(LR)*errorCur[i]
errorDown = errorCur.dot(self.W) + S
self.backProp(node.left,errorDown[:self.wvecDim])
self.backProp(node.right,errorDown[self.wvecDim:])
def _set_expected_stats(self,smoothed_mus,smoothed_sigmas,E_xtp1_xtT):
assert not np.isnan(E_xtp1_xtT).any()
assert not np.isnan(smoothed_mus).any()
assert not np.isnan(smoothed_sigmas).any()
data = self.data
EyyT = data.T.dot(data)
EyxT = data.T.dot(smoothed_mus)
ExxT = smoothed_sigmas.sum(0) + smoothed_mus.T.dot(smoothed_mus)
E_xt_xtT = \
ExxT - (smoothed_sigmas[-1]
+ np.outer(smoothed_mus[-1],smoothed_mus[-1]))
E_xtp1_xtp1T = \
ExxT - (smoothed_sigmas[0]
+ np.outer(smoothed_mus[0], smoothed_mus[0]))
E_xtp1_xtT = E_xtp1_xtT.sum(0)
def is_symmetric(A):
return np.allclose(A,A.T)
assert is_symmetric(ExxT)
assert is_symmetric(E_xt_xtT)
assert is_symmetric(E_xtp1_xtp1T)
self.E_emission_stats = np.array([EyyT, EyxT, ExxT, self.T])
self.E_dynamics_stats = np.array([E_xtp1_xtp1T, E_xtp1_xtT, E_xt_xtT, self.T-1])
def learn(self, docs, alpha=0.1, tau=5):
index = numpy.arange(len(docs))
numpy.random.shuffle(index)
for i in index:
doc = docs[i]
pre_s = [numpy.zeros(self.K)]
pre_w = [0] # <s>
for w in doc:
s = 1 / (numpy.exp(- numpy.dot(self.W, pre_s[-1]) - self.U[:, pre_w[-1]]) + 1)
z = numpy.dot(self.V, s)
y = numpy.exp(z - z.max())
y = y / y.sum()
# calculate errors
y[w] -= 1 # -e0
eh = [numpy.dot(y, self.V) * s * (s - 1)] # eh[t]
for t in xrange(min(tau, len(pre_s)-1)):
st = pre_s[-1-t]
eh.append(numpy.dot(eh[-1], self.W) * st * (1 - st))
# update parameters
pre_w.append(w)
pre_s.append(s)
self.V -= numpy.outer(y, s * alpha)
for t in xrange(len(eh)):
self.U[:, pre_w[-1-t]] += eh[t] * alpha
self.W += numpy.outer(pre_s[-2-t], eh[t]) * alpha
def generate_ODGD_spec_chirped(F1, F2, Fs, lengthOdgd=2048, Nfft=2048, \
Ot=0.5, t0=0.0, \
analysisWindowType='sinebell'):
"""
generateODGDspecChirped:
generates a waveform ODGD and the corresponding spectrum,
using as analysis window the -optional- window given as
argument.
"""
# converting input to double:
F1 = np.double(F1)
F2 = np.double(F2)
F0 = np.double(F1 + F2) / 2.0
Fs = np.double(Fs)
Ot = np.double(Ot)
t0 = np.double(t0)
# compute analysis window of given type:
if analysisWindowType == 'sinebell':
analysisWindow = sinebell(lengthOdgd)
else:
if analysisWindowType == 'hanning' or \
analysisWindowType == 'hann':
analysisWindow = hann(lengthOdgd)
# maximum number of partials in the spectral comb:
partialMax = np.floor((Fs / 2) / np.max(F1, F2))
# Frequency numbers of the partials:
frequency_numbers = np.arange(1,partialMax + 1)
# intermediate value
temp_array = 1j * 2.0 * np.pi * frequency_numbers * Ot
# compute the amplitudes for each of the frequency peaks:
amplitudes = F0 * 27 / 4 * \
(np.exp(-temp_array) \
+ (2 * (1 + 2 * np.exp(-temp_array)) / temp_array) \
- (6 * (1 - np.exp(-temp_array)) \
/ (temp_array ** 2))) \
/ temp_array
# Time stamps for the time domain ODGD
timeStamps = np.arange(lengthOdgd) / Fs + t0 / F0
# Time domain odgd:
odgd = np.exp(2.0 * 1j * np.pi \
* (np.outer(F1 * frequency_numbers,timeStamps) \
+ np.outer((F2 - F1) \
* frequency_numbers,timeStamps ** 2) \
/ (2 * lengthOdgd / Fs))) \
* np.outer(amplitudes,np.ones(lengthOdgd))
odgd = np.sum(odgd,axis=0)
# spectrum:
odgdSpectrum = np.fft.fft(real(odgd * analysisWindow), n=Nfft)
return odgd, odgdSpectrum
def pma(data,axis=0):
"""
principal modes analysis of 2D data - expects time on the axis=0 axis
returns:
sorted eigvals, and eigvects, correlation matrix
"""
i=data[0]
#MD=np.einsum('i,j->ij',i,i.T)
MD=np.outer(i,i)
#print "entering for loop"
for i in data[1:]:
#MD+=np.einsum('i,j->ij',i,i.T)
MD+=np.outer(i,i)
#print "exiting for loop"
MD/=data.shape[0]
#print "computing eigenvalues"
l,e=np.linalg.eigh(MD)
#print "done"
idxx=np.argsort(l)[::-1]
l=l[idxx]
e=e[:,idxx]
return l,e,MD
def evolveDraw(self, r):
""" This can contain triggers for things to be drawn, e.g. the shark."""
if self.mode == 3 or self.mode == 4 or self.mode == 7:
# Draw shark
self.updateShark(r)
self.axis.scatter(
self.predatorLocation[:, 0],
self.predatorLocation[:, 1],
self.predatorLocation[:, 2],
color="r",
s=4 * self.length,
)
if self.mode == 1 or self.mode == 2 or self.mode == 3 or self.mode == 6:
if self.noSphere == True:
u = np.linspace(0, 2 * np.pi, 100)
v = np.linspace(0, np.pi, 100)
self.sphere_x = self.habitatSize * np.outer(np.cos(u), np.sin(v))
self.sphere_y = self.habitatSize * np.outer(np.sin(u), np.sin(v))
self.sphere_z = self.habitatSize * np.outer(np.ones(np.size(u)), np.cos(v))
self.noSphere = False
self.axis.plot_wireframe(
self.sphere_x, self.sphere_y, self.sphere_z, rstride=13, cstride=13, color="r", alpha=0.3
)
def update(self, population):
"""Update the current covariance matrix strategy from the
*population*.
:param population: A list of individuals from which to update the
parameters.
"""
population.sort(key=lambda ind: ind.fitness, reverse=True)
lambda_succ = sum(self.parent.fitness <= ind.fitness for ind in population)
p_succ = float(lambda_succ) / self.lambda_
self.psucc = (1 - self.cp) * self.psucc + self.cp * p_succ
if self.parent.fitness <= population[0].fitness:
x_step = (population[0] - numpy.array(self.parent)) / self.sigma
self.parent = copy.deepcopy(population[0])
if self.psucc < self.pthresh:
self.pc = (1 - self.cc) * self.pc + sqrt(self.cc * (2 - self.cc)) * x_step
self.C = (1 - self.ccov) * self.C + self.ccov * numpy.outer(self.pc, self.pc)
else:
self.pc = (1 - self.cc) * self.pc
self.C = (1 - self.ccov) * self.C + self.ccov * (numpy.outer(self.pc, self.pc) + self.cc * (2 - self.cc) * self.C)
self.sigma = self.sigma * exp(1.0 / self.d * (self.psucc - self.ptarg) / (1.0 - self.ptarg))
# We use Cholesky since for now we have no use of eigen decomposition
# Basically, Cholesky returns a matrix A as C = A*A.T
# Eigen decomposition returns two matrix B and D^2 as C = B*D^2*B.T = B*D*D*B.T
# So A == B*D
# To compute the new individual we need to multiply each vector z by A
# as y = centroid + sigma * A*z
# So the Cholesky is more straightforward as we don't need to compute
# the squareroot of D^2, and multiply B and D in order to get A, we directly get A.
# This can't be done (without cost) with the standard CMA-ES as the eigen decomposition is used
# to compute covariance matrix inverse in the step-size evolutionary path computation.
self.A = numpy.linalg.cholesky(self.C)
def fft_shift_phasor_2d(shape, offset, grad=False):
"""Return phasor array used to shift an array (in real space) by
multiplication in fourier space.
Parameters
----------
shape : (int, int)
Length 2 iterable giving shape of array.
offset : (float, float)
Offset in array elements in each dimension.
Returns
-------
z : np.ndarray (complex; 2-d)
Complex array with shape ``shape``.
"""
ny, nx = shape
dy, dx = offset
yphasor = fft_shift_phasor(ny, dy, grad=grad)
xphasor = fft_shift_phasor(nx, dx, grad=grad)
if grad:
res = np.outer(yphasor[0], xphasor[0])
dres_dy = np.outer(yphasor[1], xphasor[0])
dres_dx = np.outer(yphasor[0], xphasor[1])
resgrad = np.concatenate((dres_dy[None, :, :], dres_dx[None, :, :]))
return res, resgrad
else:
return np.outer(yphasor, xphasor)
def bptt(self, x, y):
# forward pass
# save all hidden states and outpts at each time step
# because need them during back propagation.
# add one additional 0s as the initial hidden state
s = np.zeros((len(x) + 1, self.hidden_dim))
s[-1] = np.zeros(self.hidden_dim)
o = np.zeros((len(x), self.word_dim))
for t in range(len(x)):
# Note that we are indxing U by x[t]. This is the same as
# multiplying U with a one-hot vector.
s[t] = np.tanh(self.U[:,x[t]] + self.W.dot(s[t-1]))
o[t] = softmax(self.V.dot(s[t]))
# backward pass
dLdU = np.zeros(self.U.shape)
dLdV = np.zeros(self.V.shape)
dLdW = np.zeros(self.W.shape)
# dLdy = o - y
dLdy = o
dLdy[np.arange(len(y)), y] -= 1.
# For each output backwards...
for t in np.arange(len(y))[::-1]:
dLdV += np.outer(dLdy[t], s[t].T)
# Initial delta calculation
dLdz = self.V.T.dot(dLdy[t]) * (1 - (s[t] ** 2))
# Backpropagation through time (for at most self.bptt_truncate steps)
for bptt_step in np.arange(max(0, t-self.bptt_truncate), t+1)[::-1]:
# print "Backpropagation step t=%d bptt step=%d " % (t, bptt_step)
dLdW += np.outer(dLdz, s[bptt_step-1])
dLdU[:,x[bptt_step]] += dLdz
# Update delta for next step
dLdz = self.W.T.dot(dLdz) * (1 - s[bptt_step-1] ** 2)
return [dLdU, dLdV, dLdW]
def __call__(self, mat, type="wireframe", **params):
p = ParamOverrides(self, params)
from mpl_toolkits.mplot3d import axes3d
fig = plt.figure()
ax = axes3d.Axes3D(fig)
# Construct matrices for r and c values
rn, cn = mat.shape
c = np.outer(np.ones(rn), np.arange(cn * 1.0))
r = np.outer(np.arange(rn * 1.0), np.ones(cn))
if type == "wireframe":
ax.plot_wireframe(r, c, mat)
elif type == "surface":
# Sometimes fails for no obvious reason
ax.plot_surface(r, c, mat)
elif type == "contour":
# Works but not usually very useful
ax.contour3D(r, c, mat)
else:
raise ValueError("Unknown plot type " + str(type))
ax.set_xlabel('R')
ax.set_ylabel('C')
ax.set_zlabel('Value')
self._generate_figure(p)
def _get_H(self, debug=False):
"""
returns H_t as defined in algorithm 2
Reference:
https://en.wikipedia.org/wiki/Limited-memory_BFGS
http://www.ccms.or.kr/data/pdfpaper/jcms21_1/21_1_117.pdf
https://homes.cs.washington.edu/~galen/files/quasi-newton-notes.pdf
"""
I = np.identity(len(self.w))
if min(len(self.s), len(self.y)) == 0:
print "Warning: No second order information used!"
return I
assert len(self.s) > 0, "s cannot be empty."
assert len(self.s) == len(self.y), "s and y must have same length"
assert self.s[0].shape == self.y[0].shape, \
"s and y must have same shape"
assert abs(self.y[-1]).sum() != 0, "latest y entry cannot be 0!"
assert 1/np.inner(self.y[-1], self.s[-1]) != 0, "!"
I = np.identity(len(self.s[0]))
H = np.dot((np.inner(self.s[-1], self.y[-1]) / np.inner(self.y[-1],
self.y[-1])), I)
for (s_j, y_j) in itertools.izip(self.s, self.y):
rho = 1.0/np.inner(y_j, s_j)
V = I - np.multiply(rho, np.outer(s_j, y_j))
H = (V).dot(H).dot(V.T)
H += np.multiply(rho, np.outer(s_j, s_j))
return H
def calculate(self, w, ionization_data, beta_rad, t_electrons, t_rad,
beta_electron, delta_input, chi_0):
if delta_input is None:
if self.departure_coefficient is None:
departure_coefficient = 1. / w
else:
departure_coefficient = self.departure_coefficient
radiation_field_correction = -np.ones((len(ionization_data), len(
beta_rad)))
less_than_chi_0 = (
ionization_data.ionization_energy < chi_0).values
factor_a = (t_electrons / (departure_coefficient * w * t_rad))
radiation_field_correction[~less_than_chi_0] = factor_a * \
np.exp(np.outer(ionization_data.ionization_energy.values[
~less_than_chi_0], beta_rad - beta_electron))
radiation_field_correction[less_than_chi_0] = 1 - np.exp(np.outer(
ionization_data.ionization_energy.values[less_than_chi_0],
beta_rad) - beta_rad * chi_0)
radiation_field_correction[less_than_chi_0] += factor_a * np.exp(
np.outer(ionization_data.ionization_energy.values[
less_than_chi_0],beta_rad) - chi_0 * beta_electron)
else:
radiation_field_correction = np.ones((len(ionization_data),
len(beta_rad))) * delta_input
delta = pd.DataFrame(radiation_field_correction,
columns=np.arange(len(t_rad)), index=ionization_data.index)
return delta
def trainNN(self, imagesTrainSet, labelsTrainSet, etha):
self.reset_weights()
trainingSetSize = labelsTrainSet.shape[0];
j = 0
while j < 30:
i = 0
# print("Round: " + str(j + 1))
while i < trainingSetSize :
x = imagesTrainSet[i].ravel() # Convert 28x28 pixel image into a (784,) vector
x = np.array([ 0 if val == 0 else 1 for val in x ])
x_a = np.insert(x, 0, values=1, axis=0) # Augmented Feature vector
net_hidd = np.dot(self.w1, x_a)
y = self.signum(net_hidd)
y_a = np.insert(y, 0, values=1, axis=0) # Augmented Feature vector
net_out = np.dot(self.w2, y_a)
z = self.signum(net_out)
lab = np.array([ 1 if k == self.labels[i] else 0 for k in range(10) ])
J = z - lab;
J = np.sum(0.5 * J * J);
if J < 1 and self.enableWeightDecay:
break;
out_sensitivity = (lab - z) * self.signum_prime(net_out)
net_hidd_prime = self.signum_prime(net_hidd)
hid_sensitivity = np.dot(self.w2.T, out_sensitivity) * np.insert(net_hidd_prime, 0, 1)
grad_hidd_out = etha * np.outer(out_sensitivity, y_a.T)
grad_in_hidd = etha * np.outer(hid_sensitivity[1:] , x_a.T)
self.update_weights_bias(grad_in_hidd, grad_hidd_out)
i += 1
j += 1
return self.w1, self.w2
def _create_displacement_matrix(self,
disp_pairs,
site_symmetry,
rot_atom_map):
rot_disp1s = []
rot_disp2s = []
rot_pair12 = []
rot_pair21 = []
rot_pair11 = []
rot_pair22 = []
for disp_pairs_u1 in disp_pairs:
for rot_atom_num, ssym in zip(rot_atom_map, site_symmetry):
ssym_c = similarity_transformation(self._lattice, ssym)
for (u1, u2) in disp_pairs_u1[rot_atom_num]:
Su1 = np.dot(ssym_c, u1)
Su2 = np.dot(ssym_c, u2)
rot_disp1s.append(Su1)
rot_disp2s.append(Su2)
rot_pair12.append(np.outer(Su1, Su2).flatten() / 2)
rot_pair21.append(np.outer(Su2, Su1).flatten() / 2)
rot_pair11.append(np.outer(Su1, Su1).flatten() / 2)
rot_pair22.append(np.outer(Su2, Su2).flatten() / 2)
ones = np.ones(len(rot_disp1s)).reshape((-1, 1))
return np.hstack((ones, rot_disp1s, rot_disp2s,
rot_pair12, rot_pair21, rot_pair11, rot_pair22))
def TwoSampleTest(self,sample1,sample2,numShuffles=1000,method='vanilla',blockSize=20):
"""
Compute the p-value associated to the MMD between two samples
method determines the null approximation procedure:
----'vanilla': standard permutation test
----'block': block permutation test
----'wild': wild bootstrap
----'wild-center': wild bootstrap with empirical degeneration
"""
n1=shape(sample1)[0]
n2=shape(sample2)[0]
merged = concatenate( [sample1, sample2], axis=0 )
merged_len=shape(merged)[0]
numBlocks = merged_len/blockSize
K=self.kernel(merged)
mmd = mean(K[:n1,:n1])+mean(K[n1:,n1:])-2*mean(K[n1:,:n1])
null_samples = zeros(numShuffles)
if method=='vanilla':
for i in range(numShuffles):
pp = permutation(merged_len)
Kpp = K[pp,:][:,pp]
null_samples[i] = mean(Kpp[:n1,:n1])+mean(Kpp[n1:,n1:])-2*mean(Kpp[n1:,:n1])
elif method=='block':
blocks=reshape(arange(merged_len),(numBlocks,blockSize))
for i in range(numShuffles):
pb = permutation(numBlocks)
pp = reshape(blocks[pb],(merged_len))
Kpp = K[pp,:][:,pp]
null_samples[i] = mean(Kpp[:n1,:n1])+mean(Kpp[n1:,n1:])-2*mean(Kpp[n1:,:n1])
elif method=='wild' or method=='wild-center':
if n1!=n2:
raise ValueError("Wild bootstrap MMD available only on the same sample sizes")
alpha = exp(-1/float(blockSize))
coreK = K[:n1,:n1]+K[n1:,n1:]-K[n1:,:n1]-K[:n1,n1:]
for i in range(numShuffles):
"""
w is a draw from the Ornstein-Uhlenbeck process
"""
w = HelperFunctions.generateOU(n=n1,alpha=alpha)
if method=='wild-center':
"""
empirical degeneration (V_{n,2} in Leucht & Neumann)
"""
w = w - mean(w)
null_samples[i]=mean(outer(w,w)*coreK)
elif method=='wild2':
alpha = exp(-1/float(blockSize))
for i in range(numShuffles):
wx=HelperFunctions.generateOU(n=n1,alpha=alpha)
wx = wx - mean(wx)
wy=HelperFunctions.generateOU(n=n2,alpha=alpha)
wy = wy - mean(wy)
null_samples[i]=mean(outer(wx,wx)*K[:n1,:n1])+mean(outer(wy,wy)*K[n1:,n1:])-2*mean(outer(wx,wy)*K[:n1,n1:])
else:
raise ValueError("Unknown null approximation method")
return sum(mmd<null_samples)/float(numShuffles)
def test_extract_signals_pca(self):
# Prepare some testing data
x = np.linspace(0, 2*np.pi, num=int(2*np.pi*50))
signal0 = np.sin(2*x)
signal1 = np.sin(3*x)
in_weights = np.array([[0, 0.25, 0.5, 0.75, 1],
[1, 0.75, 0.5, 0.25, 0]])
features = np.outer(in_weights[0], signal0)
features += np.outer(in_weights[1], signal1)
self.assertEqual(features.shape, (5, 314))
# Extract the signals
comps, weights = extract_signals_pca(spectra=features,
n_components=2)
weights = np.swapaxes(weights, 0, 1)
# Check the results
new_features = np.outer(weights[0], comps[0])
new_features += np.outer(weights[1], comps[1])
self.assertEqual(comps.shape, (2, len(x)))
self.assertEqual(weights.shape, in_weights.shape)
plt.plot(x, comps[0], label='c0')
plt.plot(x, comps[1], label='c1')
plt.plot(x, features[0], label='f0')
plt.plot(x, features[1], label='f1')
plt.plot(x, new_features[0], label='nf0')
plt.plot(x, new_features[1], label='nf1')
plt.legend()
plt.show()
def test_strucdamping(use_GPU):
n_inputs = 3
sig_len = 5
inputs = np.outer(np.linspace(0.1, 0.9, n_inputs),
np.ones(sig_len))[:, :, None]
targets = np.outer(np.linspace(0.1, 0.9, n_inputs),
np.linspace(0, 1, sig_len))[:, :, None]
inputs = inputs.astype(np.float32)
targets = targets.astype(np.float32)
optimizer = HessianFree(CG_iter=100)
rnn = hf.RNNet(
shape=[1, 5, 1],
loss_type=[hf.loss_funcs.SquaredError(),
hf.loss_funcs.StructuralDamping(0.1, optimizer=optimizer)],
debug=True, use_GPU=use_GPU)
rnn.run_epochs(inputs, targets, optimizer=optimizer,
max_epochs=30, print_period=None)
outputs = rnn.forward(inputs, rnn.W)
assert rnn.loss.batch_loss(outputs, targets) < 1e-4
def hess(A):
"""Computes the upper Hessenberg form of A using Householder reflectors.
input: A, mxn array
output: Q, orthogonal mxm array
H, upper Hessenberg
s.t. Q.dot(H).dot(Q.T) = A
"""
# similar approach as the householder function.
# again, not perfectly optimized, but good enough.
Q = np.eye(A.shape[0]).T
H = np.array(A, order="C")
# initialize m and n for convenience
m, n = H.shape
# avoid reallocating v in the for loop
v = np.empty(A.shape[1]-1)
for k in xrange(n-2):
# get a slice of the temporary array
vk = v[k:]
# fill it with corresponding values from R
vk[:] = H[k+1:,k]
# add in the term that makes the reflection work
vk[0] += copysign(la.norm(vk), vk[0])
# normalize it so it's an orthogonal transform
vk /= la.norm(vk)
# apply projection to H on the left
H[k+1:,k:] -= 2 * np.outer(vk, vk.dot(H[k+1:,k:]))
# apply projection to H on the right
H[:,k+1:] -= 2 * np.outer(H[:,k+1:].dot(vk), vk)
# Apply it to Q
Q[k+1:] -= 2 * np.outer(vk, vk.dot(Q[k+1:]))
return Q, H
def Dgrid_zone(zone,Px):
""" Distance point to zone
A zone is a quadrilateral zone.
Parameters
----------
zone : dictionnary
xmin xmax Nx
ymin ymax Ny
Px : np.array
point
Build the distance matrix between Tx and points in the zone
Notes
-----
use broadcasting instead
"""
rx = np.linspace(zone['xmin'],zone['xmax'],zone['Nx'])
ry = np.linspace(zone['ymin'],zone['ymax'],zone['Ny'])
R_x = np.outer(np.ones(len(ry)),rx)
R_y = np.outer(ry,np.ones(len(rx)))
Dx = R_x - Px[0]
Dy = R_y - Px[1]
D = np.sqrt(Dx*Dx+Dy*Dy)
return (D)
def calculateMapSq(self, R=None, m2_uform=None):
r"""Calculate the aperture mass statistics from the correlation function.
.. math::
\langle M_{ap}^2 \rangle(R) &= \int_{0}^{rmax} \frac{r dr}{2R^2}
\left [ T_+\left(\frac{r}{R}\right) \xi_+(r) +
T_-\left(\frac{r}{R}\right) \xi_-(r) \right] \\
\langle M_\times^2 \rangle(R) &= \int_{0}^{rmax} \frac{r dr}{2R^2}
\left[ T_+\left(\frac{r}{R}\right) \xi_+(r) -
T_-\left(\frac{r}{R}\right) \xi_-(r) \right]
The ``m2_uform`` parameter sets which definition of the aperture mass to use.
The default is to use 'Crittenden'.
If ``m2_uform`` is 'Crittenden':
.. math::
U(r) &= \frac{1}{2\pi} (1-r^2) \exp(-r^2/2) \\
Q(r) &= \frac{1}{4\pi} r^2 \exp(-r^2/2) \\
T_+(s) &= \frac{s^4 - 16s^2 + 32}{128} \exp(-s^2/4) \\
T_-(s) &= \frac{s^4}{128} \exp(-s^2/4) \\
rmax &= \infty
cf. Crittenden, et al (2002): ApJ, 568, 20
If ``m2_uform`` is 'Schneider':
.. math::
U(r) &= \frac{9}{\pi} (1-r^2) (1/3-r^2) \\
Q(r) &= \frac{6}{\pi} r^2 (1-r^2) \\
T_+(s) &= \frac{12}{5\pi} (2-15s^2) \arccos(s/2) \\
&\qquad + \frac{1}{100\pi} s \sqrt{4-s^2} (120 + 2320s^2 - 754s^4 + 132s^6 - 9s^8) \\
T_-(s) &= \frac{3}{70\pi} s^3 (4-s^2)^{7/2} \\
rmax &= 2R
cf. Schneider, et al (2002): A&A, 389, 729
.. note::
This function is only implemented for Log binning.
Parameters:
R (array): The R values at which to calculate the aperture mass statistics.
(default: None, which means use self.rnom)
m2_uform (str): Which form to use for the aperture mass, as described above.
(default: 'Crittenden'; this value can also be given in the
constructor in the config dict.)
Returns:
Tuple containing
- mapsq = array of :math:`\langle M_{ap}^2 \rangle(R)`
- mapsq_im = the imaginary part of mapsq, which is an estimate of
:math:`\langle M_{ap} M_\times \rangle(R)`
- mxsq = array of :math:`\langle M_\times^2 \rangle(R)`
- mxsq_im = the imaginary part of mxsq, which is an estimate of
:math:`\langle M_{ap} M_\times \rangle(R)`
- varmapsq = array of the variance estimate of either mapsq or mxsq
"""
if m2_uform is None:
m2_uform = treecorr.config.get(self.config,'m2_uform',str,'Crittenden')
if m2_uform not in ['Crittenden', 'Schneider']:
raise ValueError("Invalid m2_uform")
if self.bin_type != 'Log':
raise ValueError("calculateMapSq requires Log binning.")
if R is None:
R = self.rnom
# Make s a matrix, so we can eventually do the integral by doing a matrix product.
s = np.outer(1./R, self.meanr)
ssq = s*s
if m2_uform == 'Crittenden':
exp_factor = np.exp(-ssq/4.)
Tp = (32. + ssq*(-16. + ssq)) / 128. * exp_factor
Tm = ssq * ssq / 128. * exp_factor
else:
Tp = np.zeros_like(s)
Tm = np.zeros_like(s)
sa = s[s<2.]
ssqa = ssq[s<2.]
Tp[s<2.] = 12./(5.*np.pi) * (2.-15.*ssqa) * np.arccos(sa/2.)
Tp[s<2.] += 1./(100.*np.pi) * sa * np.sqrt(4.-ssqa) * (
120. + ssqa*(2320. + ssqa*(-754. + ssqa*(132. - 9.*ssqa))))
Tm[s<2.] = 3./(70.*np.pi) * sa * ssqa * (4.-ssqa)**3.5
Tp *= ssq
Tm *= ssq
# Now do the integral by taking the matrix products.
# Note that dlogr = bin_size
Tpxip = Tp.dot(self.xip)
Tmxim = Tm.dot(self.xim)
mapsq = (Tpxip + Tmxim) * 0.5 * self.bin_size
mxsq = (Tpxip - Tmxim) * 0.5 * self.bin_size
Tpxip_im = Tp.dot(self.xip_im)
Tmxim_im = Tm.dot(self.xim_im)
mapsq_im = (Tpxip_im + Tmxim_im) * 0.5 * self.bin_size
mxsq_im = (Tpxip_im - Tmxim_im) * 0.5 * self.bin_size
# The variance of each of these is
# Var(<Map^2>(R)) = int_r=0..2R [1/4 s^4 dlogr^2 (T+(s)^2 + T-(s)^2) Var(xi)]
varmapsq = (Tp**2).dot(self.varxip) + (Tm**2).dot(self.varxim)
varmapsq *= 0.25 * self.bin_size**2
return mapsq, mapsq_im, mxsq, mxsq_im, varmapsq
z[i] = np.dot(weights[i - 1], nodes[i - 1]) + bias[i - 1]
nodes[i] = activation_func(z[i])
delta_z[L] = cost(nodes[L], answer, True) * activation_func(z[L], True)
'''
delta_bias[L - 1] = np.array(delta_z[L])
delta_weights[L - 1] = np.outer(delta_z[L], nodes[L - 1])
delta_nodes[L - 1] = np.dot(delta_weights[L - 1], delta_z[L]) / LAYER_SIZE[L]
delta_z[L - 1] = delta_nodes[L - 1] * activation_func(z[L - 1], True)
delta_bias[L - 2] = np.array(delta_z[L - 1])
delta_weights[L - 2] = np.outer(delta_z[L - 1], nodes[L - 2])
'''
for i in range(1, L + 1):
delta_bias[L - i] = np.array(delta_z[L + 1 - i])
delta_weights[L - i] = np.outer(delta_z[L + 1 - i], nodes[L - i])
delta_nodes[L -
i] = np.dot(delta_weights[L - i],
delta_z[L + 1 - i]) / LAYER_SIZE[L - i + 1]
delta_z[L -
i] = delta_nodes[L - 1] * activation_func(z[L - i], True)
total_delta_weights += delta_weights
total_delta_bias += delta_bias
#print(nodes)
print(str(iter) + "::" + str(j) + str(nodes[L]))
weights -= DAMP * total_delta_weights / len(inputs)
bias -= DAMP * total_delta_bias / len(inputs)
# delta_weights[i][j][k] = - DAMP * cost_deriv[j] * activ_deriv[i + 1][j] * nodes[i][j][k]
def cosine(mean_center_rating, **unused_kwargs):
_filled_rating = mean_center_rating.filled(0)
_c = np.dot(_filled_rating, _filled_rating.T)
_diag = np.sqrt(np.diag(_c))
_denom = np.outer(_diag, _diag)
return _c / _denom
def griddata(filelist,
pixPerBeam=3.5,
templateHeader=None,
gridFunction=jincGrid,
startChannel=None,
endChannel=None,
rebase=None,
rebaseorder=None,
beamSize=None,
flagSpatialOutlier=False,
projection='TAN',
outdir=None,
outname=None,
dtype=np.float64,
gainDict=None,
**kwargs):
"""Gridding code for GBT spectral scan data produced by pipeline.
Parameters
----------
filelist : list
List of FITS files to be gridded into an output
Keywords
--------
pixPerBeam : float
Number of pixels per beam FWHM
templateHeader : `Header` object
Template header used for spatial pixel grid.
gridFunction : function
Gridding function to be used. The default `jincGrid` is a
tapered circular Bessel function. The function has call
signature of func(xPixelCentre, yPixelCenter, xData, yData,
pixPerBeam)
startChannel : int
Starting channel for spectrum within the original spectral data.
endChannel : int
End channel for spectrum within the original spectral data
outdir : str
Output directory name. Defaults to current working directory.
outname : str
Output directory file name. Defaults to object name in the
original spectra.
flagSpatialOutlier : bool
Setting to True will remove scans with positions far outside the
bounding box of the regular scan pattern. Used to catch instances
where the encoder records erroneous positions.
gainDict : dict
Dictionary that has a tuple of feed and polarization numbers
as keys and returns the gain values for that feed.
Returns
-------
None
"""
eulerFlag = False
print("Starting Gridding")
if outdir is None:
outdir = os.getcwd()
if len(filelist) == 0:
warnings.warn('There are no FITS files to process ')
return
# check that every file in the filelist is valid
# If not then remove it and send warning message
for file_i in filelist:
try:
fits.open(file_i)
except:
warnings.warn('file {0} is corrupted'.format(file_i))
filelist.remove(file_i)
# pull a test structure
hdulist = fits.open(filelist[0])
s = hdulist[1].data
# Constants block
sqrt2 = np.sqrt(2)
mad2rms = 1.4826
prefac = mad2rms / sqrt2
c = 299792458.
nu0 = s[0]['RESTFREQ']
Data_Unit = s[0]['TUNIT7']
if outname is None:
outname = s[0]['OBJECT']
# New Beam size measurements use 1.18 vs. 1.22 based on GBT Memo 296.
if beamSize is None:
beamSize = 1.18 * (c / nu0 / 100.0) * 180 / np.pi # in degrees
if startChannel is None:
startChannel = 0
if endChannel is None:
endChannel = len(s[0]['DATA'])
naxis3 = len(s[0]['DATA'][startChannel:endChannel])
# Default behavior is to park the object velocity at
# the center channel in the VRAD-LSR frame
crval3 = s[0]['RESTFREQ'] * (1 - s[0]['VELOCITY'] / c)
crpix3 = s[0]['CRPIX1'] - startChannel
ctype3 = s[0]['CTYPE1']
cdelt3 = s[0]['CDELT1']
w = wcs.WCS(naxis=3)
w.wcs.restfrq = nu0
# We are forcing this conversion to make nice cubes.
w.wcs.specsys = 'LSRK'
w.wcs.ssysobs = 'TOPOCENT'
if templateHeader is None:
wcsdict = autoHeader(filelist,
beamSize=beamSize,
pixPerBeam=pixPerBeam,
projection=projection)
w.wcs.crpix = [wcsdict['CRPIX1'], wcsdict['CRPIX2'], crpix3]
w.wcs.cdelt = np.array([wcsdict['CDELT1'], wcsdict['CDELT2'], cdelt3])
w.wcs.crval = [wcsdict['CRVAL1'], wcsdict['CRVAL2'], crval3]
w.wcs.ctype = [wcsdict['CTYPE1'], wcsdict['CTYPE2'], ctype3]
naxis2 = wcsdict['NAXIS2']
naxis1 = wcsdict['NAXIS1']
w.wcs.radesys = s[0]['RADESYS']
w.wcs.equinox = s[0]['EQUINOX']
else:
w.wcs.crpix = [
templateHeader['CRPIX1'], templateHeader['CRPIX2'], crpix3
]
w.wcs.cdelt = np.array(
[templateHeader['CDELT1'], templateHeader['CDELT2'], cdelt3])
w.wcs.crval = [
templateHeader['CRVAL1'], templateHeader['CRVAL2'], crval3
]
w.wcs.ctype = [
templateHeader['CTYPE1'], templateHeader['CTYPE2'], ctype3
]
naxis2 = templateHeader['NAXIS2']
naxis1 = templateHeader['NAXIS1']
w.wcs.radesys = templateHeader['RADESYS']
w.wcs.equinox = templateHeader['EQUINOX']
pixPerBeam = np.abs(beamSize / w.pixel_scale_matrix[1, 1])
if pixPerBeam < 3.5:
warnings.warn('Template header requests {0}'.format(pixPerBeam) +
' pixels per beam.')
if (((w.wcs.ctype[0]).split('-'))[0] !=
((s[0]['CTYPE1']).split('-'))[0]):
warnings.warn('Spectral data not in same frame as template header')
eulerFlag = True
outCube = np.zeros((int(naxis3), int(naxis2), int(naxis1)), dtype=dtype)
outWts = np.zeros((int(naxis2), int(naxis1)), dtype=dtype)
xmat, ymat = np.meshgrid(np.arange(naxis1),
np.arange(naxis2),
indexing='ij')
xmat = xmat.reshape(xmat.size)
ymat = ymat.reshape(ymat.size)
xmat = xmat.astype(np.int)
ymat = ymat.astype(np.int)
ctr = 0
for thisfile in filelist:
print("Now processing {0}".format(thisfile))
print("This is file {0} of {1}".format(ctr, len(filelist)))
ctr += 1
s = fits.open(thisfile)
if len(s) < 2:
warnings.warn("Corrupted file: {0}".format(thisfile))
continue
if len(s[1].data) == 0:
warnings.warn("Corrupted file: {0}".format(thisfile))
continue
if flagSpatialOutlier:
# Remove outliers in Lat/Lon space
f = np.where(is_outlier(s[1].data['CRVAL2'], thresh=1.5) != True)
s[1].data = s[1].data[f]
f = np.where(is_outlier(s[1].data['CRVAL3'], thresh=1.5) != True)
s[1].data = s[1].data[f]
nuindex = np.arange(len(s[1].data['DATA'][0]))
flagct = 0
if eulerFlag:
if 'GLON' in s[1].data['CTYPE2'][0]:
inframe = 'galactic'
elif 'RA' in s[1].data['CTYPE2'][0]:
inframe = 'fk5'
else:
raise NotImplementedError
if 'GLON' in w.wcs.ctype[0]:
outframe = 'galactic'
elif 'RA' in w.wcs.ctype[0]:
outframe = 'fk5'
else:
raise NotImplementedError
coords = SkyCoord(s[1].data['CRVAL2'],
s[1].data['CRVAL3'],
unit=(u.deg, u.deg),
frame=inframe)
coords_xform = coords.transform_to(outframe)
if outframe == 'fk5':
longCoord = coords_xform.ra.deg
latCoord = coords_xform.dec.deg
elif outframe == 'galactic':
longCoord = coords_xform.l.deg
latCoord = coords_xform.b.deg
else:
longCoord = s[1].data['CRVAL2']
latCoord = s[1].data['CRVAL3']
spectra, outscan, specwts, tsys = preprocess(thisfile,
startChannel=startChannel,
endChannel=endChannel,
**kwargs)
for i in range(len(spectra)):
xpoints, ypoints, zpoints = w.wcs_world2pix(
longCoord[i], latCoord[i], spectra[i]['CRVAL1'], 0)
if (tsys[i] > 10) and (xpoints > 0) and (xpoints < naxis1) \
and (ypoints > 0) and (ypoints < naxis2):
pixelWeight, Index = gridFunction(xmat, ymat, xpoints, ypoints,
pixPerBeam)
vector = np.outer(outscan[i, :] * specwts[i, :],
pixelWeight / tsys[i]**2)
wts = pixelWeight / tsys[i]**2
outCube[:, ymat[Index], xmat[Index]] += vector
outWts[ymat[Index], xmat[Index]] += wts
# Temporarily do a file write for every batch of scans.
outWtsTemp = np.copy(outWts)
outWtsTemp.shape = (1, ) + outWtsTemp.shape
outCubeTemp = np.copy(outCube)
outCubeTemp /= outWtsTemp
hdr = fits.Header(w.to_header())
hdr = addHeader_nonStd(hdr, beamSize, s[1].data)
#
hdu = fits.PrimaryHDU(outCubeTemp, header=hdr)
hdu.writeto(outdir + '/' + outname + '.fits', overwrite=True)
outWts.shape = (1, ) + outWts.shape
outCube /= outWts
# Create basic fits header from WCS structure
hdr = fits.Header(w.to_header())
# Add non standard fits keyword
hdr = addHeader_nonStd(hdr, beamSize, s[1].data[0])
hdr.add_history('Using GBTPIPE gridder version {0}'.format(__version__))
hdu = fits.PrimaryHDU(outCube, header=hdr)
hdu.writeto(outdir + '/' + outname + '.fits', overwrite=True)
w2 = w.dropaxis(2)
hdr2 = fits.Header(w2.to_header())
hdu2 = fits.PrimaryHDU(outWts, header=hdr2)
hdu2.writeto(outdir + '/' + outname + '_wts.fits', overwrite=True)
if rebase:
if rebaseorder is None:
rebaseorder = blorder
if 'NH3_11' in outname:
Baseline.rebaseline(outdir + '/' + outname + '.fits',
windowFunction=Baseline.ammoniaWindow,
line='oneone',
blorder=rebaseorder,
**kwargs)
elif 'NH3_22' in outname:
Baseline.rebaseline(outdir + '/' + outname + '.fits',
windowFunction=Baseline.ammoniaWindow,
line='twotwo',
blorder=rebaseorder,
**kwargs)
elif 'NH3_33' in outname:
Baseline.rebaseline(outdir + '/' + outname + '.fits',
winfunc=Baseline.ammoniaWindow,
blorder=rebaseorder,
line='threethree',
**kwargs)
else:
Baseline.rebaseline(outdir + '/' + outname + '.fits',
blorder=rebaseorder,
windowFunction=Baseline.tightWindow,
**kwargs)
def applyLag(self, to, polyn): # 'p' is already in the digital domain; return np.outer( self.p**to, polyn )
def fit(self, **kwargs):
"""
Estimate the EDR space.
Parameters
----------
slice_n : int
Number of observations per slice
"""
# Sample size per slice
slice_n = kwargs.get("slice_n", 50)
# Number of slices
n_slice = self.exog.shape[0] // slice_n
self._prep(n_slice)
cv = [np.cov(z.T) for z in self._split_wexog]
ns = [z.shape[0] for z in self._split_wexog]
p = self.wexog.shape[1]
if not self.bc:
# Cook's original approach
vm = 0
for w, cvx in zip(ns, cv):
icv = np.eye(p) - cvx
vm += w * np.dot(icv, icv)
vm /= len(cv)
else:
# The bias-corrected approach of Li and Zhu
# \Lambda_n in Li, Zhu
av = 0
for c in cv:
av += np.dot(c, c)
av /= len(cv)
# V_n in Li, Zhu
vn = 0
for x in self._split_wexog:
r = x - x.mean(0)
for i in range(r.shape[0]):
u = r[i, :]
m = np.outer(u, u)
vn += np.dot(m, m)
vn /= self.exog.shape[0]
c = np.mean(ns)
k1 = c * (c - 1) / ((c - 1)**2 + 1)
k2 = (c - 1) / ((c - 1)**2 + 1)
av2 = k1 * av - k2 * vn
vm = np.eye(p) - 2 * sum(cv) / len(cv) + av2
a, b = np.linalg.eigh(vm)
jj = np.argsort(-a)
a = a[jj]
b = b[:, jj]
params = np.linalg.solve(self._covxr.T, b)
results = DimReductionResults(self, params, eigs=a)
return DimReductionResultsWrapper(results)
def calculateGamSq(self, R=None, eb=False):
r"""Calculate the tophat shear variance from the correlation function.
.. math::
\langle \gamma^2 \rangle(R) &= \int_0^{2R} \frac{r dr}{R^2} S_+(s) \xi_+(r) \\
\langle \gamma^2 \rangle_E(R) &= \int_0^{2R} \frac{r dr}{2 R^2}
\left[ S_+\left(\frac{r}{R}\right) \xi_+(r) +
S_-\left(\frac{r}{R}\right) \xi_-(r) \right] \\
\langle \gamma^2 \rangle_B(R) &= \int_0^{2R} \frac{r dr}{2 R^2}
\left[ S_+\left(\frac{r}{R}\right) \xi_+(r) -
S_-\left(\frac{r}{R}\right) \xi_-(r) \right] \\
S_+(s) &= \frac{1}{\pi} \left(4 \arccos(s/2) - s \sqrt{4-s^2} \right) \\
S_-(s) &= \begin{cases}
s<=2, & \frac{1}{\pi s^4} \left(s \sqrt{4-s^2} (6-s^2) - 8(3-s^2) \arcsin(s/2)\right)\\
s>=2, & \frac{1}{s^4} \left(4(s^2-3)\right)
\end{cases}
cf. Schneider, et al (2002): A&A, 389, 729
The default behavior is not to compute the E/B versions. They are calculated if
eb is set to True.
.. note::
This function is only implemented for Log binning.
Parameters:
R (array): The R values at which to calculate the shear variance.
(default: None, which means use self.rnom)
eb (bool): Whether to include the E/B decomposition as well as the total
:math:`\langle \gamma^2\rangle`. (default: False)
Returns:
Tuple containing
- gamsq = array of :math:`\langle \gamma^2 \rangle(R)`
- vargamsq = array of the variance estimate of gamsq
- gamsq_e (Only if eb is True) = array of :math:`\langle \gamma^2 \rangle_E(R)`
- gamsq_b (Only if eb is True) = array of :math:`\langle \gamma^2 \rangle_B(R)`
- vargamsq_e (Only if eb is True) = array of the variance estimate of
gamsq_e or gamsq_b
"""
if self.bin_type != 'Log':
raise ValueError("calculateGamSq requires Log binning.")
if R is None:
R = self.rnom
s = np.outer(1./R, self.meanr)
ssq = s*s
Sp = np.zeros_like(s)
sa = s[s<2]
ssqa = ssq[s<2]
Sp[s<2.] = 1./np.pi * ssqa * (4.*np.arccos(sa/2.) - sa*np.sqrt(4.-ssqa))
# Now do the integral by taking the matrix products.
# Note that dlogr = bin_size
Spxip = Sp.dot(self.xip)
gamsq = Spxip * self.bin_size
vargamsq = (Sp**2).dot(self.varxip) * self.bin_size**2
# Stop here if eb is False
if not eb: return gamsq, vargamsq
Sm = np.empty_like(s)
Sm[s<2.] = 1./(ssqa*np.pi) * (sa*np.sqrt(4.-ssqa)*(6.-ssqa)
-8.*(3.-ssqa)*np.arcsin(sa/2.))
Sm[s>=2.] = 4.*(ssq[s>=2]-3.)/ssq[s>=2]
# This already includes the extra ssq factor.
Smxim = Sm.dot(self.xim)
gamsq_e = (Spxip + Smxim) * 0.5 * self.bin_size
gamsq_b = (Spxip - Smxim) * 0.5 * self.bin_size
vargamsq_e = (Sp**2).dot(self.varxip) + (Sm**2).dot(self.varxim)
vargamsq_e *= 0.25 * self.bin_size**2
return gamsq, vargamsq, gamsq_e, gamsq_b, vargamsq_e
def cal_pca(point_cloud,
is_show=False,
desired_num_of_feature=3,
title="pca demo"):
pca = PCA(n_components=desired_num_of_feature)
pca.fit(point_cloud)
# print("Principal vectors: ",pca.components_)
# print("Singular values: ",pca.explained_variance_)
if is_show:
fig = plt.figure()
ax = fig.add_subplot(111, projection='3d')
ax.set_xlabel('X Label(unit:m)')
ax.set_ylabel('Y Label(unit:m)')
ax.set_zlabel('Z Label(unit:m)')
plt.title(title)
ax.scatter(point_cloud[:, 0],
point_cloud[:, 1],
point_cloud[:, 2],
c='y',
s=1)
xm, ym, zm = get_centroid_from_pc(point_cloud)
ax.scatter(xm, ym, zm, c='r', s=10)
print("*" * 30)
print("z 向量 %f ,%f ,%f" %
(pca.components_[2, 0], pca.components_[2, 1],
pca.components_[2, 2]))
if (np.inner(pca.components_[2, :], [0, 0, 1]) > 0):
print("pca_z向量與z方向同向，需要對x軸旋轉180度")
pca.components_[2, :] = -pca.components_[2, :]
# r = R.from_euler('x',180, degrees=True)
# r_b_o=R.from_dcm(pca.components_.T)
# r3=r_b_o*r
# pca.components_=r3.as_dcm().T
# 求出x,y的外積,應該為z 看是否與第三軸同向確認是否為正確
x_axis_matrix = np.outer(pca.components_[1, :], pca.components_[2, :])
x_axis = np.asarray([
x_axis_matrix[1, 2] - x_axis_matrix[2, 1],
x_axis_matrix[2, 0] - x_axis_matrix[0, 2],
x_axis_matrix[0, 1] - x_axis_matrix[1, 0]
])
print("*" * 30)
print("外積計算的x軸為:")
print(x_axis)
# 確認pca_x與經由外積(y,z)計算的x同向
if (np.allclose(pca.components_[0, :], x_axis)):
print("pca_x與外積(y,z)計算的x同向")
else:
# 反向，將不重要的x軸轉向
print("x方向不正確，需替換成正確的項")
pca.components_[0, :] = x_axis
if (np.inner(pca.components_[0, :], [1, 0, 0]) < 0):
# 希望夾爪朝前，這樣末端點就不需要轉太多
print("pca_x向量與x方向反向，需要對x軸旋轉180度")
r = R.from_euler('z', 180, degrees=True)
r_b_o = R.from_dcm(pca.components_.T)
r3 = r_b_o * r
pca.components_ = r3.as_dcm().T
discount = 1
print("*" * 30)
for length, vector in zip(pca.explained_variance_, pca.components_):
ax.quiver(xm,
ym,
zm,
vector[0],
vector[1],
vector[2],
length=discount)
discount /= 3
plt.show()
return pca.components_, pca.explained_variance_
def getDataObject(self, key, selection=None):
"""
Parameters:
* key: key to be read from source. It is a string
using the following formats:
"s.o": loads all counter values (s=scan number, o=order)
- if ScanType==SCAN: in a 2D array (mot*cnts)
- if ScanType==MESH: in a 3D array (mot1*mot2*cnts)
- if ScanType==MCA: single MCA in 1D array (0:channels)
"s.o.n": loads a single MCA in a 1D array (0:channels)
- if ScanType==NMCA: n is the MCA number from 1 to N
- if ScanType==SCAN+MCA: n is the scan point number (from 1)
- if ScanType==MESH+MCA: n is the scan point number (from 1)
"s.o.p.n": loads a single MCA in a 1D array (0:channels)
- if ScanType==SCAN+NMCA:
p is the point number in the scan
n is the MCA device number
- if ScanType==MESH+MCA:
p is first motor index
n is second motor index
"s.o.MCA": loads all MCA in an array
- if ScanType==SCAN+MCA: 2D array (pts*mca)
- if ScanType==NMCA: 2D array (mca_det*mca)
- if ScanType==MESH+MCA: 3D array (pts_mot1*pts_mot2*mca)
- if ScanType==SCAN+NMCA: 3D array (pts_mot1*mca_det*mca)
- if ScanType==MESH+NMCA:
creates N data page, one for each MCA device,
with a 3D array (pts_mot1*pts_mot2*mca)
"""
key_type = self.__getKeyType(key)
if key_type == "scan":
scan_key = key
elif key_type == "mca":
(scan_key, mca_no) = self.__getMcaPars(key)
if self.__source_info_cached is None:
sourceinfo = self.getSourceInfo()
sourcekeys = sourceinfo['KeyList']
else:
sourceinfo = self.__source_info_cached
sourcekeys = sourceinfo['KeyList']
if scan_key not in sourcekeys:
sourceinfo = self.getSourceInfo()
sourcekeys = sourceinfo['KeyList']
if scan_key not in sourcekeys:
raise KeyError("Key %s not in source keys" % key)
mca3D = False
if DEBUG:
print("SELECTION = ", selection)
print("key_type = ", key_type)
if key_type == "scan":
if selection is not None:
if 'mcalist' in selection:
mca3D = True
if (key_type == "scan") and (not mca3D):
output = self._getScanData(key, raw=True)
output.x = None
output.y = None
output.m = None
output.info['selection'] = selection
if selection is None:
output.info['selectiontype'] = "2D"
return output
elif type(selection) != type({}):
#I only understand index selections
raise TypeError(
"Only selections of type {x:[],y:[],m:[]} understood")
else:
if 'x' in selection:
indexlist = []
for labelindex in selection['x']:
if labelindex != 0:
if 'cntlist' in selection:
label = selection['cntlist'][labelindex]
else:
label = output.info['LabelNames'][labelindex]
else:
label = output.info['LabelNames'][labelindex]
if label not in output.info['LabelNames']:
raise ValueError("Label %s not in scan labels" %
label)
index = output.info['LabelNames'].index(label)
if output.x is None:
output.x = []
output.x.append(output.data[:, index])
indexlist.append(index)
output.info['selection']['x'] = indexlist
if 'y' in selection:
indexlist = []
for labelindex in selection['y']:
if 'cntlist' in selection:
label = selection['cntlist'][labelindex]
else:
label = output.info['LabelNames'][labelindex]
if label not in output.info['LabelNames']:
raise ValueError("Label %s not in scan labels" %
label)
index = output.info['LabelNames'].index(label)
if output.y is None:
output.y = []
output.y.append(output.data[:, index])
indexlist.append(index)
output.info['selection']['y'] = indexlist
if 'm' in selection:
indexlist = []
for labelindex in selection['m']:
if 'cntlist' in selection:
label = selection['cntlist'][labelindex]
else:
label = output.info['LabelNames'][labelindex]
if label not in output.info['LabelNames']:
raise ValueError("Label %s not in scan labels" %
label)
index = output.info['LabelNames'].index(label)
if output.m is None:
output.m = []
output.m.append(output.data[:, index])
indexlist.append(index)
output.info['selection']['m'] = indexlist
output.info['selection']['cntlist'] = output.info['LabelNames']
output.info['selectiontype'] = "1D"
if output.x is not None:
output.info['selectiontype'] = "%dD" % len(output.x)
output.data = None
elif key_type == "mca":
output = self._getMcaData(key)
selectiontype = "1D"
if selection is not None:
selectiontype = selection.get('selectiontype', "1D")
output.info['selectiontype'] = selectiontype
if output.info['selectiontype'] not in ['2D', '3D', 'STACK']:
ch0 = int(output.info['Channel0'])
output.x = [
numpy.arange(ch0,
ch0 + len(output.data)).astype(numpy.float)
]
output.y = [output.data[:].astype(numpy.float)]
output.m = None
output.data = None
else:
output.x = None
output.y = None
output.m = None
output.data = None
npoints = output.info['NbMca'] / output.info['NbMcaDet']
index = 0
scan_obj = self._sourceObjectList[index].select(scan_key)
SPECFILE = True
if isinstance(self._sourceObjectList[index],
specfile.specfilewrapper):
SPECFILE = False
for i in range(npoints):
if SPECFILE:
wmca_no = mca_no + output.info['NbMcaDet'] * i
mcaData = scan_obj.mca(wmca_no)
else:
mca_key = '%s.%d' % (scan_key, mca_no)
mcaData = self._getMcaData(mca_key).data
if i == 0:
nChannels = mcaData.shape[0]
output.data = numpy.zeros((npoints, nChannels),
numpy.float32)
output.data[i, :] = mcaData
#I have all the MCA data ready for image plot
if selectiontype == 'STACK':
output.data.shape = 1, npoints, -1
shape = output.data.shape
for i in range(len(shape)):
key = 'Dim_%d' % (i + 1, )
output.info[key] = shape[i]
output.info["SourceType"] = "SpecFileStack"
output.info["SourceName"] = self.sourceName
output.info["Size"] = shape[0] * shape[1]
output.info["NumberOfFiles"] = 1
output.info["FileIndex"] = 1
elif (key_type == "scan") and mca3D:
output = self._getScanData(key, raw=True)
output.x = None
output.y = None
output.m = None
#get the number of counters in the scan
if 'cntlist' in selection:
ncounters = len(selection['cntlist'])
else:
ncounters = output.info['LabelNames']
# For the time being assume only one mca can be selected
detectorNumber = selection['y'][0] - ncounters
#read the first mca data of the first point
mca_key = '%s.%d.%d' % (key, 1 + detectorNumber, 1)
mcaData = self._getMcaData(mca_key)
ch0 = int(mcaData.info['Channel0'])
calib = mcaData.info['McaCalib']
nChannels = float(mcaData.data.shape[0])
channels = numpy.arange(nChannels) + ch0
#apply the calibration
channels = calib[0] + calib[1] * channels +\
calib[2] * channels * channels
ones = numpy.ones(nChannels)
#get the different x components
xselection = selection.get('x', [])
if len(xselection) != 2:
raise ValueError("You have to select two X axes")
indexlist = []
for labelindex in xselection:
if labelindex != 0:
if 'cntlist' in selection:
label = selection['cntlist'][labelindex]
else:
label = output.info['LabelNames'][labelindex]
else:
label = output.info['LabelNames'][labelindex]
if label not in output.info['LabelNames']:
raise ValueError("Label %s not in scan labels" % label)
index = output.info['LabelNames'].index(label)
if output.x is None:
output.x = []
output.x.append(output.data[:, index])
indexlist.append(index)
npoints = output.x[0].shape[0]
output.info['selection'] = selection
output.info['selection']['x'] = indexlist
for i in range(len(output.x)):
output.x[i] = numpy.outer(output.x[i], ones).flatten()
tmp = numpy.outer(channels, numpy.ones(float(npoints))).flatten()
output.x.append(tmp)
output.y = [numpy.zeros(nChannels * npoints, numpy.float)]
for i in range(npoints):
mca_key = '%s.%d.%d' % (key, 1 + detectorNumber, 1)
mcaData = self._getMcaData(mca_key)
output.y[0][(i * nChannels):((i + 1) *
nChannels)] = mcaData.data[:]
if 'm' in selection:
indexlist = []
for labelindex in selection['m']:
if 'cntlist' in selection:
label = selection['cntlist'][labelindex]
else:
label = output.info['LabelNames'][labelindex]
if label not in output.info['LabelNames']:
raise ValueError("Label %s not in scan labels" % label)
index = output.info['LabelNames'].index(label)
if output.m is None:
output.m = []
output.m.append(output.data[:, index])
indexlist.append(index)
output.info['selection']['m'] = indexlist
if output.m is not None:
output.m[0] = numpy.outer(output.m[0], ones).flatten()
output.info['selection']['cntlist'] = output.info['LabelNames']
output.info['selectiontype'] = "3D"
output.info[
'LabelNames'] = selection['cntlist'] + selection['mcalist']
output.data = None
return output
l[i + 1:, i] = b / l[i, i]
sub = n - i
simd += sub // 4 + sub % 4
simd += sum(j // 4 + j % 4 for j in range(1, sub)) * 4
compute += (sub - 1) * (sub - 2) // 2
starting += sub + 12
finish = max(finish, starting + (sub - 1) * (sub - 2) // 2)
aa = a.copy()
aa[i, i] = 1
aa[i, i + 1:] = numpy.zeros(n - i - 1)
aa[i + 1:, i] = numpy.zeros(n - i - 1)
aa[i + 1:,
i + 1:] = a[i + 1:, i + 1:] - numpy.outer(numpy.conj(b), b) / a[i, i]
# Mathematically, it is L = L * l. However, in this case, we can just copy the corresponding
# column to L, because the speciality of L and l
L[i:, i] = l[i:, i]
#L = numpy.dot(L, l)
a = aa
numpy.testing.assert_allclose(origin,
numpy.dot(numpy.conj(L), L.transpose()),
rtol=1e-4)
print("Correctness check pass!")
output.print_complex_array('ref.data', L.flatten())
print("New data generated!")
init = 12 * n
def sparsery(ops):
rez, max_proj = get_mov(ops)
ops['max_proj'] = max_proj
nframes, Ly, Lx = rez.shape
ops['Lyc'] = Ly
ops['Lxc'] = Lx
sdmov = get_sdmov(rez, ops)
rez /= sdmov
#rez *= -1
lx = [ops['spatial_hp']]
c1 = square_conv2(np.ones((Ly,Lx)), lx)
movu = square_conv2(rez, lx)
rez -= movu/c1
LL = np.meshgrid(np.arange(Lx), np.arange(Ly))
Lyp = np.zeros(5, 'int32')
Lxp = np.zeros(5,'int32')
gxy = [np.array(LL).astype('float32')]
dmov = rez
movu = []
for j in range(5):
movu.append(square_conv2(dmov, [3]))
dmov = 2 * downsample(dmov)
gxy0 = downsample(gxy[j], False)
gxy.append(gxy0)
nfr, Lyp[j], Lxp[j] = movu[j].shape
movu[j] = np.reshape(movu[j], (nfr,-1))
nfr, Lyc,Lxc = rez.shape
V0 = []
ops['Vmap'] = []
for j in range(len(movu)):
V0.append(np.amax(movu[j], axis=0))
#V0.append(np.sum(movu[j]**2 * np.float32(movu[j]>Th2), axis=0)**.5)
V0[j] = np.reshape(V0[j], (Lyp[j], Lxp[j]))
ops['Vmap'].append(V0[j].copy())
I = np.zeros((len(gxy), gxy[0].shape[1], gxy[0].shape[2]))
for t in range(1,len(gxy)-1):
gmodel = RectBivariateSpline(gxy[t][1,:,0], gxy[t][0, 0,:], ops['Vmap'][t],
kx=min(3, gxy[t][1,:,0].size-1), ky=min(3, gxy[t][0,0,:].size-1))
I[t] = gmodel.__call__(gxy[0][1,:,0], gxy[0][0, 0,:])
I0 = np.amax(I, axis=0)
ops['Vcorr'] = I0
imap = np.argmax(I, axis=0).flatten()
ipk = np.abs(I0 - maximum_filter(I0, size=(11,11))).flatten() < 1e-4
isort = np.argsort(I0.flatten()[ipk])[::-1]
im, nm = mode(imap[ipk][isort[:50]])
if ops['spatial_scale'] > 0:
im = max(1, min(4, ops['spatial_scale']))
fstr = 'FORCED'
else:
fstr = 'estimated'
if im==0:
print('ERROR: best scale was 0, everything should break now!')
Th2 = ops['threshold_scaling']*5*max(1,im)
vmultiplier = max(1, np.float32(rez.shape[0])/1200)
print('NOTE: %s spatial scale ~%d pixels, time epochs %2.2f, threshold %2.2f '%(fstr, 3*2**im, vmultiplier, vmultiplier*Th2))
ops['spatscale_pix'] = 3*2**im
V0 = []
ops['Vmap'] = []
for j in range(len(movu)):
#V0.append(np.amax(movu[j], axis=0))
V0.append(np.sum(movu[j]**2 * np.float32(movu[j]>Th2), axis=0)**.5)
V0[j] = np.reshape(V0[j], (Lyp[j], Lxp[j]))
ops['Vmap'].append(V0[j].copy())
I = np.zeros((len(gxy), gxy[0].shape[1], gxy[0].shape[2]))
for t in range(1,len(gxy)-1):
gmodel = RectBivariateSpline(gxy[t][1,:,0], gxy[t][0, 0,:], ops['Vmap'][t],
kx=min(3, gxy[t][1,:,0].size-1), ky=min(3, gxy[t][0,0,:].size-1))
I[t] = gmodel.__call__(gxy[0][1,:,0], gxy[0][0, 0,:])
I0 = np.amax(I, axis=0)
ops['Vcorr'] = I0
xpix,ypix,lam = [],[],[]
rez = np.reshape(rez, (-1,Ly*Lx))
lxs = 3 * 2**np.arange(5)
nscales = len(lxs)
niter = 250 * ops['max_iterations']
Vmax = np.zeros((niter))
ihop = np.zeros((niter))
vrat = np.zeros((niter))
Npix = np.zeros((niter))
t0 = tic()
for tj in range(niter):
v0max = np.array([np.amax(V0[j]) for j in range(5)])
imap = np.argmax(v0max)
imax = np.argmax(V0[imap])
yi, xi = np.unravel_index(imax, (Lyp[imap], Lxp[imap]))
yi, xi = gxy[imap][1,yi,xi], gxy[imap][0,yi,xi]
Vmax[tj] = np.amax(v0max)
if Vmax[tj] < vmultiplier*Th2:
break
ls = lxs[imap]
ihop[tj] = imap
ypix0, xpix0, lam0 = add_square(int(yi),int(xi),ls,Ly,Lx)
xproj = rez[:, ypix0*Lx+ xpix0] @ lam0
goodframe = np.nonzero(xproj>Th2)[0]
for j in range(3):
ypix0, xpix0, lam0 = iter_extend(ypix0, xpix0, rez, Ly,Lx, goodframe)
xproj = rez[:, ypix0*Lx+ xpix0] @ lam0
goodframe = np.nonzero(xproj>Th2)[0]
if len(goodframe)<1:
break
if len(goodframe)<1:
break
vrat[tj], ipack = two_comps(rez[:, ypix0*Lx+ xpix0], lam0, Th2)
if vrat[tj]>1.25:
lam0, xp, goodframe = ipack
xproj[goodframe] = xp
ix = lam0>lam0.max()/5
xpix0 = xpix0[ix]
ypix0 = ypix0[ix]
lam0 = lam0[ix]
# update residual on raw movie
rez[np.ix_(goodframe, ypix0*Lx+ xpix0)] -= xproj[goodframe][:,np.newaxis] * lam0
# update filtered movie
ys, xs, lms = multiscale_mask(ypix0,xpix0,lam0, Lyp, Lxp)
for j in range(nscales):
movu[j][np.ix_(goodframe,xs[j]+Lxp[j]*ys[j])] -= np.outer(xproj[goodframe], lms[j])
#V0[j][xs[j] + Lxp[j]*ys[j]] = np.amax(movu[j][:,xs[j]+Lxp[j]*ys[j]], axis=0)
Mx = movu[j][:,xs[j]+Lxp[j]*ys[j]]
#V0[j][xs[j] + Lxp[j]*ys[j]] = np.sum(Mx**2 * np.float32(Mx>Th2), axis=0)**.5
V0[j][ys[j], xs[j]] = np.sum(Mx**2 * np.float32(Mx>Th2), axis=0)**.5
#V0[j][xs[j] + Lxp[j]*ys[j]] = np.sum(movu[j][:,xs[j]+Lxp[j]*ys[j]]**2 * np.float32(movu[j][:,xs[j]+Lxp[j]*ys[j]]>Th2), axis=0)**.5
xpix.append(xpix0)
ypix.append(ypix0)
lam.append(lam0)
if tj%1000==0:
print('%d ROIs, score=%2.2f'%(tj, Vmax[tj]))
#print(tj, time.time()-t0, Vmax[tj])
ops['Vmax'] = Vmax
ops['ihop'] = ihop
ops['Vsplit'] = vrat
stat = [{'ypix':ypix[n], 'lam':lam[n]*sdmov[ypix[n], xpix[n]], 'xpix':xpix[n]} for n in range(len(xpix))]
stat = get_stat(ops, stat)
return ops,stat
def _compute_pam_sd(wcs, shape=None, blc=(1, 1), idcscale=1.0, cdscale=1.0):
"""
Computes Pixel Area Map (PAM) using the distortion model defined in WCS
and described through Simple Image Polynomials (SIP) by computing
the Jacobian of the distortion model.
This function computes the Jacobian of the distortion model using
*symbolic differentiation* of Simple Image Polynomials.
Parameters
----------
wcs: astropy.wcs.WCS, stwcs.wcsutil.HSTWCS
A ``WCS`` object containing the distortion model.
shape: tuple of int, None, optional
A tuple of two integers (ny, nx) indicating the size of the PAM image
to be generated. When the default value is used (`None`), the size
of the returned PAM array will be determined from ``wcs.array_shape``
attribute of the supplied ``WCS`` object.
blc: tuple of int or float, optional
A tuple indicating the coordinates of the bottom-left pixel of the
PAM array to be computed. These coordinates should be given
in the image coordinate system defined by the input ``WCS`` (in which,
for example, ``WCS.crpix`` is defined). The first element specifies
the column (``"x"``-coordinate) and the second element specifies
the row (``"y"``-coordinate).
idcscale: float, optional
A positive number indicating the pixel scale used in the
"Instrument Distortion Correction" for HST instruments. For
non-HST instruments this parameter may be set to be equal
to ``cdscale``.
cdscale: float, optional
A positive number indicating the pixel scale as computed from the
CD matrix. HST instruments CD matrix includes linear distortion
terms.
Returns
-------
PAM: numpy.ndarray
Pixel area map.
"""
if shape is None:
shape = wcs.array_shape
# rescale factor:
rf = (cdscale / idcscale)**2
# distortion does not exist or is linear:
if wcs.sip is None or wcs.sip.a_order < 1 or wcs.sip.b_order < 1 or \
(wcs.sip.a_order == 1 and wcs.sip.b_order == 1):
return rf * np.ones(shape, dtype=np.float64)
# prepare coordinates:
x = np.arange(shape[1], dtype=float) - wcs.sip.crpix[0] + float(blc[0])
y = np.arange(shape[0], dtype=float) - wcs.sip.crpix[1] + float(blc[1])
ar = np.arange(wcs.sip.a_order + 1)
br = np.arange(wcs.sip.b_order + 1)
ones_a = np.ones(wcs.sip.a_order + 1)
ones_b = np.ones(wcs.sip.b_order + 1)
# "coordinate vectors" (e.g., (1, x, x**2, x**3, ...)) used in
# distortion bilinear forms:
ax = np.outer(x, ones_a)**ar
ay = np.outer(y, ones_a)**ar
bx = np.outer(x, ones_b)**br
by = np.outer(y, ones_b)**br
# derivatives of the "coordinate vectors" with regard to x & y:
adx = np.roll(ax, 1, 1) * ar
ady = np.roll(ay, 1, 1) * ar
bdx = np.roll(bx, 1, 1) * br
bdy = np.roll(by, 1, 1) * br
# derivatives of the binomial forms:
A = wcs.sip.a.T
B = wcs.sip.b.T
dadx = 1.0 + np.tensordot(ay.T, np.tensordot(A, adx, (1, 1)), (0, 0))
dady = np.tensordot(ady.T, np.tensordot(A, ax, (1, 1)), (0, 0))
dbdx = np.tensordot(by.T, np.tensordot(B, bdx, (1, 1)), (0, 0))
dbdy = 1.0 + np.tensordot(bdy.T, np.tensordot(B, bx, (1, 1)), (0, 0))
# compute rescaled Jacobian
jacobian = rf * np.abs(dadx * dbdy - dady * dbdx)
return jacobian
#set_num_threads(3)
if ex_c:
print('\n'+20*'-'+'2x2 latice'+'-'*20+'\n')
# Initial conditions for Monte Carlo simulation
max_cycles = 1e7 # Max MC cycles
L = 2 # Number of spins
temp = 1 # [kT/J] Temperature
J = 1 # binding constant
log_scale = np.logspace(2, int(np.log10(max_cycles)),\
(int(np.log10(max_cycles))-1), endpoint=True)
MC_runs = np.outer(log_scale, [1,5]).flatten() # taking the outer product
MC_runs = MC_runs[1:-1] # removing first and last value
n_sim = len(MC_runs)
print(f'Running {n_sim} simulations, max Monte-Carlo cycles:\n{MC_runs}')
# Analytic solutions
A_E, A_Cv, A_M, A_X, A_MAbs = Analytical_2x2(J, L, temp)
Analyticals = DataFrameSolution(A_E, A_Cv, A_M, A_X, A_MAbs)
# Numerical solutions
list_num_dfs = twoXtwo(L, temp, MC_runs)
Numericals = pd.concat(list_num_dfs)
print('\nTable of Analytical Solutions of 2x2 Ising-Model:','\n'+'-'*49+'\n')
import matplotlib
matplotlib.rc('xtick', labelsize=25)
matplotlib.rc('ytick', labelsize=25)
# %% low-rank network test
N = 30
dt = 0.1
T = 1000
lt = int(T/dt)
n = np.random.randn(N)
m = np.random.randn(N)
g= .1
tau = 0.1
noise = 0.
J = np.outer(n,m)/N + g**2*np.random.randn(N,N)/N
xs = np.zeros((N,lt))
rs = np.zeros((N,lt))
Is = np.ones((N,lt))#np.random.randn(N,lt)
for tt in range(lt-1):
rs[:,tt] = np.tanh(xs[:,tt])
xs[:,tt+1] = xs[:,tt] + dt*(1/tau)*(-xs[:,tt] + J @ rs[:,tt] + n/N*Is[:,tt] + noise*np.random.randn(N)*np.sqrt(dt))
plt.figure()
plt.imshow(rs,aspect='auto')
# %%
def LowD(Is, J, dt, T, noise=0):
N = J.shape[0]
lt = int(T/dt)
xs = np.zeros((N,lt))
def calculate_BLOSUM(self):
'''
I'm using amino acid substitution matrices from protein blocks, that is, the original
BLOSUM paper. since coverage varies from amino acid to amino acid, each amino acid is
considered its own block within the language used in the paper. this means w = 1 the number
of blocks equals the number of unique position identifiers within the variability table.
https://www.ncbi.nlm.nih.gov/pmc/articles/PMC50453/pdf/pnas01096-0363.pdf, and a good
explanation can be found here:
http://www.cs.columbia.edu/4761/assignments/assignment1/reference1.pdf
Q is the matrix of qij elements in the first equation of the paper
P is the matrix of pij elements in the second equation of the paper
'''
# for debugging
#self.items = ["A", "S"]
#self.master = pd.DataFrame(np.array([[9, 1], [0, 0]]), columns=self.items, index=self.items)
# CALCUTING Q
# ===========
for first in self.items:
for second in self.items:
# this condition ensures no double counting, i.e j <= i in equation 1
if self.items.index(second) > self.items.index(first):
self.master[first + second] = 0
continue
if first == second:
self.master[first + second] = self.master[first] * (
self.master[first] - 1) / 2.0
else:
self.master[
first +
second] = self.master[first] * self.master[second]
# The frequency table has redundancy, i.e. accounts for AB and BA
frequency_table = np.asarray(
self.master.loc[:, self.items[0] + self.items[0]:self.items[-1] +
self.items[-1]].sum(axis=0))
# just what equation 1 is
self.Q = frequency_table.reshape((len(self.items), len(self.items)))
self.Q = self.Q / np.sum(self.Q)
# CALCUTING P
# ===========
# this one's a thinker
self.P = np.sum(self.Q, axis=1) / 2 + np.sum(self.Q, axis=0) / 2
# CALCUTING E
# ===========
# this one's even more of a thinker
outer_product = np.outer(self.P, self.P)
self.E = np.tril(2 * outer_product -
np.identity(len(self.P)) * self.P**2)
# CALCUTING S
# ===========
self.S = np.round(2 * np.log2(self.Q / self.E))
for i in range(len(self.items)):
for j in range(i):
self.S[j, i] = self.S[i, j]
self.S = pd.DataFrame(self.S, columns=self.items, index=self.items)
def CalculateCov(arguments):
noe = arguments.noe
nsamples = arguments.nos
fpath = arguments.fpath
spath = arguments.spath
NOE = 0
SX = np.zeros(2000, dtype=np.float64)
SXI = np.zeros(2000, dtype=np.float64)
SXX = np.zeros((2000, 2000), dtype=np.float64)
SI = np.zeros(1, dtype=np.float64)
SI2 = np.zeros(1, dtype=np.float64)
fileslist = glob.glob(fpath + 'SStofPeaks_part*.mat')
nfiles = len(fileslist)
count = 0
for i in range(0, nfiles):
if i % (size) != rank:
continue # different ranks look at different file
specstruct = sio.loadmat(fileslist[i])
tofs = specstruct['peaks']
for k in range(0, len(tofs[0, :])):
peaksarray = tofs[0, k]
if len(peaksarray) == 0:
continue
tof, edges = np.histogram(np.concatenate(peaksarray),
bins=np.linspace(0, 8000, 2001))
tmpI = tof.sum()
SXX += np.outer(tof, tof)
SXI += tof * tmpI
SI += tmpI
SI2 += tmpI**2
SX += tof
count += 1
if i % (size) == 0:
print '%i/%i' % (i, np.round(nfiles / 4.0))
NOE = noe * count
# Save the data in text files.
try:
if not os.path.exists(spath):
os.makedirs(spath)
except:
print 'Tried to create folder simultaneously for several nodes'
savenameNOE = 'NOE_part%i_%s.dat' % (rank, str(count).zfill(4))
savenameSX = 'SX_part%i_%s.dat' % (rank, str(count).zfill(4))
savenameSXI = 'SXI_part%i_%s.dat' % (rank, str(count).zfill(4))
savenameSXX = 'SXX_part%i_%s.dat' % (rank, str(count).zfill(4))
savenameSI = 'SI_part%i_%s.dat' % (rank, str(count).zfill(4))
savenameSI2 = 'SI2_part%i_%s.dat' % (rank, str(count).zfill(4))
#np.savetxt(spath+savenameNOE,NOE)
np.savetxt(spath + savenameSX, SX)
np.savetxt(spath + savenameSX, SX)
np.savetxt(spath + savenameSXI, SXI)
np.savetxt(spath + savenameSXX, SXX)
np.savetxt(spath + savenameSI, SI)
np.savetxt(spath + savenameSI2, SI2)
def Der(self, ydat, Time_Initial): ydat = np.array(ydat,np.float) self.ydatM = np.outer(ydat,ydat) return np.sum(self.Coef1D*ydat,axis=1)+np.sum(np.sum(self.ydatM*self.Coef2D,axis=1),axis=1)+self.Synth
def SDP(n, npoints, convex, niter, printOut = False):
#Keep time and a running total of the RMSE to average later on
start = time.time()
totalRMSE = 0
for iteration in range(niter):
if convex:
(a, p, adjacency) = generate(n, npoints)
else:
(a, p, adjacency) = generate_inside_hull(n, npoints)
# Compute the Euclidian distances to the anchor points
adjSize = len(p) + len(a)
asize = len(a)
d = []
#Retrieve the distances
for i in range(adjSize):
for j in range(adjSize):
if(j > i and adjacency[i][j] > 0 and i < asize):
d.append((adjacency[i][j], j - asize, i, True))
elif(j > i and adjacency[i][j] > 0):
d.append((adjacency[i][j], i - asize, j - asize, False))
#Dimension of our adjancency matrix and our z matrix
T = n + npoints
z = cvx.Semidef(T)
#The following code constructs all the constraints of the SDP
#problem.
eyeConstraint = []
anchorConstraints = []
pointConstraints = []
for i in range(n):
temp = np.zeros((T,T))
temp[i][i] = 1
eyeConstraint.append(temp)
temp = np.zeros((T,T))
for i in range(n):
for j in range(n):
temp[i][j] = 1
eyeConstraint.append(temp)
for (distance, i, j, truth) in d:
if truth:
temp = np.zeros(npoints)
temp[i] = -1.
anchorConstraints.append((np.outer(np.append(a[j], temp), \
np.append(a[j], temp)), distance))
else:
tempi = np.zeros(npoints)
tempj = np.zeros(npoints)
tempi[i] = 1.
tempj[j] = 1.
temp = tempi - tempj
corner = np.zeros(n)
temp = np.append(corner, temp)
pointConstraints.append((np.outer(temp,temp), distance))
#Another empty states list
states = []
#Construct the cost and constraints
cost = cvx.norm(0)
constr = []
for i, mat in enumerate(eyeConstraint):
if i < len(eyeConstraint) - 1:
constr.append(cvx.sum_entries(cvx.mul_elemwise(mat, z)) == 1)
else:
constr.append(cvx.sum_entries(cvx.mul_elemwise(mat, z)) == n)
for mat in anchorConstraints:
constr.append(cvx.sum_entries(cvx.mul_elemwise(mat[0], z)) == \
mat[1] ** 2)
for mat in pointConstraints:
constr.append(cvx.sum_entries(cvx.mul_elemwise(mat[0], z)) == \
mat[1] ** 2)
#Force the matrix to be SDP
constr.append(z >> 0)
#Add the constraints and cost function
states.append(cvx.Problem(cvx.Minimize(cost), constr))
#Solve the SDP relaxation problem
prob = sum(states)
prob.solve();
#Compute this trials RMSE
RMSE = 0
for i in range(npoints):
soln1 = z.value.A[0:n, i + n]
point1 = p[i]
if printOut:
print("Sensor " + str(i) + " is located at " + str(soln1) + " and the actual value is " + str(point1))
RMSE += np.linalg.norm(np.asarray(soln1) - np.asarray(point1)) ** 2
#Add the trial's RMSE to the running total
totalRMSE += RMSE
#Compute the total RMSE
end = time.time()
print "Total Time Elapsed: ", end - start
print "Average RMSE: ", math.sqrt(totalRMSE / niter)
import torch
from numpy.testing import assert_array_equal, assert_array_almost_equal
from scipy.signal import triang
from pylops.signalprocessing import Convolve1D
from pylops_gpu.utils.backend import device
from pylops_gpu.utils import dottest
from pylops_gpu.signalprocessing import Convolve1D as gConvolve1D
from pylops_gpu.optimization.cg import cg
# filters
nfilt = (5, 7)
h1 = torch.from_numpy(triang(nfilt[0], sym=True).astype(np.float32))
h2 = torch.from_numpy(
np.outer(triang(nfilt[0], sym=True), triang(nfilt[1],
sym=True)).astype(np.float32))
par1_1d = {
'nz': 21,
'ny': 51,
'nx': 51,
'offset': nfilt[0] // 2,
'dir': 0
} # zero phase, first direction
par2_1d = {
'nz': 21,
'ny': 51,
'nx': 51,
'offset': 2,
'dir': 0
} # non-zero phase, first direction
sigma = 600
a = pd.read_csv('pj2.dat', sep='\s+', header=None)
a = a.values.T
if (timemax > len(a[0])):
timemax = len(a[0])
time = a[0, :timemax] * myload.args['mapdtime']
# and add Gauss smooth
real = a[1, :timemax] * np.exp(-time * time /
(2 * sigma**2)) # real part of <phi|U|Psi>
imag = a[2, :timemax] * np.exp(-time * time /
(2 * sigma**2)) # imaginary part of <phi|U|Psi>
E = np.linspace(-0.03, -0.01, Ndivide + 1)
Et = np.outer(E, time)
expiEt_r = np.cos(Et)
expiEt_i = np.sin(Et)
realpart = (np.dot(expiEt_r, real) -
np.dot(expiEt_i, imag)) * myload.args['mapdtime']
imagpart = (np.dot(expiEt_r, imag) +
np.dot(expiEt_i, real)) * myload.args['mapdtime']
plt.xlabel('E [a.u.]')
plt.ylabel(' [Im Arbitrary] ')
plt.plot(E, imagpart, 'b-', linewidth=0.5)
plt.savefig('espec.png')
plt.show()
def admm(X, y, max_iter=3000):
# solve by inner point method
m, n = X.shape
X = np.column_stack((X, np.ones((m, 1))))
y = y.astype(np.float64)
data_num = len(y)
C = 1.0
kernel = np.dot(X, np.transpose(X))
p = np.matrix(np.multiply(kernel,np.outer(y, y)))
e = np.matrix(np.ones([data_num, 1], np.float64))
bounds = (0, C)
low, up = bounds
x = np.ones((m,1))
tau = 1.618
sigma = 1
# initial
u = np.ones((m, 1))
t = x
A = p + sigma * np.eye(m)
I = np.eye(m)
invA = cg(A, I)
for it in range(max_iter):
# update x
b = e + u + sigma * t
x = invA * b
# update y
t = x - (1/sigma)*u
t[t < low] = low
t[t > up] = up
# update u
u = u - tau*sigma*(x-t)
#----bug----
#dual = -(0.5*x.T*(p*x) - e.T*x)
dual = -(0.5*x.T*(1.3811429435906217*p*x) - e.T*x)
dual = dual.item()
y1 = np.reshape(y, (-1, 1))
lambda1 = np.multiply(x, y1)
w = np.dot(X.T, lambda1)
w = np.matrix(w).reshape(-1, 1)
tmp = np.maximum(1-np.multiply(y1, X*w),0)
primal = 0.5*np.linalg.norm(w)**2 + 1 * np.sum(tmp)
primal = primal.item()
# stop criteria
if np.abs(dual-primal)/(1+np.abs(dual)+np.abs(primal)) < 1e-12:
break
# print(t, np.linalg.norm(gradient))
# print(np.min(x), np.max(x))
# print(np.sum(x < -1e-4), np.sum(x>1+1e-4))
# print(np.abs(dual-primal)/(1+np.abs(dual)+np.abs(primal)))
y1 = np.reshape(y, (-1, 1))
alpha1 = x
lambda1 = np.multiply(y1,alpha1)
w = np.dot(X.T, lambda1)
w = np.array(w).reshape(-1)
b = w[n]
w = w[0:n]
return w, b
def setup(self):
surfaces = self.options['surfaces']
system_size = 0
# Loop through the surfaces to compute the total number of panels;
# the system_size
for surface in surfaces:
mesh = surface['mesh']
nx = mesh.shape[0]
ny = mesh.shape[1]
system_size += (nx - 1) * (ny - 1)
self.system_size = system_size
self.add_input('freestream_velocities', shape=(system_size, 3), units='m/s')
self.add_output('mtx', shape=(system_size, system_size), units='1/m')
self.add_output('rhs', shape=system_size, units='m/s')
# Set up indicies arrays for sparse Jacobians
vel_indices = np.arange(system_size * 3).reshape((system_size, 3))
mtx_indices = np.arange(system_size * system_size).reshape((system_size, system_size))
rhs_indices = np.arange(system_size)
self.declare_partials('rhs', 'freestream_velocities',
rows=np.einsum('i,j->ij', rhs_indices, np.ones(3, int)).flatten(),
cols=vel_indices.flatten()
)
ind_1 = 0
ind_2 = 0
# Loop through each surface to add inputs and set up derivatives.
# We keep track of the surface's indices within the total system's
# indices to access the matrix in the correct locations for the derivs.
# This is because the AIC linear system has information for all surfaces
# together.
for surface in surfaces:
mesh=surface['mesh']
nx = mesh.shape[0]
ny = mesh.shape[1]
name = surface['name']
num = (nx - 1) * (ny - 1)
ind_2 += num
# Get the correct names for each vel_mtx and normals, then
# add them to the component
vel_mtx_name = '{}_{}_vel_mtx'.format(name, 'coll_pts')
normals_name = '{}_normals'.format(name)
self.add_input(vel_mtx_name,
shape=(system_size, nx - 1, ny - 1, 3), units='1/m')
self.add_input(normals_name, shape=(nx - 1, ny - 1, 3))
velocities_indices = np.arange(system_size * num * 3).reshape(
(system_size, nx - 1, ny - 1, 3)
)
normals_indices = np.arange(num * 3).reshape((num, 3))
# Declare each set of partials based on the indices, ind_1 and ind_2
self.declare_partials('mtx', vel_mtx_name,
rows=np.einsum('ij,k->ijk', mtx_indices[:, ind_1:ind_2], np.ones(3, int)).flatten(),
cols=velocities_indices.flatten(),
)
self.declare_partials('mtx', normals_name,
rows=np.einsum('ij,k->ijk', mtx_indices[ind_1:ind_2, :], np.ones(3, int)).flatten(),
cols=np.einsum('ik,j->ijk', normals_indices, np.ones(system_size, int)).flatten(),
)
self.declare_partials('rhs', normals_name,
rows=np.outer(rhs_indices[ind_1:ind_2], np.ones(3, int)).flatten(),
cols=normals_indices.flatten(),
)
ind_1 += num
self.mtx_n_n_3 = np.zeros((system_size, system_size, 3))
self.normals_n_3 = np.zeros((system_size, 3))
self.set_check_partial_options(wrt='*', method='fd', step=1e-5)
def _l1qc_newton(x0, u0, A, b, epsilon, tau, newtontol, newtonmaxiter, cgtol, cgmaxiter, verbose, use_CG):
# line search parameters
alpha = 0.01
beta = 0.5
AtA = np.dot(A.T,A)
x = x0.flatten()
u = u0.flatten()
r = np.dot(A, x).flatten() - b.flatten()
fu1 = x - u
fu2 = -x - u
fe = 0.5*(np.asscalar(np.dot(r.T,r)) - epsilon**2)
f = np.sum(u) - (1.0/tau) * (np.sum(np.log(-fu1)) + np.sum(np.log(-fu2)) + np.log(-fe))
niter = 0
done = 0
while (not(done)):
atr = np.dot(A.T, r)
ntgz = 1.0/fu1 - 1.0/fu2 + 1.0/fe * atr
ntgu = -tau - 1.0/fu1 - 1.0/fu2
gradf = - (1.0/tau) * np.hstack([ntgz, ntgu])
sig11 = 1.0/fu1**2 + 1.0/fu2**2
sig12 = -1.0/fu1**2 + 1.0/fu2**2
sigx = sig11 - sig12**2/sig11
w1p = ntgz - sig12/sig11 *ntgu
H11p = np.diag(sigx.reshape(len(sigx))) - (1.0/fe) * AtA + (1.0/fe)**2 * np.outer(atr,atr)
if use_CG:
dx, cgres, cgiter = _CG_solve(H11p, w1p, cgmaxiter, cgtol)
else:
dx = np.linalg.solve(H11p, w1p).flatten()
cgres = np.linalg.norm(np.dot(H11p, dx).flatten() - w1p.flatten()) / np.linalg.norm(w1p)
cgiter = -1
if (cgres > 0.5):
if verbose:
print("cgres = " + str(cgres) )
print('Cannot solve system. Returning previous iterate.' )
xp = x.flatten()
up = u.flatten()
return xp, up, 0
Adx = np.dot(A,dx).flatten()
du = (1.0/sig11) * ntgu - (sig12/sig11)*dx
# minimum step size that stays in the interior
aqe = np.dot(Adx.T, Adx)
bqe = 2.0*np.dot(r.T, Adx)
cqe = np.asscalar(np.dot(r.T,r)) - epsilon**2
smax = np.min(np.hstack([ 1.0,np.min(np.hstack([-fu1[(dx-du) > 0] / (dx[(dx-du) > 0] - du[(dx-du) > 0]),\
-fu2[(-dx-du) > 0] / (-dx[(-dx-du) > 0] - du[(-dx-du) > 0]), \
np.reshape((-bqe + np.sqrt(bqe**2 - 4 * aqe * cqe)) / (2.0*aqe), (1,)) ] ))]))
s = (0.99) * smax
# backtracking line search
suffdec = 0
backiter = 0
while not(suffdec):
xp = x + s*dx
up = u + s*du
rp = r + s*Adx
fu1p = xp - up
fu2p = -xp - up
fep = 0.5 * (np.linalg.norm(rp)**2 - epsilon**2)
fp = np.sum(up) - (1.0/tau) * (np.sum(np.log(-fu1p)) + np.sum(np.log(-fu2p)) + np.log(-fep))
flin = f + alpha * s * (np.dot(gradf.T, np.hstack([dx, du])))
suffdec = (fp <= flin)
s = beta * s
backiter = backiter + 1
if (backiter > 32):
if verbose:
print('Stuck on backtracking line search, returning previous iterate.')
xp = x.copy()
up = u.copy()
return xp,up,niter
# set up for next iteration
x = xp.copy()
u = up.copy()
r = rp.copy()
fu1 = fu1p.copy()
fu2 = fu2p.copy()
fe = fep.copy()
f = fp.copy()
lambda2 = -(np.dot(gradf, np.hstack([dx, du])))
stepsize = s*np.linalg.norm(np.hstack([dx, du]))
niter = niter + 1
done = (lambda2/2 < newtontol) | (niter >= newtonmaxiter)
if verbose:
print('Newton iter = ' + str(niter) + ', Functional = ' + str(f) + ', Newton decrement = ' + str(lambda2/2) + ', Stepsize = ' + str(stepsize))
print(' CG Res = ' + str(cgres) + ', CG Iter = ' + str(cgiter))
return xp, up, niter
contours_points = (prim_n - np.array([int(x_center), int(y_center)]))*np.array([1, -1])
# print(contours_points, contours_points.shape)
c_3d = max(ellipsis_out[1]) / 2
a_3d = ellipsis_out[1][0] / 2
b_3d = ellipsis_out[1][1] / 2
Z_3 = c_3d*(np.abs(1-contours_points[:, :, 1]**2/b_3d**2-contours_points[:, :, 0]**2/a_3d**2))**0.5
# print(Z_3, Z_3.shape, Z_3.min(), Z_3.max())
# # #############绘图##############
with plt.style.context('ggplot'):
fig = plt.figure()
ax = Axes3D(fig)
u = np.linspace(0, 2 * np.pi, 100)
v = np.linspace(0, np.pi, 100)
x_3 = a_3d * np.outer(np.cos(u), np.sin(v))
y_3 = b_3d * np.outer(np.sin(u), np.sin(v))
z_3 = c_3d * np.outer(np.ones(np.size(u)), np.cos(v))
ax.plot_surface(x_3, y_3, z_3, cmap=cm.gray)
ax.scatter(contours_points[:, :, 0].ravel(), contours_points[:, :, 1].ravel(), Z_3.ravel(), s=1, c='k', cmap='coolwarm')
ax.axis('off')
fig.savefig('F:/wangqianwen/transient/'+str(dir_ind)+"_"+str(ctr_o)+"_"+str(bgt_o)+'3d.png')
fig2 = plt.figure()
plt.scatter(contours_points[:, :, 0].ravel(), contours_points[:, :, 1].ravel(), s=1)
fig2.savefig('F:/wangqianwen/transient/'+str(dir_ind)+"_"+str(ctr_o)+"_"+str(bgt_o)+'3d_Re.png')
fig3 = plt.figure()
plt.scatter(cell_info(contours_out)[0], cell_info(contours_out)[1], s=1)
fig3.savefig('F:/wangqianwen/transient/' + str(dir_ind) + "_" + str(ctr_o) + "_" + str(bgt_o) + 'area-length.png')
def plot_CNMF_results(pathMouse,session,extent):
f = loadmat(pathMouse + '/matching/results_matching_multi_std=0_thr=70_w=33_OnACID.mat')
assignments = f['assignments']
path = pathMouse + 'Session%02d/results_OnACID.mat'%session[0]
f = loadmat(path);
Cn0 = f['Cn']
for s in range(session[0],session[1]+1):
path = pathMouse + 'Session%02d/results_OnACID.mat'%s
f = loadmat(path);
dims = f['Cn'].shape
d1,d2 = dims
A = f['A']
if 'ndarray' not in str(type(extent)):
extent = np.array(extent)
Coor = np.matrix([np.outer(np.ones(d2), np.arange(d1)).ravel(),
np.outer(np.arange(d2), np.ones(d1)).ravel()], dtype=A.dtype)
Anorm = scipy.sparse.vstack([a/a.sum() for a in A.T]).T
cm = np.array((Coor * Anorm).T)
cmap='viridis';
level = 0.98
fig = pl.figure(figsize=(2.5,2.5),frameon=False)
ax = fig.add_axes([0, 0, 1, 1])
pl.rcParams['pdf.fonttype'] = 42
font = {'family': 'Myriad Pro',
'weight': 'regular',
'size': 10}
pl.rc('font', **font)
lp, hp = np.nanpercentile(f['Cn'], [5, 95])
ax.imshow(f['Cn'], vmin=lp, vmax=hp, cmap=cmap)
C = np.fft.fftshift(np.real(np.fft.ifft2(np.fft.fft2(Cn0) * np.fft.fft2(np.rot90(f['Cn'],2)))))
max_pos = np.where(C==np.max(C))
x_shift = (max_pos[0] - (dims[0]/2-1)).astype(int)
y_shift = (max_pos[1] - (dims[1]/2-1)).astype(int)
#also load assignments and distinguish between matched and non-matched (for those sessions) - plot in red (matched) vs white (non-matched)
for n in range(f['A'].shape[1]):
if (cm[n,:] > extent[:,0]-10).all() and (cm[n,:] < extent[:,1]+10).all():
c = np.where(assignments[:,s-1]==n)[0]
if c:
nMatch = sum(sum(~np.isnan(assignments[c,session[0]-1:session[1]])));
if nMatch == 3:
#print(c)
#print(assignments[c,session[0]-1:session[1]])
col = 'r'
lw = 2
elif nMatch == 2:
col = 'y'
lw = 1
else:
col = 'r'
lw = 0.8
ax.contour(norm_nrg(np.reshape(f['A'][:,n].transpose().toarray(),dims,order='F')), levels=[level], colors=col, linewidths=lw)
#[pl.contour(norm_nrg(np.reshape(mm.toarray(),dims,order='F')), levels=[level], colors=col, linewidths=1) for mm in f['A'][:,:500].transpose()]
#if s == session[0]:
text_str = "Session %02d"%s
y_pos = extent[0,1]+y_shift-7
#else:
#text_str = "Session %02d \nShift: (%d,%d)"%(s,x_shift,y_shift)
#y_pos = extent[0,1]+y_shift-13
#ax.plot([210,230],[60,70],'k','LineWidth',5)
ax.plot([extent[1,0],extent[1,0]]+x_shift,[extent[0,0],extent[0,1]]+y_shift,'k',Linewidth=12)
ax.plot([extent[1,1],extent[1,1]]+x_shift,[extent[0,0],extent[0,1]]+y_shift,'k',LineWidth=12)
ax.plot([extent[1,0],extent[1,1]]+x_shift,[extent[0,0],extent[0,0]]+y_shift,'k',LineWidth=12)
ax.plot([extent[1,0],extent[1,1]]+x_shift,[extent[0,1],extent[0,1]]+y_shift,'k',LineWidth=12)
ax.text(extent[1,0]+x_shift+4,y_pos,text_str,fontsize=16,bbox=dict(facecolor='w', alpha=0.8))
ax.set_xlim(extent[1,:]+x_shift)
ax.set_ylim(extent[0,:]+y_shift)
#pl.title('Matches')
#ax.axis('off')
#pl.draw()
pl.show(block=False)
pl.savefig("/media/wollex/Analyze_AS3/Data/879/Figures/ROIs_s=%02d.png"%s)
def inner_point(X, y, max_iter=5000):
m, n = X.shape
X = np.column_stack((X, np.ones((m, 1))))
y = y.astype(np.float64)
data_num = len(y)
C = 1.0
kernel = np.dot(
X, np.transpose(X)) + np.diag(np.ones(data_num, np.float64)) * .5 / C
p = np.matrix(np.multiply(kernel, np.outer(y, y)))
q = np.matrix(-np.ones([data_num, 1], np.float64))
bounds = (0, np.inf)
low, up = bounds
x = np.random.normal(size=(m, 1))
l = 0.001
for k in range(max_iter * 5): # heavy on matrix operations
g0 = p * x + q
# saving previous x
x = x - l * g0
x[x < low] = low
x[x > up] = up
dual = -(0.5 * x.T * (p * x) + q.T * x)
dual = dual.item()
y1 = np.reshape(y, (-1, 1))
lambda1 = np.multiply(x, y1)
w = np.dot(X.T, lambda1)
w = np.matrix(w).reshape(-1, 1)
tmp = np.maximum(1 - np.multiply(y1, X * w), 0)
primal = 0.5 * np.linalg.norm(w)**2 + 1 * np.sum(tmp)
primal = primal.item()
#if k % 1000 == 0:
# print('GD:', np.abs(dual - primal) / (1 + np.abs(dual) + np.abs(primal)))
for k in range(500): # heavy on matrix operations
for i in range(m):
tmpx = x.copy()
tmpx[i, 0] = 0
temp = (p[i, :] * tmpx) + q[i]
# if temp > 0 and x[i] == 0:
# continue
temp = temp.item()
if p[i, i] > 0:
xi = -(temp / p[i, i]).item()
xi = np.maximum(low, xi)
elif p[i, i] < 0:
#----bug----
#xi = -1
xi = +1
#print('error')
else:
if temp > 0:
xi = low
x[i, 0] = xi
# for u in range(m):
# i = -1-u
# tmpx = x.copy()
# tmpx[i, 0] = 0
# temp = (p[i, :] * tmpx) + q[i]
# # if temp > 0 and x[i] == 0:
# # continue
# temp = temp.item()
# if p[i, i] > 0:
# xi = -(temp / p[i, i]).item()
# xi = np.maximum(low, xi)
# xi = np.minimum(up, xi)
# elif p[i, i] < 0:
# print('error')
# else:
# if temp > 0:
# xi = low
# x[i, 0] = xi
dual = -(0.5 * x.T * (p * x) + q.T * x)
dual = dual.item()
y1 = np.reshape(y, (-1, 1))
lambda1 = np.multiply(x, y1)
w = np.dot(X.T, lambda1)
w = np.matrix(w).reshape(-1, 1)
tmp = np.maximum(1 - np.multiply(y1, X * w), 0)
primal = 0.5 * np.linalg.norm(w)**2 + 1 * np.sum(np.square(tmp))
primal = primal.item()
# stop criteria
#if k % 1000 == 0:
# print('CD:', np.abs(dual - primal) / (1 + np.abs(dual) + np.abs(primal)))
# print(np.abs(dual - primal) / (1 + np.abs(dual) + np.abs(primal)))
if np.abs(dual - primal) / (1 + np.abs(dual) + np.abs(primal)) < 1e-12:
#print('success')
break
return w
def printObstacleObbSphere(name, ax):
print(name)
file_obj = open("E:\\graduateDesignTxt\\点云\\" + name + ".txt",
encoding='UTF-8')
x = []
y = []
z = []
for line in file_obj.readlines():
line = line.rstrip("\n")
arr = line.split(",")
x.append(float(arr[0]))
y.append(float(arr[1]))
z.append(float(arr[2]))
size = len(x)
points = numpy.zeros((size, 3))
for i in range(size):
points[i][0] = x[i]
points[i][1] = y[i]
points[i][2] = z[i]
maxL = -100000
maxW = -100000
maxH = -100000
minL = 100000
minW = 100000
minH = 100000
for point in points:
maxL = max(maxL, point[0])
minL = min(minL, point[0])
maxW = max(maxW, point[1])
minW = min(minW, point[1])
maxH = max(maxH, point[2])
minH = min(minH, point[2])
pointA = [(maxL + minL) / 2, (maxW + minW) / 2, (maxH + minH) / 2]
halfLengthA = [(maxL - minL) / 2, (maxW - minW) / 2, (maxH - minH) / 2]
# vector = [[1, 0, 0], [0, 1, 0], [0, 0, 1]]
# printOct(halfLengthA, pointA, vector, ax)
r = 0
for point in points:
r = max(
r,
math.sqrt(
pow(pointA[0] - point[0], 2) + pow(pointA[1] - point[1], 2) +
pow(pointA[2] - point[2], 2)))
# s = sphere(pos=(pointA))
# s.radius=r
# ax.plot(s)
center = pointA
radius = r
print(2 * 3.14 * r * r)
# data
u = np.linspace(0, 2 * np.pi, 100)
v = np.linspace(0, np.pi, 100)
x1 = radius * np.outer(np.cos(u), np.sin(v)) + center[0]
y1 = radius * np.outer(np.sin(u), np.sin(v)) + center[1]
z1 = radius * np.outer(np.ones(np.size(u)), np.cos(v)) + center[2]
# plot
# surface plot
# wire frame
ax.plot_wireframe(x1, y1, z1, rstride=10, cstride=10)
ax.scatter(x, y, z, s=1)
def reset(self):
"""Reset the environment to its original state. Must be called before the first step.
Returns
-------
users : OrderedDict
The initial users where the key represents the user id and the value represents
the visible features associated with the user.
items : OrderedDict
The initial items where the key represents the item id and the value represents
the visible features associated with the item.
ratings : dict
The initial ratings where the key is a double whose first element is the user id
and the second element is the item id. The value represents the features associated
with the setting in which the rating was made.
"""
# Initialize the state of the environment.
self._timestep = -1
self._reset_state()
self._user_histories = collections.defaultdict(list)
num_users = len(self._users)
num_items = len(self._items)
self._user_prob = self._get_user_prob()
# We will lazily compute dense ratings.
self._dense_ratings = None
# Sample initial observed user-item pairs.
if isinstance(self._initial_sampling, str):
item_idx = np.random.permutation(num_items)
if self._initial_sampling == "uniform":
item_probs = np.ones(num_items) / num_items
elif self._initial_sampling == "powerlaw":
# Sample according to a powerlaw-like beta distribution,
# parameters were fit to MovieLens 100k.
item_probs = scipy.stats.beta.pdf(item_idx,
a=0.75431,
b=3.22225,
loc=-1,
scale=num_items + 1)
# Normalize the item probabilities to convert from continuous to discrete distributions.
item_probs /= item_probs.sum()
idx_1d = self._init_random.choice(
num_users * num_items,
self._num_init_ratings,
replace=False,
p=np.outer(self._user_prob, item_probs).flatten(),
)
user_ids = idx_1d // num_items
item_ids = idx_1d % num_items
else:
if len(self._initial_sampling) > 0:
user_ids, item_ids = zip(*self._initial_sampling)
else:
user_ids, item_ids = [], []
user_ids, item_ids = np.array(user_ids), np.array(item_ids)
# Fill the rating dict with initial data.
self._ratings = {}
for user_id, item_id in zip(user_ids, item_ids):
# TODO: This is a hack, but I don't think we should necessarily put the burden
# of having to implement a version of _rate_item that knows whether it's being called
# in reset or not on people deriving from this class. Need to think of a better way
# than doing this though.
temp_random = self._dynamics_random
self._dynamics_random = self._init_random
self._ratings[user_id, item_id] = (
self._rate_items(user_id, np.array([item_id]))[0],
self._rating_context(user_id),
)
self._dynamics_random = temp_random
# Finally, set the users that will be online for the first step.
self._online_users = self._select_online_users()
self._timestep += 1
return self._users.copy(), self._items.copy(), self._ratings.copy()
def parallel_axis_theorem(I, m, R):
return I + m * (np.dot(R, R) * np.eye(3) - np.outer(R, R))
def track(self, img):
w_z = self.size[0] + CONTEXT_AMOUNT * np.sum(self.size)
h_z = self.size[1] + CONTEXT_AMOUNT * np.sum(self.size)
s_z = np.sqrt(w_z * h_z)
scale_z = EXEMPLAR_SIZE / s_z
score_size = (INSTANCE_SIZE - EXEMPLAR_SIZE) // ANCHOR_STRIDE + 1 + BASE_SIZE
hanning = np.hanning(score_size)
window = np.outer(hanning, hanning)
window = np.tile(window.flatten(), self.anchor_num)
anchors = self.generate_anchor(score_size)
s_x = s_z * (INSTANCE_SIZE / EXEMPLAR_SIZE)
x_crop = self.get_subwindow(img, self.center_pos,
INSTANCE_SIZE,
round(s_x), self.channel_average)
outputs = self.model.track(x_crop)
score = self._convert_score(outputs['cls'])
pred_bbox = self._convert_bbox(outputs['loc'], anchors)
def change(r):
return np.maximum(r, 1. / r)
def sz(w, h):
pad = (w + h) * 0.5
return np.sqrt((w + pad) * (h + pad))
# scale penalty
s_c = change(sz(pred_bbox[2, :], pred_bbox[3, :]) /
(sz(self.size[0]*scale_z, self.size[1]*scale_z)))
# aspect ratio penalty
r_c = change((self.size[0]/self.size[1]) /
(pred_bbox[2, :]/pred_bbox[3, :]))
penalty = np.exp(-(r_c * s_c - 1) * PENALTY_K)
pscore = penalty * score
# window penalty
pscore = pscore * (1 - WINDOW_INFLUENCE) + \
window * WINDOW_INFLUENCE
# get 'best_score' and the most important 'scores' and 'boxes' and 'lr'
best_idx = np.argmax(pscore)
best_score = pscore[best_idx]
best_idx16 = np.argsort(pscore)[::-1][:16]
best_idx16 = best_idx16[pscore[best_idx16] > pscore[best_idx]*0.95].tolist()
bbox = pred_bbox[:, best_idx16] / scale_z
lr = penalty[best_idx16] * score[best_idx16] * LR
# get position and size
if best_score >= 0.65:
cx = bbox[0,0] + self.center_pos[0]
cy = bbox[1,0] + self.center_pos[1]
width = self.size[0] * (1 - lr[0]) + bbox[2,0] * lr[0]
height = self.size[1] * (1 - lr[0]) + bbox[3,0] * lr[0]
self.cx16 = bbox[0,:] + self.center_pos[0]
self.cy16 = bbox[1,:] + self.center_pos[1]
self.width16 = self.size[0] * (1 - lr) + bbox[2,:] * lr
self.height16 = self.size[1] * (1 - lr) + bbox[3,:] * lr
else:
cx = self.center_pos[0]
cy = self.center_pos[1]
width = self.size[0]
height = self.size[1]
self.cx16 = np.array([cx])
self.cy16 = np.array([cy])
self.width16 = np.array([width])
self.height16 = np.array([height])
# clip boundary
cx, cy, width, height = self._bbox_clip(cx, cy, width, height, img.shape[:2])
# udpate state
self.center_pos = np.array([cx, cy])
self.size = np.array([width, height])
bbox = [cx - width / 2,
cy - height / 2,
width,
height]
bbox16 = [self.cx16 - self.width16 / 2,
self.cy16 - self.height16 / 2,
self.width16,
self.height16]
return {
'bbox': bbox,
'bbox16': bbox16,
'best_score': best_score,
}
