def is_palindrome(x):
"""
is palindrome
:param x:
:return:
"""
if x < 0:
return False
if x < 10:
return True
# return x == int(str(x)[::-1])
length = len(str(x))
latter = ""
temp = x
half_len = length // 2
for i in range(half_len):
latter += str(temp % 10)
temp = temp // 10
if length % 2 == 0:
# if x = 1234554321, then return x = (1234554321 // 10000)
x = x // math.pow(10, half_len)
else:
# if x = 12345654321, then return x = (12345654321 // 100000)
x = x // math.pow(10, half_len + 1)
return str(int(x)) == latter
def DPLSW(self, thetaTild, countXVec, myMDP, featuresMatrix, gamma, epsilon, delta,batchSize, initStateDist="uniform", pi="uniform"):
dim=len(featuresMatrix.T)
alpha=(5.0*numpy.sqrt(2*numpy.math.log(2.0/delta)))/epsilon
beta= (epsilon/4)/(dim+numpy.math.log(2.0/delta))
Gamma_w=myMDP.getGammaMatrix()
for i in range(len(countXVec)):
Gamma_w[i][i]=Gamma_w[i][i]*countXVec[i]/batchSize
GammaSqrt= (Gamma_w)
for i in range(len(GammaSqrt)):
GammaSqrt[i][i]=math.sqrt(Gamma_w[i][i])
GammaSqrtPhi= numpy.mat(GammaSqrt) *numpy.mat(featuresMatrix)
if self.is_invertible(GammaSqrtPhi):
GammaSqrtPhiInv=linalg.inv(GammaSqrtPhi)
else:
GammaSqrtPhiInv=linalg.pinv(GammaSqrtPhi)
PsiBetaX= self.SmootBound_LSW(myMDP, Gamma_w, countXVec, beta, myMDP.startStateDistribution())
sigmmaX= (alpha*myMDP.getMaxReward())/(1-gamma)
sigmmaX=sigmmaX*numpy.linalg.norm(GammaSqrtPhiInv)
sigmmaX=sigmmaX*math.pow(PsiBetaX, .5)
cov_X=math.pow(sigmmaX,2)*numpy.identity(dim)
mean=numpy.zeros(dim)
ethaX=numpy.random.multivariate_normal(mean,cov_X)
thetaTild=numpy.squeeze(numpy.asarray(thetaTild))
ethaX=numpy.squeeze(numpy.asarray(ethaX))
thetaTild_priv=thetaTild+ethaX
return [thetaTild_priv,thetaTild,math.pow(sigmmaX,2)]
def update_creature_properties(self, creature: Creature, creature_actions: CreatureActions) -> None:
"""
Updates the creature properties according to its actions.
"""
# The more the creature moves, the higher its fitness.
distance = math.sqrt(math.pow(creature_actions.x, 2) + math.pow(creature_actions.y, 2))
creature.fitness += distance
creature.distance_travelled += distance
creature.age += 1
if int(creature.distance_travelled) % 30 == 0:
creature.age -= 5
if creature.age >= MAX_AGE:
self.dead_creatures.append(creature)
def computeEligibilityVector(self, lambdaCoef, gamma, index,dim,featureMatrix):
z_i= numpy.zeros((1,dim))
for j in range(index):
phi_j=featureMatrix[j][:]
temp=math.pow(gamma*lambdaCoef, index-j)
z_i+=temp*(phi_j)
return z_i
def LSL_subSampleAggregate(self, batch, s, numberOfsubSamples,myMDP,featuresMatrix,regCoef, pow_exp,numTrajectories,epsilon,delta,distUB):
dim=len(featuresMatrix.T)
alpha=(5.0*numpy.sqrt(2*numpy.math.log(2.0/delta)))/epsilon
#beta= (epsilon/4)*(dim+numpy.math.log(2.0/delta))
beta= (s/(2*numberOfsubSamples))
subSamples=self.subSampleGen(batch, numberOfsubSamples)
z=numpy.zeros((len(subSamples),dim))
for i in range(len(subSamples)):
FVMC=self.FVMCPE(myMDP, featuresMatrix, subSamples[i])
#regc=self.computeLambdas(myMDP, featuresMatrix,regCoef, len(subSamples[i]), pow_exp)
#z[i]=numpy.ravel(self.LSL(FVMC[2], myMDP, featuresMatrix,regc[0][0],len(subSamples[i]),FVMC[1]))
SA_reidge_coef=100*math.pow(len(subSamples[i]), 0.5)
z[i]=numpy.ravel(self.LSL(FVMC[2], myMDP, featuresMatrix,SA_reidge_coef,len(subSamples[i]),FVMC[1]))
#z[i]= numpy.squeeze(numpy.asarray(FVMC[0]))#this is LSW
partitionPoint=int((numberOfsubSamples+math.sqrt(numberOfsubSamples))/2)
g= self.generalized_median(myMDP,z,partitionPoint,distUB)
#g= self.aggregate_median(myMDP,z)
#To check the following block
S_z=self.computeAggregateSmoothBound(z, beta, s,myMDP,distUB)
#print(S_z)
cov_X=(S_z/alpha)*numpy.identity((dim))
ethaX=numpy.random.multivariate_normal(numpy.zeros(dim),cov_X)
#print(S_z)
#noise=(S_z/alpha)*ethaX
#numpy.mat(featuresMatrix)*numpy.mat(g[1]+ethaX)
temp_priv=numpy.mat(featuresMatrix)*numpy.mat(g[1]+ethaX).T
return [temp_priv, numpy.mat(featuresMatrix)*numpy.mat(g[1]).T]
def iswt(coefficients, wavelet):
"""
Input parameters:
coefficients
approx and detail coefficients, arranged in level value
exactly as output from swt:
e.g. [(cA1, cD1), (cA2, cD2), ..., (cAn, cDn)]
wavelet
Either the name of a wavelet or a Wavelet object
"""
output = coefficients[0][0].copy() # Avoid modification of input data
#num_levels, equivalent to the decomposition level, n
num_levels = len(coefficients)
for j in range(num_levels,0,-1):
step_size = int(math.pow(2, j-1))
last_index = step_size
_, cD = coefficients[num_levels - j]
for first in range(last_index): # 0 to last_index - 1
# Getting the indices that we will transform
indices = arange(first, len(cD), step_size)
# select the even indices
even_indices = indices[0::2]
# select the odd indices
odd_indices = indices[1::2]
# perform the inverse dwt on the selected indices,
# making sure to use periodic boundary conditions
x1 = pywt.idwt(output[even_indices], cD[even_indices],wavelet, 'per')
x2 = pywt.idwt(output[odd_indices], cD[odd_indices],wavelet, 'per')
# perform a circular shift right
x2 = roll(x2, 1)
# average and insert into the correct indices
output[indices] = (x1 + x2)/2.
return output
def computeEligibilityVector(self, lambdaCoef, gamma, index, dim,featureMatrix):
z_i= numpy.zeros((1,dim))
for j in range(index):
phi_j=featureMatrix[j][:]
temp=math.pow(gamma*lambdaCoef, index-j)
z_i+=temp*(phi_j)
return z_i
def isSame(cur, now):
bool = True
for i in range(0, len(cur)):
if abs(cur[i] - now[i]) > math.pow(10, -3):
bool = False
break
return bool
def getFitness(self, model):
scores = []
for i in range(0, 1):
scores.append(
model.score(self._dataSet.getXTest(),
self._dataSet.getYTest()))
return copy(math.pow(100, mean(scores)))
def iswt(coefficients, wavelet):
"""
Input parameters:
coefficients
approx and detail coefficients, arranged in level value
exactly as output from swt:
e.g. [(cA1, cD1), (cA2, cD2), ..., (cAn, cDn)]
wavelet
Either the name of a wavelet or a Wavelet object
"""
output = coefficients[0][0].copy() # Avoid modification of input data
#num_levels, equivalent to the decomposition level, n
num_levels = len(coefficients)
for j in range(num_levels, 0, -1):
step_size = int(math.pow(2, j - 1))
last_index = step_size
_, cD = coefficients[num_levels - j]
for first in range(last_index): # 0 to last_index - 1
# Getting the indices that we will transform
indices = arange(first, len(cD), step_size)
# select the even indices
even_indices = indices[0::2]
# select the odd indices
odd_indices = indices[1::2]
# perform the inverse dwt on the selected indices,
# making sure to use periodic boundary conditions
x1 = pywt.idwt(output[even_indices], cD[even_indices], wavelet,
'per')
x2 = pywt.idwt(output[odd_indices], cD[odd_indices], wavelet,
'per')
# perform a circular shift right
x2 = roll(x2, 1)
# average and insert into the correct indices
output[indices] = (x1 + x2) / 2.
return output
def crossfade(wave, percent=.1):
n = int(len(wave) * percent / 2)
weight = lambda i: math.pow(i, 2)
i_max = weight(n)
for i in range(n):
amp = weight(i) / i_max
wave[i] *= amp
wave[-(i + 1)] *= amp
def DPLSW(self, FirstVisitVector, countXVec, myMDP, featuresMatrix, gamma, epsilon, delta,batchSize, initStateDist="uniform", pi="uniform"):
dim = len(featuresMatrix.T)
alpha = (5.0*numpy.sqrt(2*numpy.math.log(2.0/delta)))/epsilon
beta = (epsilon/4)/(dim+numpy.math.log(2.0/delta))
Gamma_w = myMDP.getGammaMatrix()
# for i in range(len(countXVec)):
# Gamma_w[i][i] = Gamma_w[i][i]*countXVec[i]/batchSize
GammaSqrt = Gamma_w
for i in range(len(GammaSqrt)):
GammaSqrt[i][i] = math.sqrt(Gamma_w[i][i])
GammaSqrt = GammaSqrt * numpy.mat(featuresMatrix)
if self.is_invertible(GammaSqrt):
GammaSqrtPhiInv = linalg.inv(GammaSqrt)
else:
GammaSqrtPhiInv = linalg.pinv(GammaSqrt)
GammaTemp = numpy.mat(featuresMatrix).T * Gamma_w * numpy.mat(featuresMatrix)
#GammaSqrtPhi = numpy.mat(GammaSqrt) * numpy.mat(featuresMatrix)
if self.is_invertible(GammaTemp):
GammaTempPhiInv = linalg.inv(GammaTemp)
else:
GammaTempPhiInv = linalg.pinv(GammaTemp)
FirstVisitVector = numpy.reshape(FirstVisitVector, (len(FirstVisitVector), 1))
thetaTild = GammaTempPhiInv * numpy.mat(featuresMatrix.T)
thetaTild = thetaTild * numpy.mat(Gamma_w) * numpy.mat(FirstVisitVector)
PsiBetaX = self.SmootBound_LSW(myMDP, Gamma_w, countXVec, beta, myMDP.startStateDistribution())
sigmmaX = (alpha*myMDP.getMaxReward())/(1-self.gamma_factor)
sigmmaX = sigmmaX*numpy.linalg.norm(GammaSqrtPhiInv,2)
sigmmaX = sigmmaX*math.pow(PsiBetaX, 0.5)
cov_X = math.pow(sigmmaX,2)*numpy.identity(dim)
mean = numpy.zeros(dim)
ethaX = numpy.random.multivariate_normal(mean, cov_X)
thetaTild = numpy.squeeze(numpy.asarray(thetaTild))
ethaX = numpy.squeeze(numpy.asarray(ethaX))
thetaTild_priv = thetaTild + ethaX
return [thetaTild_priv, thetaTild, math.pow(sigmmaX,2)]
def get_retention(t):
# c = 1.8
# d = 1.21
c = average([1.8, 1.34, 0.9, 1.36])
d = average([1.21, 0.873, 0.9, 1.36])
print(c, d)
if t <= 1.0:
return 1.0
innr = math.pow(math.log(t, 10.0), d)
return c / (innr + c)
def mse(x, locations, distances):
# This function calculates the mean squared error of the reference points
# and the point x
mse = 0.0
for location, distance in zip(locations, distances):
distance_calculated = great_circle_distance(x[0], x[1], location[0],
location[1])
mse += math.pow(distance_calculated - distance, 2.0)
return mse / 3
def GS_based_DPLSL (self, FirstVisitVector, countXVec, myMDP, featuresMatrix, gamma, epsilon, delta, regCoef, numTrajectories, rho, pi="uniform"):
dim=len(featuresMatrix.T)
Rho=numpy.reshape(rho,(len(rho),1))
thetaTil_X= self.LSL(FirstVisitVector, myMDP, featuresMatrix, regCoef, numTrajectories,countXVec)
normPhi=numpy.linalg.norm(featuresMatrix)
maxRho=numpy.linalg.norm(Rho,Inf)
l2Rho=numpy.linalg.norm(Rho)
alpha=(5.0*numpy.sqrt(2*numpy.math.log(2.0/delta)))/epsilon
sigma_X=float(2*alpha*myMDP.getMaxReward()*normPhi/((1-myMDP.getGamma())*(regCoef-maxRho*numpy.math.pow(normPhi, 2))))
varphi_lamda=normPhi*maxRho*math.sqrt(numTrajectories)/(math.sqrt(2*regCoef))+l2Rho
sigma_X=sigma_X*varphi_lamda
cov_X=math.pow(sigma_X,2)*numpy.identity(dim)
mean=numpy.zeros(dim)
ethaX=numpy.random.multivariate_normal(mean,cov_X)
#thetaTil_X=numpy.squeeze(numpy.asarray(thetaTil_X))
#ethaX=numpy.squeeze(numpy.asarray(ethaX))
thetaTil_X_priv=thetaTil_X+numpy.reshape(ethaX, (dim,1))
return [thetaTil_X_priv, thetaTil_X,math.pow(sigma_X,2)]
def GS_based_DPLSL (self, FirstVisitVector, countXVec, myMDP, featuresMatrix, gamma, epsilon, delta, regCoef, numTrajectories, rho, pi="uniform"):
dim=len(featuresMatrix.T)
Rho=numpy.reshape(rho,(len(rho),1))
thetaTil_X= self.LSL(FirstVisitVector, myMDP, featuresMatrix, regCoef, numTrajectories,countXVec)
normPhi=numpy.linalg.norm(featuresMatrix)
maxRho=numpy.linalg.norm(Rho,Inf)
l2Rho=numpy.linalg.norm(Rho)
alpha=(5.0*numpy.sqrt(2*numpy.math.log(2.0/delta)))/epsilon
sigma_X=float(2*alpha*myMDP.getMaxReward()*normPhi/((1-myMDP.getGamma())*(regCoef-maxRho*numpy.math.pow(normPhi, 2))))
varphi_lamda=normPhi*maxRho*math.sqrt(numTrajectories)/(math.sqrt(2*regCoef))+l2Rho
sigma_X=sigma_X*varphi_lamda
cov_X=math.pow(sigma_X,2)*numpy.identity(dim)
mean=numpy.zeros(dim)
ethaX=numpy.random.multivariate_normal(mean,cov_X)
#thetaTil_X=numpy.squeeze(numpy.asarray(thetaTil_X))
#ethaX=numpy.squeeze(numpy.asarray(ethaX))
thetaTil_X_priv=thetaTil_X+numpy.reshape(ethaX, (dim,1))
return [thetaTil_X_priv, thetaTil_X,math.pow(sigma_X,2)]
def getFitnessFold(self, individual):
kf = KFold(n_splits=10)
X = np.array(self._dataSet.getX())
y = np.array(self._dataSet.getY())
parameter = individual.getParam()
model = self._estimator.set_params(**parameter)
scores = []
for train_index, test_index in kf.split(X):
# print("TRAIN:", train_index, "TEST:", test_index)
X_train, X_test = X[train_index], X[test_index]
y_train, y_test = y[train_index], y[test_index]
model.fit(X_train, y_train)
scores.append(model.score(X_test, y_test))
return copy(math.pow(100, mean(scores)))
def DPLSL (self, LSL_Vector, countXVec, myMDP, featuresMatrix, gamma, epsilon, delta, regCoef, numTrajectories, rho, pi="uniform"):
regCoef=regCoef
dim=len(featuresMatrix.T)
Rho=numpy.reshape(rho,(len(rho),1))
thetaTil_X= LSL_Vector #self.LSL(FirstVisitVector, myMDP, featuresMatrix, regCoef, numTrajectories,countXVec)
normPhi=numpy.linalg.norm(featuresMatrix)
maxRho=numpy.linalg.norm(Rho,Inf)
alpha=(5.0*numpy.sqrt(2*numpy.math.log(2.0/delta)))/epsilon
beta= (epsilon/4)/(dim+numpy.math.log(2.0/delta))
PsiBetaX = self.SmoothBound_LSL(featuresMatrix, myMDP, countXVec, myMDP.startStateDistribution(), regCoef, beta, numTrajectories)
sigma_X=2*alpha*myMDP.getMaxReward()*normPhi/(1-myMDP.getGamma())
sigma_X=sigma_X/(regCoef-maxRho*numpy.math.pow(normPhi, 2))
sigma_X=sigma_X*(numpy.math.pow(PsiBetaX,0.5))
#print(sigma_X)
cov_X=math.pow(sigma_X,2)*numpy.identity(dim)
mean=numpy.zeros(dim)
ethaX=numpy.random.multivariate_normal(mean,cov_X)
ethaX=numpy.reshape(ethaX,(len(ethaX),1))
thetaTil_X_priv=thetaTil_X+ethaX
return [thetaTil_X_priv, thetaTil_X,math.pow(sigma_X,2)]
def DPLSL(self, LSL_Vector, countXVec, myMDP, featuresMatrix, gamma, epsilon, delta, regCoef, numTrajectories, rho, pi="uniform"):
regCoef = regCoef
dim = len(featuresMatrix.T)
Rho = numpy.reshape(rho , (len(rho),1))
thetaTil_X = LSL_Vector #self.LSL(FirstVisitVector, myMDP, featuresMatrix, regCoef, numTrajectories,countXVec)
normPhi = numpy.linalg.norm(featuresMatrix, 2)
maxRho = numpy.linalg.norm(Rho, Inf)
alpha = (5.0*numpy.sqrt(2*numpy.math.log(2.0/delta)))/epsilon
beta = (epsilon/4)/(dim+numpy.math.log(2.0/delta))
PsiBetaX = self.SmoothBound_LSL(featuresMatrix, myMDP, countXVec, myMDP.startStateDistribution(), regCoef, beta, numTrajectories)
sigma_X = 2*alpha*myMDP.getMaxReward()*normPhi/(1-myMDP.getGamma())
sigma_X = sigma_X/(regCoef-maxRho*numpy.math.pow(normPhi, 2))
sigma_X = sigma_X*(numpy.math.pow(PsiBetaX,0.5))
#print(sigma_X)
cov_X = math.pow(sigma_X,2)*numpy.identity(dim)
mean = numpy.zeros(dim)
ethaX = numpy.random.multivariate_normal(mean,cov_X)
ethaX = numpy.reshape(ethaX,(len(ethaX),1))
thetaTil_X_priv = thetaTil_X + ethaX
return [thetaTil_X_priv.reshape((dim,1)), thetaTil_X, math.pow(sigma_X,2)]
def SmoothBound_LSL(self,featurmatrix, myMDP, countXVec, rho, regCoef,beta,numTrajectories):
normPhi = numpy.linalg.norm(featurmatrix,2)
maxRho = numpy.linalg.norm(rho,Inf)
c_lambda = normPhi*maxRho/(math.sqrt(2*regCoef))
#print('===============================================')
#print(regCoef)
l2Rho = numpy.linalg.norm(rho)
phi_k = 0
Vals = []
for k in range(0, numTrajectories+1):
minVal=0
for s in range(len(myMDP.getStateSpace())):
minVal = minVal+rho[s] * min(countXVec[s]+k, numTrajectories)
phi_k = c_lambda*math.sqrt(minVal)+l2Rho
Vals.append((math.pow(phi_k,2))*math.exp(-k*beta))
upperBound=max(Vals)
return upperBound
def SmoothBound_LSL(self,featurmatrix, myMDP, countXVec, rho, regCoef,beta,numTrajectories):
normPhi=numpy.linalg.norm(featurmatrix)
maxRho=numpy.linalg.norm(rho,Inf)
c_lambda=normPhi*maxRho/(math.sqrt(2*regCoef))
#print('===============================================')
#print(regCoef)
l2Rho=numpy.linalg.norm(rho)
phi_k=0
Vals=[]
for k in range(0,numTrajectories):
minVal=0
for s in range(len(myMDP.getStateSpace())-1):
minVal=minVal+rho[s]*min(countXVec[s]+k,numTrajectories)
phi_k=c_lambda*math.sqrt(minVal)+l2Rho
Vals.append((math.pow(phi_k,2))*math.exp(-k*beta))
upperBound=max(Vals)
#print('Smooth upper bound= '+str(upperBound))
return upperBound
def computeEligibilities(self, featureMatrix, lambda_coef, trajectory, gamma):
dim=len(featureMatrix)
upBound=len(trajectory)-1
Z=numpy.zeros(shape=(upBound,dim))
temp=numpy.zeros((dim,1))
temp=numpy.reshape(temp,(dim,1))
for i in range(upBound):
temp=numpy.zeros(dim)
temp=numpy.reshape(temp,(dim,1))
for k in range(i):
s_k=trajectory[k][0]
temp2=featureMatrix[s_k,:]
temp2=numpy.reshape(temp2, (dim,1))
#temp2= temp2.T
temp+=math.pow((lambda_coef*gamma),(i-k))*temp2
for k in range(dim):
Z[i,k]=temp[k]
return Z
def lsl_sub_sample_aggregate(self, batch, num_sub_sample_rooted, numberOfsubSamples, myMDP, featuresMatrix,
epsilon, delta, epsilon_star, delta_star, rho, subSampleSize):
dim = len(featuresMatrix.T)
alpha_star = (5.0*numpy.sqrt(2*numpy.math.log(2.0/delta_star)))/epsilon_star
beta_star = (epsilon_star/4)*(dim+numpy.math.log(2.0/delta_star))
alpha = (5.0*numpy.sqrt(2*numpy.math.log(2.0/delta)))/epsilon
beta = (epsilon/4)*(dim+numpy.math.log(2.0/delta))
beta_generalized_median = (num_sub_sample_rooted/(2*numberOfsubSamples))
subSamples = self.subSampleGen(batch, numberOfsubSamples, subSampleSize)
# sub_sampled_lsl_vectors = numpy.zeros((len(subSamples), dim))
# sub_sampled_dplsl_vectors = numpy.zeros((len(subSamples), dim))
sub_sampled_lsl_vectors = numpy.zeros((dim, 1))
sub_sampled_dplsl_vectors = numpy.zeros((dim, 1))
for i in range(len(subSamples)):
FVMC = self.FVMCPE(myMDP, featuresMatrix, subSamples[i])
#regc=self.computeLambdas(myMDP, featuresMatrix,regCoef, len(subSamples[i]), pow_exp)
#z[i]=numpy.ravel(self.LSL(FVMC[2], myMDP, featuresMatrix,regc[0][0],len(subSamples[i]),FVMC[1]))
SA_reidge_coef = 4 * math.pow(len(subSamples[i]), 0.5)
#SA_reidge_coef = 1000 * math.pow(len(subSamples[i]), 0.4) # new setting for the ridge coefficient
lsl_vector = self.LSL(FVMC[2], myMDP, featuresMatrix, SA_reidge_coef, len(subSamples[i]), FVMC[1])
sub_sampled_lsl_vectors += lsl_vector
sub_sampled_dplsl_vectors += self.DPLSL(lsl_vector, FVMC[1], myMDP, featuresMatrix, myMDP.getGamma(),
epsilon, delta, SA_reidge_coef, len(subSamples[i]), rho)[0]
#z[i]= numpy.squeeze(numpy.asarray(FVMC[0]))#this is LSW
# partitionPoint = int((numberOfsubSamples+math.sqrt(numberOfsubSamples))/2)
# g = self.generalized_median(myMDP, sub_sampled_lsl_vectors, partitionPoint, distUB)
#g= self.aggregate_median(myMDP,z)
#To check the following block
# S_z = self.computeAggregateSmoothBound(sub_sampled_lsl_vectors, beta_generalized_median, num_sub_sample_rooted,
# myMDP, distUB)
# cov_X = (S_z/alpha_star)*numpy.identity((dim))
# ethaX = numpy.random.multivariate_normal(numpy.zeros(dim),cov_X)
# temp_priv = numpy.mat(featuresMatrix)*numpy.mat(g[1]+ethaX).T
return sub_sampled_lsl_vectors/numberOfsubSamples, numpy.mat(featuresMatrix)*numpy.mat(sub_sampled_dplsl_vectors/
numberOfsubSamples)
def computeEligibilities(self, featureMatrix, lambda_coef, trajectory, gamma):
dim=len(featureMatrix)
upBound=len(trajectory)-1
Z=numpy.zeros(shape=(upBound,dim))
temp=numpy.zeros((dim,1))
temp=numpy.reshape(temp,(dim,1))
for i in range(upBound):
temp=numpy.zeros(dim)
temp=numpy.reshape(temp,(dim,1))
for k in range(i):
s_k=trajectory[k][0]
temp2=featureMatrix[s_k,:]
temp2=numpy.reshape(temp2, (dim,1))
#temp2= temp2.T
temp+=math.pow((lambda_coef*gamma),(i-k))*temp2
for k in range(dim):
Z[i,k]=temp[k]
return Z
def gelu(x):
return 0.5 * x * (1 + math.tanh(
math.sqrt(2 / math.pi) * (x + 0.044715 * math.pow(x, 3))))
def exact_sol(k,y0,t):
return y0 * m.pow(m.e,-k*t)
def getLineData(pixels, x1, y1, x2, y2, lineW=2, theZ=0, theC=0, theT=0):
"""
Grabs pixel data covering the specified line, and rotates it horizontally
so that x1,y1 is to the left,
Returning a numpy 2d array. Used by Kymograph.py script.
Uses PIL to handle rotating and interpolating the data. Converts to numpy
to PIL and back (may change dtype.)
@param pixels: PixelsWrapper object
@param x1, y1, x2, y2: Coordinates of line
@param lineW: Width of the line we want
@param theZ: Z index within pixels
@param theC: Channel index
@param theT: Time index
"""
from numpy import asarray
sizeX = pixels.getSizeX()
sizeY = pixels.getSizeY()
lineX = x2 - x1
lineY = y2 - y1
rads = math.atan(float(lineX) / lineY)
# How much extra Height do we need, top and bottom?
extraH = abs(math.sin(rads) * lineW)
bottom = int(max(y1, y2) + extraH / 2)
top = int(min(y1, y2) - extraH / 2)
# How much extra width do we need, left and right?
extraW = abs(math.cos(rads) * lineW)
left = int(min(x1, x2) - extraW)
right = int(max(x1, x2) + extraW)
# What's the larger area we need? - Are we outside the image?
pad_left, pad_right, pad_top, pad_bottom = 0, 0, 0, 0
if left < 0:
pad_left = abs(left)
left = 0
x = left
if top < 0:
pad_top = abs(top)
top = 0
y = top
if right > sizeX:
pad_right = right - sizeX
right = sizeX
w = int(right - left)
if bottom > sizeY:
pad_bottom = bottom - sizeY
bottom = sizeY
h = int(bottom - top)
tile = (x, y, w, h)
# get the Tile
plane = pixels.getTile(theZ, theC, theT, tile)
# pad if we wanted a bigger region
if pad_left > 0:
data_h, data_w = plane.shape
pad_data = zeros((data_h, pad_left), dtype=plane.dtype)
plane = hstack((pad_data, plane))
if pad_right > 0:
data_h, data_w = plane.shape
pad_data = zeros((data_h, pad_right), dtype=plane.dtype)
plane = hstack((plane, pad_data))
if pad_top > 0:
data_h, data_w = plane.shape
pad_data = zeros((pad_top, data_w), dtype=plane.dtype)
plane = vstack((pad_data, plane))
if pad_bottom > 0:
data_h, data_w = plane.shape
pad_data = zeros((pad_bottom, data_w), dtype=plane.dtype)
plane = vstack((plane, pad_data))
pil = numpyToImage(plane)
#pil.show()
# Now need to rotate so that x1,y1 is horizontally to the left of x2,y2
toRotate = 90 - math.degrees(rads)
if x1 > x2:
toRotate += 180
# filter=Image.BICUBIC see
# http://www.ncbi.nlm.nih.gov/pmc/articles/PMC2172449/
rotated = pil.rotate(toRotate, expand=True)
#rotated.show()
# finally we need to crop to the length of the line
length = int(math.sqrt(math.pow(lineX, 2) + math.pow(lineY, 2)))
rotW, rotH = rotated.size
cropX = (rotW - length) / 2
cropX2 = cropX + length
cropY = (rotH - lineW) / 2
cropY2 = cropY + lineW
cropped = rotated.crop((cropX, cropY, cropX2, cropY2))
#cropped.show()
return asarray(cropped)
def getLineData(pixels, x1, y1, x2, y2, lineW=2, theZ=0, theC=0, theT=0):
"""
Grabs pixel data covering the specified line, and rotates it horizontally
so that x1,y1 is to the left,
Returning a numpy 2d array. Used by Kymograph.py script.
Uses PIL to handle rotating and interpolating the data. Converts to numpy
to PIL and back (may change dtype.)
@param pixels: PixelsWrapper object
@param x1, y1, x2, y2: Coordinates of line
@param lineW: Width of the line we want
@param theZ: Z index within pixels
@param theC: Channel index
@param theT: Time index
"""
from numpy import asarray
sizeX = pixels.getSizeX()
sizeY = pixels.getSizeY()
lineX = x2-x1
lineY = 1 if y2-y1 == 0 else y2-y1
rads = math.atan(float(lineX) / lineY)
# How much extra Height do we need, top and bottom?
extraH = abs(math.sin(rads) * lineW)
bottom = int(max(y1, y2) + extraH/2)
top = int(min(y1, y2) - extraH/2)
# How much extra width do we need, left and right?
extraW = abs(math.cos(rads) * lineW)
left = int(min(x1, x2) - extraW)
right = int(max(x1, x2) + extraW)
# What's the larger area we need? - Are we outside the image?
pad_left, pad_right, pad_top, pad_bottom = 0, 0, 0, 0
if left < 0:
pad_left = abs(left)
left = 0
x = left
if top < 0:
pad_top = abs(top)
top = 0
y = top
if right > sizeX:
pad_right = right - sizeX
right = sizeX
w = int(right - left)
if bottom > sizeY:
pad_bottom = bottom - sizeY
bottom = sizeY
h = int(bottom - top)
tile = (x, y, w, h)
# get the Tile
plane = pixels.getTile(theZ, theC, theT, tile)
# pad if we wanted a bigger region
if pad_left > 0:
data_h, data_w = plane.shape
pad_data = zeros((data_h, pad_left), dtype=plane.dtype)
plane = hstack((pad_data, plane))
if pad_right > 0:
data_h, data_w = plane.shape
pad_data = zeros((data_h, pad_right), dtype=plane.dtype)
plane = hstack((plane, pad_data))
if pad_top > 0:
data_h, data_w = plane.shape
pad_data = zeros((pad_top, data_w), dtype=plane.dtype)
plane = vstack((pad_data, plane))
if pad_bottom > 0:
data_h, data_w = plane.shape
pad_data = zeros((pad_bottom, data_w), dtype=plane.dtype)
plane = vstack((plane, pad_data))
pil = numpyToImage(plane)
# pil.show()
# Now need to rotate so that x1,y1 is horizontally to the left of x2,y2
toRotate = 90 - math.degrees(rads)
if x1 > x2:
toRotate += 180
# filter=Image.BICUBIC see
# http://www.ncbi.nlm.nih.gov/pmc/articles/PMC2172449/
rotated = pil.rotate(toRotate, expand=True)
# rotated.show()
# finally we need to crop to the length of the line
length = int(math.sqrt(math.pow(lineX, 2) + math.pow(lineY, 2)))
rotW, rotH = rotated.size
cropX = (rotW - length)/2
cropX2 = cropX + length
cropY = (rotH - lineW)/2
cropY2 = cropY + lineW
cropped = rotated.crop((cropX, cropY, cropX2, cropY2))
# cropped.show()
return asarray(cropped)
def lineMagnitude(x1, y1, x2, y2):
lineMagnitude = np.math.sqrt(
np.math.pow((x2 - x1), 2) + math.pow((y2 - y1), 2))
return lineMagnitude
def vinc_dir(f, a, coordinate_a, alpha12, s):
"""
Returns the lat and long of projected point and reverse azimuth
given a reference point and a distance and azimuth to project.
lats, longs and azimuths are passed in decimal degrees
Returns ( phi2, lambda2, alpha21 ) as a tuple
"""
phi1, lambda1 = coordinate_a.lat, coordinate_a.lon
piD4 = math.atan(1.0)
two_pi = piD4 * 8.0
phi1 = phi1 * piD4 / 45.0
lambda1 = lambda1 * piD4 / 45.0
alpha12 = alpha12 * piD4 / 45.0
if alpha12 < 0.0:
alpha12 += two_pi
if alpha12 > two_pi:
alpha12 -= two_pi
b = a * (1.0 - f)
tan_u1 = (1 - f) * math.tan(phi1)
u1 = math.atan(tan_u1)
sigma1 = math.atan2(tan_u1, math.cos(alpha12))
sin_alpha = math.cos(u1) * math.sin(alpha12)
cos_alpha_sq = 1.0 - sin_alpha * sin_alpha
u2 = cos_alpha_sq * (a * a - b * b) / (b * b)
# @todo: look into replacing A and B with vincenty's amendment, see if speed/accuracy is good
A = 1.0 + (u2 / 16384) * (4096 + u2 * (-768 + u2 * (320 - 175 * u2)))
B = (u2 / 1024) * (256 + u2 * (-128 + u2 * (74 - 47 * u2)))
# Starting with the approx
sigma = (s / (b * A))
last_sigma = 2.0 * sigma + 2.0 # something impossible
# Iterate the following 3 eqs unitl no sig change in sigma
# two_sigma_m , delta_sigma
while abs((last_sigma - sigma) / sigma) > 1.0e-9:
two_sigma_m = 2 * sigma1 + sigma
delta_sigma = B * math.sin(sigma) * (math.cos(two_sigma_m) + (B / 4) * (math.cos(sigma) *
(-1 + 2 * math.pow(
math.cos(two_sigma_m), 2) -
(B / 6) * math.cos(two_sigma_m) *
(-3 + 4 * math.pow(math.sin(sigma),
2)) *
(-3 + 4 * math.pow(
math.cos(two_sigma_m), 2)))))
last_sigma = sigma
sigma = (s / (b * A)) + delta_sigma
phi2 = math.atan2((math.sin(u1) * math.cos(sigma) + math.cos(u1) * math.sin(sigma) * math.cos(alpha12)),
((1 - f) * math.sqrt(math.pow(sin_alpha, 2) +
pow(math.sin(u1) * math.sin(sigma) - math.cos(u1) * math.cos(
sigma) * math.cos(alpha12), 2))))
lmbda = math.atan2((math.sin(sigma) * math.sin(alpha12)), (math.cos(u1) * math.cos(sigma) -
math.sin(u1) * math.sin(sigma) * math.cos(alpha12)))
C = (f / 16) * cos_alpha_sq * (4 + f * (4 - 3 * cos_alpha_sq))
omega = lmbda - (1 - C) * f * sin_alpha * (sigma + C * math.sin(sigma) *
(math.cos(two_sigma_m) + C * math.cos(sigma) *
(-1 + 2 * math.pow(math.cos(two_sigma_m), 2))))
lambda2 = lambda1 + omega
alpha21 = math.atan2(sin_alpha, (-math.sin(u1) * math.sin(sigma) +
math.cos(u1) * math.cos(sigma) * math.cos(alpha12)))
alpha21 += two_pi / 2.0
if alpha21 < 0.0:
alpha21 += two_pi
if alpha21 > two_pi:
alpha21 -= two_pi
phi2 = phi2 * 45.0 / piD4
lambda2 = lambda2 * 45.0 / piD4
alpha21 = alpha21 * 45.0 / piD4
return Coordinate(lat=phi2, lon=lambda2), alpha21
def frequency(steps, start=-9, base=440.0):
return base * math.pow(2.0, (start + steps) / 12.0)
def FEM_1order_Solver (preProcData):
print ('Solving...')
timeStart=time.time()
#===============================================================================
# Initial data
#===============================================================================
shapeFunctions=ShapeFuncions()
gradN=shapeFunctions.grad_nod_tri_1order()
mu0=4*np.pi*math.pow(10, -7)
#------------------------------------------------------------------------------
# Pre-processor
meshData=preProcData.MeshData
elemNodes= meshData.ElemNodes
nodesCoordenates=meshData.NodesCoordenates
elemTags=meshData.ElemTags
elemType=meshData.ElemType
n=len(nodesCoordenates)
#------------------------------------------------------------------------------
# Materials library
materialProp=preProcData.MaterialProp
#------------------------------------------------------------------------------
# Setup
regionMaterial=preProcData.RegionMaterial
regionExcitation=preProcData.RegionExcitation
boundary=preProcData.BC
#===============================================================================
#Integration points for Gauss method
#===============================================================================
u=np.array([1.0/6.0,2.0/3.0,1.0/6.0])
v=np.array([1.0/6.0,1.0/6.0,2.0/3.0])
w=1.0/6.0;
qtdNumInted=3
Integdudv=0.5
#===============================================================================
# Global Matrix initialization
#===============================================================================
MatGlobal_esq=np.zeros((n,n))
MatGlobal_dir=np.zeros((n,1))
#===============================================================================
# Main loop over the elements
#===============================================================================
for k in range (0,len(elemTags)):
if elemType[k]==2:
#------------------------------------------------------------------------------
# Element material propriety
prop=0
elemMatProperties=0
for eachRegion in regionMaterial:
if eachRegion.RegionNumber==elemTags[k][0]:
materialName=eachRegion.MaterialName
elemMatProperties=materialProp[materialName].Permeability
break
prop=1.0/(mu0*elemMatProperties)
#------------------------------------------------------------------------------
# Nodes coordinates
nodes=[]
nodes.append(elemNodes[k][0])
nodes.append(elemNodes[k][1])
nodes.append(elemNodes[k][2])
coordJ=np.array([[nodesCoordenates[nodes[0]][0], nodesCoordenates[nodes[0]][1]],
[nodesCoordenates[nodes[1]][0], nodesCoordenates[nodes[1]][1]],
[nodesCoordenates[nodes[2]][0], nodesCoordenates[nodes[2]][1]]])
#------------------------------------------------------------------------------
# Jacobian
# coordJ=coordJ.T
# gradN=gradN.T
operations=Operations()
Jac=operations.get_jacobian_triangle(k,elemNodes,nodesCoordenates)
invJac=np.linalg.inv(Jac)
detJac=np.linalg.det(Jac)
invJacGradN=invJac*gradN
#------------------------------------------------------------------------------
# Left side integral
matLocal_esq=np.zeros((3,3))
for pinteg in range(0,qtdNumInted):
matLocal_esq=matLocal_esq+np.transpose(invJacGradN)*invJacGradN*detJac*Integdudv*w*prop
# Left side matrix assembling
for im in range (0,3):
for jm in range(0,3):
MatGlobal_esq[elemNodes[k][im],elemNodes[k][jm]]=MatGlobal_esq[elemNodes[k][im],elemNodes[k][jm]]+ matLocal_esq[im,jm]
#------------------------------------------------------------------------------
# Right side integral
matLocal_dir=np.zeros((3,1))
# Get Js
Js=0
for eachregion in regionExcitation:
if eachregion.RegionNumber==elemTags[k][0]:
Js=eachregion.Value
break
# Right side integral
for pinteg in range(0,qtdNumInted):
uinteg=u[pinteg]
vinteg=v[pinteg]
N_prim=shapeFunctions.Nod_Tri_1order(uinteg, vinteg)
matLocal_dir=matLocal_dir+detJac*Integdudv*w*Js*np.transpose(N_prim)
# Right side matrix assembling
for im in range (0,3):
MatGlobal_dir[elemNodes[k][im],0]=MatGlobal_dir[elemNodes[k][im],0]+matLocal_dir[im,0]
#===============================================================================
# Boundary conditions
#===============================================================================
# Get values
nodesBC = GetBCs(elemNodes, elemTags, boundary)
# Apply values
for eachNodeBC in nodesBC:
MatGlobal_esq[eachNodeBC,:]=np.zeros(n)
MatGlobal_esq[eachNodeBC,eachNodeBC]=1.0
MatGlobal_dir[eachNodeBC]=nodesBC[eachNodeBC]
#===============================================================================
# Linear system
#===============================================================================
results=np.linalg.solve(MatGlobal_esq,MatGlobal_dir)
#===============================================================================
# Time control
#===============================================================================
timeEnd=time.time()
dtime=timeEnd-timeStart
print ('Solved in '+ str(dtime)+'ms')
#===============================================================================
# Save the results
#===============================================================================
write_file("resultsFile",results,'Results')
def convert_size(size_bytes):
if size_bytes == 0:
return "0B" # pragma: no cover
size_name = ("B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB")
i = int(math.floor(math.log(size_bytes, 1024)))
return "%s %s" % (int(size_bytes / math.pow(1024, i)), size_name[i])
def convert_size(size_bytes):
if size_bytes == 0:
return "0B" # pragma: no cover
size_name = ("B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB")
i = int(math.floor(math.log(size_bytes, 1024)))
return "%s %s" % (int(size_bytes / math.pow(1024, i)), size_name[i])
def __get_differentiation_control_factor(self, t):
exp = 1 - (self.max_efos / (self.max_efos - t))
w = math.pow(_e, exp)
r = round(random.uniform(0, 1), 2)
return F0 + F1 * math.pow(2, w) * r
from numpy import math a=7 b=67 d=118 c=math.pow(a,b) print(c+d)
def get_line_data(image, x1, y1, x2, y2, line_w=2, the_z=0, the_c=0, the_t=0):
"""
Grab pixel data covering the specified line, and rotates it horizontally.
Uses current rendering settings and returns 8-bit data.
Rotates it so that x1,y1 is to the left,
Returning a numpy 2d array. Used by Kymograph.py script.
Uses PIL to handle rotating and interpolating the data. Converts to numpy
to PIL and back (may change dtype.)
@param pixels: PixelsWrapper object
@param x1, y1, x2, y2: Coordinates of line
@param line_w: Width of the line we want
@param the_z: Z index within pixels
@param the_c: Channel index
@param the_t: Time index
"""
size_x = image.getSizeX()
size_y = image.getSizeY()
line_x = x2 - x1
line_y = y2 - y1
rads = math.atan2(line_y, line_x)
# How much extra Height do we need, top and bottom?
extra_h = abs(math.sin(rads) * line_w)
bottom = int(max(y1, y2) + extra_h / 2)
top = int(min(y1, y2) - extra_h / 2)
# How much extra width do we need, left and right?
extra_w = abs(math.cos(rads) * line_w)
left = int(min(x1, x2) - extra_w)
right = int(max(x1, x2) + extra_w)
# What's the larger area we need? - Are we outside the image?
pad_left, pad_right, pad_top, pad_bottom = 0, 0, 0, 0
if left < 0:
pad_left = abs(left)
left = 0
x = left
if top < 0:
pad_top = abs(top)
top = 0
y = top
if right > size_x:
pad_right = right - size_x
right = size_x
w = int(right - left)
if bottom > size_y:
pad_bottom = bottom - size_y
bottom = size_y
h = int(bottom - top)
# get the Tile - render single channel white
image.set_active_channels([the_c + 1], None, ['FFFFFF'])
jpeg_data = image.renderJpegRegion(the_z, the_t, x, y, w, h)
pil = Image.open(StringIO(jpeg_data))
# pad if we wanted a bigger region
if pad_left > 0 or pad_right > 0 or pad_top > 0 or pad_bottom > 0:
img_w, img_h = pil.size
new_w = img_w + pad_left + pad_right
new_h = img_h + pad_top + pad_bottom
canvas = Image.new('RGB', (new_w, new_h), '#ff0000')
canvas.paste(pil, (pad_left, pad_top))
pil = canvas
# Now need to rotate so that x1,y1 is horizontally to the left of x2,y2
to_rotate = math.degrees(rads)
# filter=Image.BICUBIC see
# http://www.ncbi.nlm.nih.gov/pmc/articles/PMC2172449/
rotated = pil.rotate(to_rotate, expand=True)
# finally we need to crop to the length of the line
length = int(math.sqrt(math.pow(line_x, 2) + math.pow(line_y, 2)))
rot_w, rot_h = rotated.size
crop_x = (rot_w - length) / 2
crop_x2 = crop_x + length
crop_y = (rot_h - line_w) / 2
crop_y2 = crop_y + line_w
cropped = rotated.crop((crop_x, crop_y, crop_x2, crop_y2))
# return numpy array
rgb_plane = asarray(cropped)
# greyscale image. r, g, b all same. Just use first
return rgb_plane[::, ::, 0]
def doGaussNewton_ref(param_value,
param_range,
UM_value,
obs,
cov,
scalings,
olist,
constraint,
studyJSON,
trace=False):
"""
Reference version of doGaaussNewton -- Kuniko's orginal version. This allows
Easy comparision between Kuniko's original code and re-enginnered code.
the function doGaussNewton_ref does the calculation for the n+1 initial runs in an iteration,
and can be invoked from (usually) the class GaussNewton, using data from studies, or from
test python scripts for arbitrary inputs
:param param_value: array of parameter values
:param param_range: array of max/min of range for each parameter in order
:param UM_value: array of simulated observables
:param obs: array of target values
:param cov: covariance array (header defines the observables in use) TODO inconsistent with use of constraint_target in JSON file?
:param scalings: observables are related arrays get scaled before statistical analysis
:param olist: name of the observables - probabaly unused except perhaps in diagnostic
:param constraint: TODO what is this??
:param studyJSON The complete dictionary from the study's JSON of which
this function might lift simple values to give extensibility.
Other arguments have had some manipulation in reading the JSON (scalings, slist... to give correspondence across arrays)
:param trace: turn on/off trace of what happening
:returns: yyT: array defining the next set of parameter values ## change the name of this to somethign sensible...
:returns: err: error values associated with input set of observables, one err per run.
:returns outdict: dictionary of data destined for json file
"""
# Through out code use the following is used:
# n - no. parameters # change to nParam
# m - no. observables # change to nObs
# TODO: replace has_key(x) throughout with x in dict
# get constants from JSON's directory:
alphas = studyJSON['alphas']
terminus = studyJSON['terminus']
# optional ones....
if studyJSON.has_key("sigma"):
sigma = studyJSON['sigma']
else:
sigma = 0
if (sigma != 0):
## need constraint_target and if not fail.
if studyJSON.has_key("constraint_target"):
constraint_target = studyJSON['constraint_target']
else:
sys.exit("Please specifiy constraint_target when sigma != 0."
) ## exit with error
## get the mu value
if studyJSON.has_key("mu"):
mu = studyJSON['mu']
else:
mu = 1
m = len(obs)
n = len(param_value[0, :])
# include constraint
coef1 = 1. / (m + sigma)
coef2 = sigma / ((m + 1) * (2. * mu))
if sigma == 0:
constraint = zeros(n + 1) # TODO make np.zeros
constraint[:] = 0. # todo Unnesc
else: # sigma != 0 so have constraint..
use_constraint = constraint.copy(
) - constraint_target #TODO what this doing?
## print out some info if trace on
if trace:
print "Version is ", version
# print 'constraint.shape()=',constraint
# print 'constraint_target=',constraint_target
print " n %d m %d \n" % (n, m)
print "param_range"
print param_range
na = len(alphas) # TODO na->nAlphas or nLineSearchRuns
optStatus = "Continue" # default optStatus
# transpose arrays TODO -remove transpose and just do calculations directly
param_valueT = transpose(param_value)
param_rangeT = transpose(param_range)
UM_valueT = transpose(UM_value)
if trace:
print "UM_valueT", UM_valueT
# scale observables, observations and covariance matrix
# TODO replace with UM_value=UM_value*scalings taking advantage of broadcasting.
# and
s_UM_valueT = UM_valueT.copy()
for i in range(n + 1): # CHANGED FROM m
s_UM_valueT[:, i] = UM_valueT[:, i] * scalings
s_obs = obs * scalings # leave
if trace:
print "obs"
print zip(olist, zip(obs, scalings))
print zip(olist, s_obs)
# TOD cov=cov*np.outer(scalings,scalings)
for i in range(m):
cov[:, i] = cov[:, i] * scalings # scale rows
cov = cov * scalings # using broadcasting to scale columns?
## now want to regularize s_cov though should be done after constraint included.
if 'covar_cond' in studyJSON and studyJSON[
'covar_cond'] is not None: # specified a condition number for the covariance matrix
cov = regularize_cov_ref(cov, studyJSON['covar_cond'], trace=trace)
# constraint C_bar
s_cov_constraint = zeros((m + 1, m + 1))
covT = linalg.inv(
cov) # TODO replace with pseudo inverse and call it covInv
#TODO move all stuff with constraint together and check theory too.
#TODO merge constraint into simulated and observed vector/matrix
s_cov_constraint[0:m, 0:m] = coef1 * covT
s_cov_constraint[m, m] = coef2
if trace:
print "Scaled and regularized cov"
print cov
UM = s_UM_valueT[:, 0]
# commented out by SFBT
# param_min = param_rangeT[ 1, : ]
#
#
# param_max = param_rangeT[ 0, : ]
## ADDED by SFBT
param_min = param_rangeT[0, :]
param_max = param_rangeT[1, :]
if trace:
print "param_rangeT"
print param_rangeT
print "param_min"
print param_min
print "param_max"
print param_max
## TODO -- scale from 1 to 10 based on param_min and param_max
## param_scale=(param-param_min)/(param_max-param_min)*9+1
## NB this will change the results so do after all tests passed.
param_scale = ones((n))
for i in range(n):
tmp = fabs(param_valueT[i, 0]) * param_scale[i]
##tmp = fabs(param_value[ 0, i ]) * param_scale[i]
while tmp < 1:
param_scale[i] = param_scale[i] * 10
tmp = tmp * 10
## TODO clean JACOBIAN calculation
## DO NOT TRANSPOSE JACOBIAN!!!
Jacobian = zeros((m, n))
##Jacobian = zeros( (n,m) ) # TODO -- remove commented code.
##Jacobian = zeros( (m,n) )
# constraint Jacobian_bar
Jacobian_constraint = zeros((m + 1, n))
for i in range(n):
##Jacobian[ :, i ] = (UM_valueT[ :,i+1 ] - UM) \
Jacobian[ :, i ] = (s_UM_valueT[ :,i+1 ] - UM) \
/((param_valueT[ i,i+1 ]-param_valueT[ i,0 ])*param_scale[ i ])
##Jacobian[ : , i ] = (UM_value[ i+1, : ] - UM) \
##/((param_value[ i+1, i ]-param_value[ 0, i ])*param_scale[ i ])
# constraint Jacobian_bar
Jacobian_constraint[0:m, :] = Jacobian
for i in range(n):
# 02/02/2015 changed below to incorporate TOA imbalance of 0.5 W/m^2
#Jacobian_constraint[m,i] = (constraint[i+1]-constraint[0]) \
Jacobian_constraint[m,i] = (use_constraint[i+1]-use_constraint[0]) \
/((param_valueT[ i,i+1 ]-param_valueT[ i,0 ])*param_scale[ i ])
F = UM - s_obs # This is difference of previous best case from obs
##F = UM - obs
# TODO understand what this is doing.
# constraint r_bar
F_constraint = zeros(m + 1)
F_constraint[0:m] = F
F_constraint[m] = use_constraint[0]
if trace:
print "Jacobian before transpose"
print Jacobian
## save("Jacobian",Jacobian)
## read with j= np.load("Jacobian.npy")
##TODO figure out what happens if no constraint
JacobianT = Jacobian.transpose()
# constraint J_bar T
Jacobian_constraintT = Jacobian_constraint.transpose()
## 29/09/2014
## modified below to take account of cov in r & henceforth Jacobian
if len(cov) == 0:
J = dot(JacobianT, Jacobian)
else:
# constraint approximate Hessian of f(x)
dum = dot(Jacobian_constraintT, s_cov_constraint)
J = dot(dum, Jacobian_constraint)
if trace:
print "after dot, J is "
print J
## save("dottedJ",J)
## TOD clean regularisation up, rename J to hessian
perJ = J
if trace:
print 'm, linalg.matrix_rank(perJ)', m, linalg.matrix_rank(perJ)
con = linalg.cond(J)
k = 8
eye = identity(n)
if trace: ## SFBT added to reference doGaussNewton
print "con %e k %d" % (con, k)
while con > 10e10:
if k >= 4:
k = k - 1
perJ = J + math.pow(10, -k) * eye
con = linalg.cond(perJ)
if trace:
print "conREF %e k %d" % (con, k)
# 11/05/2015 if con does not ge small enough quit w/ message
# 22/04/2015 if con does not ge small enough
else:
print "regularisation insufficient, stopping: con %e k %d" % (con,
k)
optStatus = "Fatal"
dictionary = {"jacobian": Jacobian, "hessian": J, "condnum": con}
return optStatus, None, None, None, dictionary
## calculate eval_min, eval_max, eval_star
#eval_max, evec_max = eigs(J, k=1, which='LM')
#eval_min, evec_min = eigs(J, k=1, sigma=0, which='LM')
#eval_star = eval_max*10.**(-10) - eval_min
#perJ = J + eval_star*eye
#con = linalg.cond(perJ)
#print "con %e eval_min %e eval_star %e k %d"%(con, eval_min, eval_star, k)
# 22/04/2015 break out of the while loop
#break
J = perJ
## 29/09/2014
## modified below to take account of cov in r & henceforth Jacobian
#tmp = dot(JTcovT,F)
# constraint approximate Jacobian of f(x)
tmp = dot(dum,
F_constraint) # TODO look at maths and understand this line.
##tmp = dot(JacobianT,F)
s = -linalg.solve(J, tmp) # work out direction to go in.
if trace:
fn_label = 'DOGN_REF'
print fn_label + ": hessian =", np.round(J, 2)
print fn_label + ": J^T C^-1 F = ", np.round(
Jacobian_constraintT.dot(s_cov_constraint).dot(F_constraint), 2)
print fn_label + ": s=", s
x = param_scale * param_valueT[:, 0]
# err & constrained err
nsize = n + 1
nloop = n + 1
ioffset = 0
err, err_constraint = calcErr_ref(s_UM_valueT, s_obs, covT, use_constraint,
coef1, coef2, nsize, nloop, ioffset)
## TODO work with scaled parameters throughout
test_max = param_max * param_scale
test_min = param_min * param_scale
y = zeros((na, n))
##y = zeros( (n,na) )
## deal with boundaries TODO use numpy stuff to avoid loops.
for i in range(na):
##y[ i,: ] = x + math.pow(0.1,i)*s
y[i, :] = x + alphas[i] * s
##y[ :, i ] = x + math.pow(0.1,i)*s
for j in range(n):
if y[i, j] > test_max[j]:
y[i, j] = test_max[j]
elif y[i, j] < test_min[j]:
y[i, j] = test_min[j]
else:
y[i, j] = y[i, j]
##if y[ j, i] > test_max[j]:
## y[ j, i] = test_max[j]
##elif y[ j, i] < test_min[j]:
## y[ j, i] = test_min[j]
##else:
## y[ j, i] = y[ j, i]
yy = y
for i in range(na):
##for i in range(m):
yy[i, :] = y[i, :] / param_scale
##yy[ : , i] = y[ :, i]/param_scale
# transpose backward
yyT = transpose(yy)
################################ snip ################################
dictionary = {
"jacobian": Jacobian,
"hessian": J,
"condnum": con,
"objfn_GN": err,
'InvCov': covT
}
dictionary['software'] = version
# hack for time being: output in files
date = datetime.now()
sdate = date.strftime('%d-%m-%Y_%H:%M:%S')
outdir = './' # with /exports/work/geos_cesd_workspace/OptClim/Data/kytest/
# the framework fails.
# This puts files in the iteration directory,
# or if you are runnign standalone tests, to the directory you are in.
# outfile = outdir+'jacob_'+sdate+'.dat'
# save(outfile,Jacobian)
#
# outfile = outdir+'hessi_'+sdate+'.dat'
# save(outfile,J)
#
# outfile = outdir+'parav_'+sdate+'.dat'
# save(outfile,param_value)
#
# outfile = outdir+'condn_'+sdate+'.txt'
# f = open(outfile,'w')
# f.write(str(con)+'\n')
# f.close()
################################ snip ################################
#return yyT,err,dictionary
# constraint err
return optStatus, yyT, err, err_constraint, dictionary
def extract_check_calc_specific_sites(recalc_data_oxides_cats_OX_list):
print(
"\n\tFORMULA=> extract_check_calc_specific_sites(recalc_data_oxides_cats_OX__list)"
)
print("\t\tDEVO SEPARARE PER LISTE DI MINERALE")
print("\t\t\tPER OGNI LISTA DI MINERALE FARE I CALCOLI")
print("\t\t\t\tRITORNARE LISTE DI LISTE DI MINERALI CON specific sites")
print()
a_args = []
print(recalc_data_oxides_cats_OX_list)
lista = []
for each_analysis in recalc_data_oxides_cats_OX_list:
lista.append(each_analysis)
print("LISTA ", lista)
for l in lista:
print("l: ", l)
if l['mineral'] not in a_args:
a_args += [l['mineral']]
new_list = [[]] * len(a_args)
dict_of_list = {}
for i in range(len(a_args)):
for l in [l for l in lista if l['mineral'] == a_args[i]]:
new_list[i] = new_list[i] + [l]
sublist_list = new_list[i]
dict_of_list[a_args[i]] = sublist_list
for mine, value in dict_of_list.items():
print("\nmineral group = ", mine, 'is: ', value)
global zzz
zzz = 100.00001
if mine == 'grt':
#GARNET#
for single in value:
alm = single['Fe2'] / (single['Fe2'] + single['Mg'] +
single['Ca'] + single['Mn'])
py = single['Mg'] / (single['Fe2'] + single['Mg'] +
single['Ca'] + single['Mn'])
gr = single['Ca'] / (single['Fe2'] + single['Mg'] +
single['Ca'] + single['Mn'])
sps = single['Mn'] / (single['Fe2'] + single['Mg'] +
single['Ca'] + single['Mn'])
XFe = single['Fe2'] / (single['Fe2'] + single['Mg'])
XMg = single['Mg'] / (single['Fe2'] + single['Mg'])
'''
Fe3+ = 2*X*(1-T/S)
X=>oxigens in formula
T=>ideal number of cations
S=>observed cations
'''
if 'Fe3' in single:
print("good to know")
pass
else:
print("CATTTIONI SUM GRT: ", single['SUMcat'])
Fe3 = 2 * 12 * (1 - 8 / single['SUMcat'])
single.update({'Fe3': round(Fe3, 3)})
pass
single.update({'alm': round(alm, 3)})
single.update({'py': round(py, 3)})
single.update({'gr': round(gr, 3)})
single.update({'sps': round(sps, 3)})
single.update({'XFe': round(XFe, 3)})
single.update({'XMg': round(XMg, 3)})
print("every mineral analysis: ", single, "")
elif mine == 'amph':
#AMPH#
for single in value:
if 8 - single['Si'] > 0:
aliv = 8 - single['Si']
else:
aliv = 0
single.update({'aliv': aliv})
alvi = single['Al'] - aliv
single.update({'alvi': round(alvi, 3)})
single.update({'T': zzz})
if 'Fe3' in single:
print("good to know")
pass
else:
print("CATTTIONI SUM: ", single['SUMcat'])
Fe3 = 2 * 12 * (1 - 8 / single['SUMcat'])
single.update({'Fe3': round(Fe3, 3)})
pass
print("every mineral analysis: ", single, "")
elif mine == 'px':
#PYROXENE#
for single in value:
if 2 - single['Si'] > 0:
aliv = 2 - single['Si']
else:
aliv = 0
single.update({'aliv': round(aliv, 3)})
alvi = single['Al'] - aliv
single.update({'alvi': round(alvi, 3)})
jd1 = single['Na'] * 2
single.update({'jd1': round(jd1, 3)})
if single['alvi'] > (single['Na'] + single['K']):
jd2 = single['alvi']
else:
jd2 = single['Na'] + single['K']
single.update({'jd2': round(jd2, 3)})
if single['alvi'] > (single['Na'] + single['K']):
acm = single['Na'] + single['K'] - single['alvi']
else:
acm = 0
single.update({'acm': round(acm, 3)})
if 'Fe3' in single:
print("good to know")
pass
else:
print("CATTTIONI SUM: ", single['SUMcat'])
Fe3 = 2 * 12 * (1 - 8 / single['SUMcat'])
single.update({'Fe3': round(Fe3, 3)})
pass
if (single['Fe3'] + single['Cr']) / 2 > single['acm']:
CaFeTs = (single['Fe3'] + single['Cr']) / 2
else:
CaFeTs = 0
single.update({'CaFeTs': round(CaFeTs, 3)})
CaTiTs = single['Ti']
single.update({'CaTiTs': round(CaTiTs, 3)})
if ((single['aliv'] + single['alvi'] - single['jd2'] -
2 * single['Ti']) / 2) > 0:
CaTs = (single['aliv'] + single['alvi'] - single['jd2'] -
2 * single['Ti']) / 2
else:
CaTs = 0
single.update({'CaTs': CaTs})
if (single['Ca'] - single['CaFeTs'] - single['CaTiTs'] -
single['CaTs']) > 0:
woll = single['Ca'] - single['CaFeTs'] - single[
'CaTiTs'] - single['CaTs']
else:
woll = 0
single.update({'woll': round(woll, 3)})
if 'Ni' in single.keys():
en = (single['Mg'] + single['Ni']) / 2
else:
en = (single['Mg'])
single.update({'en': round(en, 3)})
fs = (single['Mn'] + single['Fe2']) / 2
single.update({'fs': round(fs, 3)})
print("every mineral analysis: ", single, "")
elif mine == 'bt':
for single in value:
#single.update({'Jdddd':zzz})
print("every mineral analysis: ", single, "")
#print("TIIIII: ", single['Ti'])
#global T_henry2005
if (single['Ti'] > 0.06 and single['Ti'] < 0.6):
#print("TIIIIIAAA: ", single['Ti'])
b = 4.6482E-09
a = -2.3594
#b = 4648200000
c = -1.7283
lnTi = round(math.log(single['Ti']), 3)
xmg = round(single['Mg'] / (single['Mg'] + single['Fe2']),
3)
print("lnTi ", lnTi)
print("xmg ", xmg)
primo = lnTi
secondo = a
terzo = round(c * (math.pow(xmg, 3)), 3)
print("terzo", terzo)
quarto = round((primo - secondo - terzo), 3)
quinto = round((quarto / b), 3)
print("quarto ", quarto)
print("quinto", quinto)
if quinto > 0:
finale = math.pow(quinto, 0.333)
T_henry2005 = finale
else:
print(
"cannot use Henry's calibration, see original paper"
)
single.update({'T_henry2005': 'OutOf_XMg_Range'})
pass
else:
print("cannot use Henry's calibration, see original paper")
single.update({'T_henry2005': 'OutOf_Ti_Range'})
pass
if (T_henry2005 > 400 and T_henry2005 < 800):
single.update({'T_henry2005': round(T_henry2005, 3)})
else:
print("cannot use Henry's calibration, see original paper")
single.update({'T_henry2005': 'OutOf_T_Range'})
return dict_of_list ##LISTE_DI_LISTE_DI_MINERALI_CON_specific_sites
def euclidean(x1, x2):
return math.sqrt(
math.pow(x1[0] - x2[0], 2) + math.pow(x1[1] - x2[1], 2))
def get_line_data(pixels, x1, y1, x2, y2, line_w=2, the_z=0, the_c=0, the_t=0):
"""
Grabs pixel data covering the specified line, and rotates it horizontally
so that x1,y1 is to the left,
Returning a numpy 2d array. Used by Kymograph.py script.
Uses PIL to handle rotating and interpolating the data. Converts to numpy
to PIL and back (may change dtype.)
@param pixels: PixelsWrapper object
@param x1, y1, x2, y2: Coordinates of line
@param line_w: Width of the line we want
@param the_z: Z index within pixels
@param the_c: Channel index
@param the_t: Time index
"""
size_x = pixels.getSizeX()
size_y = pixels.getSizeY()
line_x = x2 - x1
line_y = y2 - y1
rads = math.atan(float(line_x) / line_y)
# How much extra Height do we need, top and bottom?
extra_h = abs(math.sin(rads) * line_w)
bottom = int(max(y1, y2) + extra_h / 2)
top = int(min(y1, y2) - extra_h / 2)
# How much extra width do we need, left and right?
extra_w = abs(math.cos(rads) * line_w)
left = int(min(x1, x2) - extra_w)
right = int(max(x1, x2) + extra_w)
# What's the larger area we need? - Are we outside the image?
pad_left, pad_right, pad_top, pad_bottom = 0, 0, 0, 0
if left < 0:
pad_left = abs(left)
left = 0
x = left
if top < 0:
pad_top = abs(top)
top = 0
y = top
if right > size_x:
pad_right = right - size_x
right = size_x
w = int(right - left)
if bottom > size_y:
pad_bottom = bottom - size_y
bottom = size_y
h = int(bottom - top)
tile = (x, y, w, h)
# get the Tile
plane = pixels.getTile(the_z, the_c, the_t, tile)
# pad if we wanted a bigger region
if pad_left > 0:
data_h, data_w = plane.shape
pad_data = zeros((data_h, pad_left), dtype=plane.dtype)
plane = hstack((pad_data, plane))
if pad_right > 0:
data_h, data_w = plane.shape
pad_data = zeros((data_h, pad_right), dtype=plane.dtype)
plane = hstack((plane, pad_data))
if pad_top > 0:
data_h, data_w = plane.shape
pad_data = zeros((pad_top, data_w), dtype=plane.dtype)
plane = vstack((pad_data, plane))
if pad_bottom > 0:
data_h, data_w = plane.shape
pad_data = zeros((pad_bottom, data_w), dtype=plane.dtype)
plane = vstack((plane, pad_data))
pil = script_utils.numpy_to_image(plane, (plane.min(), plane.max()), int32)
# Now need to rotate so that x1,y1 is horizontally to the left of x2,y2
to_rotate = 90 - math.degrees(rads)
if x1 > x2:
to_rotate += 180
# filter=Image.BICUBIC see
# http://www.ncbi.nlm.nih.gov/pmc/articles/PMC2172449/
rotated = pil.rotate(to_rotate, expand=True)
# rotated.show()
# finally we need to crop to the length of the line
length = int(math.sqrt(math.pow(line_x, 2) + math.pow(line_y, 2)))
rot_w, rot_h = rotated.size
crop_x = (rot_w - length) / 2
crop_x2 = crop_x + length
crop_y = (rot_h - line_w) / 2
crop_y2 = crop_y + line_w
cropped = rotated.crop((crop_x, crop_y, crop_x2, crop_y2))
return asarray(cropped)
def getHalfToneFreq(self, numHalfTones=0):
return self.baseFreq * math.pow(math.pow(2, 1 / 12), numHalfTones)