def SPSA_kem(V, normalise):
eta = 1e-4
Delta = np.random.choice([-1,1], V.shape) * eta
V0, V1 = V+eta*Delta, V-eta*Delta
normalised0, normalised1 = normalise(V0), normalise(V1)
K0, K1 = MarkovChain(normalised0).K, MarkovChain(normalised1).K
return (K0-K1)/(2*eta*Delta)
def derivFD_K(P, normalise, i ,j): h=1e-4 P1 = P.copy() P1[i,j] += h #P[i,j] -= h K1 = MarkovChain(normalise(P1)).K K2 = MarkovChain(normalise(P)).K return (K1-K2)/2*h
def main():
print("===============================================================")
start_time = time.time()
####simulation####
#Initialize
n = 4
P = np.ones((n, n)) / n
MC = MarkovChain('Courtois')
P = MC.P
r = 2 #0,1,..,n-1 The state for which the stationary distribution will be maximized
c = 1 #0, 1, .., n-2
Theta = toTheta(P)
epsilon = 0.2
gamma = 0.001 #at M = 10/gamma (=100) the effect of M to the objective function will now start to become noticable
N = 500 #Number of iterations
#Start
print(np.round(P, 3))
print(np.round(MC.pi, 3))
simulation(P, r, epsilon, gamma, N)
#Running time
print('Running time of the programme:', time.time() - start_time)
print("===============================================================")
def derivativeM(P, r, c):
'''
Purpose:
Calculate the derivative of the mean first passage time matrix of P(Theta) w.r.t. Theta(r,c)
Input:
P - a (nxn)-matrix (numpy array) of transition probabilities
r - indicates the row of the theta element we differentiate to, i.e., for Theta(r,c).
c - indicates the column of the theta element we differentiate to, i.e., for Theta(r,c).
Output:
derivM - (nxn)-matrix (numpy array) of the derivative of the M matrix
'''
n, _ = P.shape
Ones = np.ones((n, n))
MC = MarkovChain(P)
pi = MC.pi[0]
Pi = MC.Pi
D = MC.D
dg_Pi = diag(Pi)
dg_D = diag(D)
derivD = derivativeD(P, r, c)
dg_derivD = diag(derivD)
#derivative of the inverse of dg_Pi
deriv_dg_Pi_inv = np.zeros((n, n))
deriv_dg_Pi_inv[r, r] = -1 * (derivative_pi(P, r, c)[r]) / (pi[r]**2)
#using derivative of M formula
derivM = (-derivD + Ones @ dg_derivD) @ np.linalg.inv(
dg_Pi) + deriv_dg_Pi_inv @ (
np.identity(n) - D + Ones @ dg_D
) #Column r and row r are (for most) nonzero, does this make sense?
return derivM
def Kemeny(P): with torch.no_grad(): tmp = P.clone().detach().numpy() tmp = torch.from_numpy(tmp) K = torch.FloatTensor(MarkovChain(tmp).K) return K
def derivativeM(Theta, r, c, Delta):
'''
Purpose:
Calculate the derivative of the mean first passage time matrix of P(Theta) w.r.t. Theta(r,c)
Input:
Theta - a (nx(n-1))-matrix (numpy array) with angles
r - indicates the row of the theta element we differentiate to, i.e., for Theta(r,c).
c - indicates the column of the theta element we differentiate to, i.e., for Theta(r,c).
Output:
derivM - (nxn)-matrix (numpy array) of the derivative of the M matrix
'''
n, _ = Theta.shape
Ones = np.ones((n, n))
MC = MarkovChain(toP(Theta))
pi = MC.pi[0]
Pi = MC.Pi
D = MC.D
dg_Pi = diag(Pi)
dg_D = diag(D)
#derivD = derivativeD(Theta,r,c)
#derivD = derivativeD_SPSA(Theta,r,c, Delta)
derivD = derivativeD_H(Theta, r, c)
dg_derivD = diag(derivD)
#derivative of the inverse of dg_Pi
deriv_dg_Pi_inv = np.zeros((n, n))
deriv_dg_Pi_inv[r, r] = -1 * (derivative_pi(Theta, r, c)[r]) / (pi[r]**2)
#using derivative of M formula
derivM = (-derivD + Ones @ dg_derivD) @ np.linalg.inv(
dg_Pi) + deriv_dg_Pi_inv @ (np.identity(n) - D + Ones @ dg_D)
return derivM
def Kemeny(indices, values, size, tol=1e-8):
"""Calculate the Kemeny constant."""
values[abs(values) < tol] = 0.0 #cast values close to zero to zero.
#NOTE: this is no longer needed if we do not allow values to drop below eps!
P = torch.sparse.FloatTensor(indices, values, size)
MC = MarkovChain(P.detach().to_dense().numpy())
return torch.FloatTensor([MC.K])
def start():
print("===============================================================")
start_time = time.time()
####simulation####
#Initialize
n = 8
MC = MarkovChain('Courtois')
P = MC.P
#P = np.ones((n,n))/n
#MC = MarkovChain(P)
r = 0 #0,1,..,n-1 The state for which the stationary distribution will be maximized
#c = 5 #0, 1, .., n-2
Theta = toTheta(P)
epsilon = 0.1
gamma = 0.0005
N = 300 #Number of iterations
#Start
print('###########')
print('First stationary probabilities:\n', MC.pi[0])
print('First Objective value:', objJ(Theta, r, gamma))
print('First transition probabilities:\n', np.round(MC.P, 3))
#print ('First M value:\n', np.round(MC.M*gamma,2))
simulation(Theta, r, epsilon, gamma, N)
#Running time
print('Running time of the programme:', time.time() - start_time)
print("===============================================================")
def start():
print("===============================================================")
start_time = time.time()
####simulation####
#Initialize
n = 4
MC = MarkovChain('Courtois')
P = MC.P
#P = np.ones((n,n))/n
#MC = MarkovChain(P)
r = 1 #0,1,..,n-1 The state for which the stationary distribution will be maximized
#c = 5 #0, 1, .., n-2
Theta = toTheta(P)
epsilon = 0.1 #Epsilon = 0.005 and update (>200) epsilon decreasing factor with factor 10 worked for all r in example Courtois
gamma = 10**-6
N = 300 #Number of iterations
subset = [0]
psi = 3
simulation(Theta, r, epsilon, subset, gamma, psi, N)
#Running time
print('Running time of the programme:', time.time() - start_time)
print("===============================================================")
def Kemeny_spsa(indices, values, size, eta):
"""Calculate gradient of Kemeny constant using SPSA."""
#NOTE: the gradients are only calculated for the edges in the network.
n = values.shape[0]
delta = np.random.choice([-1, 1], n)
#is this implementation oke? Using clamp(0,1), but also using SPSA for not all params.
#Use one of the normalisation functions!
values1 = (values + eta * delta).clamp(0, 1)
values2 = (values - eta * delta).clamp(0, 1)
P1 = torch.sparse.FloatTensor(indices, values1, size)
P2 = torch.sparse.FloatTensor(indices, values2, size)
K1 = MarkovChain(P1.to_dense().numpy()).K
K2 = MarkovChain(P2.to_dense().numpy()).K
grads = torch.FloatTensor([(K1 - K2) / 2 * eta * delta])
return grads
def simulation(Theta, r, epsilon, gamma, N):
'''
Purpose:
perform a simulation to optimize pi[r]
'''
P = toP(Theta)
P_old = P
n, _ = P.shape
mTheta_changes = np.zeros((N, n - 1))
mStatDist_changes = np.zeros((N, n))
vObj_changes = np.zeros(N)
vPr_changes = np.zeros(N)
pi_old = stationaryDist(Theta, None)
pi_new = pi_old
for step_nr in range(N):
if step_nr > 150: epsilon = 1 / (step_nr)
if step_nr > 300: epsilon = 1 / (2 * step_nr)
if step_nr > 450: epsilon = 1 / (4 * step_nr)
if step_nr > 600: epsilon = 1 / (8 * step_nr)
if step_nr > 750: epsilon = 1 / (16 * step_nr)
if step_nr > 900: epsilon = 1 / (32 * step_nr)
v_gradJ = np.zeros(n - 1) #store derivJ for all Theta[r,.] angles
for c in range(n - 1): #c represents angle index
derivJ = deriv_objJ(Theta, r, c, gamma)
v_gradJ[c] = derivJ
#Update row Theta[r]
#Theta[r] = Theta[r] + epsilon*v_gradJ
#Update singel Theta[r,c_max]
c_max = np.argmax(abs(v_gradJ))
Theta[r, c_max] = Theta[r, c_max] + epsilon * v_gradJ[c_max]
#Update MC measures
P = toP(Theta)
pi = stationaryDist(Theta, None)
MC = MarkovChain(P)
#Track changes
mStatDist_changes[step_nr] = pi
vObj_changes[step_nr] = objJ(Theta, r, gamma)
mTheta_changes[step_nr] = Theta[r]
vPr_changes[step_nr] = P[r, r]
print('==========')
print('theta[r]=', np.round(Theta, 3)[r])
print('Gradient Obj=', np.round(v_gradJ, 3))
print('Gradient Obj x Epsilon=', np.round(v_gradJ * epsilon, 3))
print('epsilon=', epsilon)
print('iteration=', step_nr)
print('pi_new', np.round(pi_new, 4))
print('P[r]=', np.round(P[r], 3))
print('Obj=', np.round(vObj_changes[step_nr], 3))
print('==========')
print('Maximum M value=', np.max(MC.M))
print('Final self probability=', round(P[r, r], 3))
plotThetaprocess(mTheta_changes, mStatDist_changes, vObj_changes,
vPr_changes, step_nr, r)
def derivativeD_SPSA(Theta, r, c, Delta):
'''
Purpose:
Estimate the derivative w.r.t. Theta(r,c) of the deviation matrix using SPSA
Input:
Theta - a (nx(n-1))-matrix (numpy array) with angles
r - indicates the row of the theta element we differentiate to, i.e., for Theta(r,c).
c - indicates the column of the theta element we differentiate to, i.e., for Theta(r,c).
Output:
derivD - (nxn)-matrix (numpy array) of the derivative of the D matrix
'''
n, _ = Theta.shape
nabla = 0.0001
Theta1 = copy.deepcopy(Theta)
Theta2 = copy.deepcopy(Theta)
Theta1[r] = Theta[r] - nabla * Delta
Theta2[r] = Theta[r] + nabla * Delta
D1 = MarkovChain(toP(Theta1)).D
D2 = MarkovChain(toP(Theta2)).D
derivD = (D2 - D1) / (2 * nabla * Delta[c])
return derivD
def simulation(P, r, epsilon, gamma, N):
'''
Purpose:
perform a simulation to optimize pi[r]
'''
Theta = toTheta(P)
n, _ = P.shape
mTheta_changes = np.zeros((N, n - 1))
mStatDist_changes = np.zeros((N, n))
for step_nr in range(N):
v_derivJ = np.zeros(n - 1) #store derivJ for all Theta[r,.] angles
for c in range(n - 1): #c represents angle index
derivJ = deriv_objJ(P, r, c, gamma)
v_derivJ[c] = derivJ
#find angle with highest effect on the objective value
c_max = np.argmax(abs(v_derivJ))
#c_change = random.randint(0,n-2)
#Update angle Theta[r,c_max], we are maximizing
Theta[r, c_max] = Theta[r, c_max] + epsilon * v_derivJ[c_max]
mTheta_changes[step_nr] = Theta[r]
#New P matrix
P = toP(Theta)
MC = MarkovChain(P)
mStatDist_changes[step_nr] = MC.pi[0]
#stop condition for reducibility
if (MC.bUniChain == False):
print(
'The Markov chain described by P is no longer uni-chain! This should not happen.'
)
exit()
#print current progress
print('r=', r)
print('gradient obj=', v_derivJ)
print('P leaving state r=', P[r])
print('P entering state r=', P[:, r])
print('pi=', MC.pi[0])
print('\n')
#print Final progress (again)
print('\n\n\n')
print('Final stationary probabilities:', MC.pi[0])
print('Final Objective value:', objJ(P, r, gamma))
print('Final transition probabilities:\n', np.round(P, 3))
print('Final M value:\n', MC.M)
plotThetaprocess(mTheta_changes, mStatDist_changes)
def objJ(Theta, r):
'''
Purpose:
Compute the value of the objective function. The objective is to maximize pi[r] under the condition that the Markov chain remains irreducible,
for this the (relevant) values of M are used as penalty in the objective function.
Input:
Theta - a (nxn)-matrix (numpy array) of transition probabilities
r - indicates the row of the theta element we differentiate to, i.e., for Theta(r,c).
Output:
J - real number, the objective value
'''
MC = MarkovChain(toP(Theta))
pi = MC.pi[0]
return pi[r]
def objJ(P, r, gamma):
'''
Purpose:
Compute the value of the objective function. The objective is to maximize pi[r] under the condition that the Markov chain remains irreducible,
for this the (relevant) values of M are used as penalty in the objective function.
Input:
P - a (nxn)-matrix (numpy array) of transition probabilities
r - indicates the row of the theta element we differentiate to, i.e., for Theta(r,c).
gamma - real number, the penalty factor for M
Output:
J - real number, the objective value
'''
MC = MarkovChain(P)
pi = MC.pi[0]
M = MC.M
return pi[r] #+ gamma*(sum(M[r]) + sum(M[:,r]))
def plot():
n = 500
Theta = toTheta(MarkovChain('Courtois').P)
theta_rnd = np.array([0] +
list(np.sort(np.random.rand(n - 2) * np.pi / 2)) +
[np.pi / 2])
mObj = np.zeros((n, 2))
vP = np.zeros(n)
gamma = 0.0000005
r = 7
'''
for i in range(n):
Theta_tmp1 = Theta
Theta_tmp1[r,6] = theta_rnd[i]
mObj[i,0] = objJ(Theta_tmp1,r,gamma)
vP[i] = toP(Theta_tmp1)[r,r]
#Theta_tmp2 = Theta
#Theta_tmp2[r,5] = theta_rnd[i]
#Theta_tmp3 = Theta
#Theta_tmp3[r,4] = theta_rnd[i]
#Theta_tmp4 = Theta
#Theta_tmp4[r,3] = theta_rnd[i]
#Theta_tmp5 = Theta
#Theta_tmp5[r,2] = theta_rnd[i]
#Theta_tmp6 = Theta
#Theta_tmp6[r,1] = theta_rnd[i]
#Theta_tmp7 = Theta
#Theta_tmp7[r,0] = theta_rnd[i]
# mObj[i,1] = objJ(Theta_tmp2,r,gamma)
plt.plot(theta_rnd, vP, label ='P[r,r]')
plt.legend()
plt.show()
plt.plot(theta_rnd, mObj[:,0], label='J1')
plt.legend()
plt.show()
'''
mDerivJ = np.zeros((n, 7))
for i in range(n):
Theta_tmp1 = Theta
Theta_tmp1[r, 0] = theta_rnd[i]
for c in range(7):
mDerivJ[i, c] = deriv_objJ(Theta_tmp1, r, c, gamma)
for c in range(7):
plt.plot(theta_rnd, mDerivJ[:, c], label=c)
plt.legend()
plt.show()
def simulation(Theta, r, epsilon, gamma, N):
'''
Purpose:
perform a simulation to optimize pi[r]
'''
P = toP(Theta)
n, _ = P.shape
mTheta_changes = np.zeros((N, n - 1))
mStatDist_changes = np.zeros((N, n))
mObj_changes = np.zeros(N)
for step_nr in range(N):
if (step_nr > 100): epsilon = 1 / (step_nr)
v_derivJ = np.zeros(n - 1) #store derivJ for all Theta[r,.] angles
for c in range(n - 1): #c represents angle index
derivJ = deriv_objJ(Theta, r, c, gamma)
v_derivJ[c] = derivJ
#find angle with highest effect on the objective value
c_max = np.argmax(abs(v_derivJ))
#c_max = np.random.randint(n-1)
#Update angle Theta[r,c_max], we are maximizing
Theta[r, c_max] = Theta[r, c_max] + epsilon * v_derivJ[c_max]
mTheta_changes[step_nr] = Theta[r]
#New P matrix
P = toP(Theta)
MC = MarkovChain(P)
mStatDist_changes[step_nr] = MC.pi[0]
mObj_changes[step_nr] = objJ(Theta, r, gamma)
print('==========')
print('iteration=', step_nr)
'''
print ('epsilon=', epsilon)
print ('P[r]=', np.round(P[r],3))
print ('theta[r]=', np.round(mTheta_changes[step_nr],3))
print ('pi=', np.round(mStatDist_changes[step_nr],3))
print ('Derivative Obj=', np.round(v_derivJ,3))
print ('c max=', c_max)
print ('Obj=', np.round(mObj_changes[step_nr],3))
print ('==========')
'''
#print and plot final results
print('Final Objective value:', objJ(Theta, r, gamma))
print('Final transition probabilities:\n', np.round(MC.P, 3))
print('Final stationary probabilities:\n', MC.pi[0])
#print ('Final M value:\n', np.round(MC.M*gamma,2))
plotThetaprocess(mTheta_changes, mStatDist_changes, mObj_changes)
def stationaryDist(Theta, mu):
'''
Purpose:
Determine the stationary distribution of the Markov chain described by the transition matrix P
Input:
Theta - a (nx(n-1))-matrix (numpy array) with angles
mu - a (nx1)-vector (numpy array) for the Markov chain's 'start values' or None
Output:
pi - a (nx1)-vector (numpy array) with the (unique) steady state probabilities
'''
MC = MarkovChain(toP(Theta))
if MC.bUniChain: #P is a uni-chain
pi = MC.pi[0]
else: #P is a multi-chain
Pi = MC.Pi
pi = mu @ Pi
return pi
def objJ(Theta, r, subset, gamma, psi, Theta_ini):
'''
Purpose:
Compute the value of the objective function. The objective is to maximize pi[r] under the condition that the Markov chain remains irreducible,
for this the (relevant) values of M are used as penalty in the objective function.
Input:
Theta - a (nxn)-matrix (numpy array) of transition probabilities
r - indicates the row of the theta element we differentiate to, i.e., for Theta(r,c).
gamma - real number, the penalty factor for M
Output:
J - real number, the objective value
'''
MC = MarkovChain(toP(Theta))
pi = MC.pi[0]
#M = MC.M
P_first = toP(Theta_ini)
return pi[r] - psi * pen_subset(P_first, MC.P, subset, r)
def derivative_pi(P, r, c):
'''
Purpose:
Determine the derivative of the stationary distribution w.r.t. Theta(r,c)
Input:
P - a (nxn)-matrix (numpy array) of transition probabilities
r - indicates the row of the theta element we differentiate to, i.e., for Theta(r,c).
c - indicates the column of the theta element we differentiate to, i.e., for Theta(r,c).
Output:
deriv_pi - (nx1)-vector (numpy array) the derivative of pi
'''
n, _ = P.shape
MC = MarkovChain(P)
pi = MC.pi[0]
D = MC.D
deriv_P = derivativeP(P, r, c)
deriv_pi = pi @ deriv_P @ D
return deriv_pi
def derivativeD_H(Theta, r, c):
'''
Purpose:
Calculate the derivative of the deviation matrix of P w.r.t. Theta(r,c)
Input:
Theta - a (nx(n-1))-matrix (numpy array) with angles
r - indicates the row of the theta element we differentiate to, i.e., for Theta(r,c).
c - indicates the column of the theta element we differentiate to, i.e., for Theta(r,c).
Output:
derivD - (nxn)-matrix (numpy array) of the derivative of the D matrix
'''
n, _ = Theta.shape
P = toP(Theta)
MC = MarkovChain(P)
Pi = MC.Pi
D = MC.D
derivP = derivativeP(Theta, r, c)
derivPi = derivativePi(Theta, r, c)
derivD = -derivPi @ D + D @ derivP @ D
return derivD
def derivativeD(Theta, r, c):
'''
Purpose:
Calculate the derivative of the deviation matrix of P w.r.t. Theta(r,c)
Input:
Theta - a (nx(n-1))-matrix (numpy array) with angles
r - indicates the row of the theta element we differentiate to, i.e., for Theta(r,c).
c - indicates the column of the theta element we differentiate to, i.e., for Theta(r,c).
Output:
derivD - (nxn)-matrix (numpy array) of the derivative of the D matrix
'''
n, _ = Theta.shape
P = toP(Theta)
MC = MarkovChain(P)
Pi = MC.Pi
derivP = derivativeP(Theta, r, c)
derivPi = derivativePi(Theta, r, c)
inv_term = np.linalg.inv((np.identity(n) - P + Pi))
derivD = -inv_term @ (-derivP + derivPi) @ inv_term - derivPi
return derivD
def derivativePi(Theta, r, c):
'''
Purpose:
Calculate the derivative of the ergodic projector of P(Theta) w.r.t. Theta(r,c)
Input:
Theta - a (nx(n-1))-matrix (numpy array) with angles
r - indicates the row of the theta element we differentiate to, i.e., for Theta(r,c).
c - indicates the column of the theta element we differentiate to, i.e., for Theta(r,c).
Output:
derivPi - (nxn)-matrix (numpy array) of the derivative of the Pi matrix
'''
n, _ = Theta.shape
MC = MarkovChain(toP(Theta))
Pi = MC.Pi
if (MC.bUniChain == False):
print('The MC is not a unichain, derivative not yet determined')
exit()
#We have uni-chain
return np.array([derivative_pi(Theta, r, c)] * n)
def derivative_pi(Theta, r, c):
'''
Purpose:
Determine the derivative of the stationary distribution w.r.t. Theta(r,c)
Input:
Theta - a (nx(n-1))-matrix (numpy array) with angles
r - indicates the row of the theta element we differentiate to, i.e., for Theta(r,c).
c - indicates the column of the theta element we differentiate to, i.e., for Theta(r,c).
Output:
deriv_pi - (nx1)-vector (numpy array) the derivative of pi
'''
n, _ = Theta.shape
MC = MarkovChain(toP(Theta))
pi = MC.pi[0]
D = MC.D
deriv_P = derivativeP(Theta, r, c)
#deriv_P = np.zeros((n,n))
#deriv_P[r,c] = 1.0
deriv_pi = pi @ deriv_P @ D
return deriv_pi
def derivativeD(P, r, c):
'''
Purpose:
Calculate the derivative of the deviation matrix of P w.r.t. Theta(r,c)
Input:
P - a (nxn)-matrix (numpy array) of transition probabilities
r - indicates the row of the theta element we differentiate to, i.e., for Theta(r,c).
c - indicates the column of the theta element we differentiate to, i.e., for Theta(r,c).
Output:
derivD - (nxn)-matrix (numpy array) of the derivative of the D matrix
'''
n, _ = P.shape
MC = MarkovChain(P)
Pi = MC.Pi
derivP = derivativeP(P, r, c)
derivPi = derivativePi(P, r, c)
inv_term = np.linalg.inv((np.identity(n) - P + Pi))
derivD = -inv_term @ (
-derivP + derivPi
) @ inv_term - derivPi #Column r and row r are (for most) nonzero, does this make sense?
return derivD
def start(r):
#print ("===============================================================")
start_time = time.time()
####simulation####
#Initialize
n=8
MC = MarkovChain('Courtois')
P = MC.P
#P = np.ones((n,n))/n
#MC = MarkovChain(P)
#r = 2 #0,1,..,n-1 The state for which the stationary distribution will be maximized
#c = 5 #0, 1, .., n-2
Theta = toTheta(P)
epsilon = 0.0005
N = 1500 #Number of iterations
#Start
pi = simulation(Theta,r,epsilon,N)
print ('Final stationary distribution for r=%d:\n' % r, np.round(pi,3))
#Running time
print ('Running time of the programme:', time.time() - start_time)
print ("===============================================================")
def start():
print("===============================================================")
start_time = time.time()
####simulation####
#Initialize
n = 8
MC = MarkovChain('Courtois')
P = MC.P
#P = np.ones((n,n))/n
#MC = MarkovChain(P)
r = 1 #0,1,..,n-1 The state for which the stationary distribution will be maximized
#c = 5 #0, 1, .., n-2
Theta = toTheta(P)
gamma = 0.01
epsilon = 0.1
N = 5000 #Number of iterations
#Start
simulation(Theta, r, epsilon, gamma, N)
#Running time
print('Running time of the programme:', time.time() - start_time)
print("===============================================================")
def objJ(Theta, r, gamma):
'''
Purpose:
Compute the value of the objective function. The objective is to maximize pi[r] under the condition that the Markov chain remains irreducible,
for this the (relevant) values of M are used as penalty in the objective function.
Input:
Theta - a (nxn)-matrix (numpy array) of transition probabilities
r - indicates the row of the theta element we differentiate to, i.e., for Theta(r,c).
gamma - real number, the penalty factor for M
Output:
J - real number, the objective value
'''
n, _ = Theta.shape
MC = MarkovChain(toP(Theta))
pi = MC.pi[0]
M = MC.M
K = 10**5
pen1 = np.max(M[:, r] - np.ones(n) * K, 0)
pen2 = np.max(M[r, :] - np.ones(n) * K, 0)
return pi[r] - gamma * (
euclidean(pen1, np.zeros(n)) + euclidean(pen2, np.zeros(n))
) #gamma*sum(sum(M)) #gamma*(max(np.max(M[r]), np.max((M[:,r])), 1))
def spsa(mx, eta):
n = mx.shape[0]
delta = np.random.choice([-1, 1], (n, n))
P1, P2 = normalise1(mx + eta * delta), normalise1(mx - eta * delta)
K1, K2 = MarkovChain(P1).K, MarkovChain(P2).K
return (K1 - K2) / 2 * eta * delta
def SPSA_kem_sph(sph, eta):
#eta = 1e-4
Delta = np.random.choice([-1, 1], sph.shape) * eta
sph0, sph1 = sph + eta * Delta, sph - eta * Delta
K0, K1 = MarkovChain(sph_to_mx(sph0)).K, MarkovChain(sph_to_mx(sph1)).K
return K0, K1, (K0 - K1) / (2 * eta * Delta)