def SPSA_kem(V, normalise): eta = 1e-4 Delta = np.random.choice([-1,1], V.shape) * eta V0, V1 = V+eta*Delta, V-eta*Delta normalised0, normalised1 = normalise(V0), normalise(V1) K0, K1 = MarkovChain(normalised0).K, MarkovChain(normalised1).K return (K0-K1)/(2*eta*Delta)
def derivFD_K(P, normalise, i ,j): h=1e-4 P1 = P.copy() P1[i,j] += h #P[i,j] -= h K1 = MarkovChain(normalise(P1)).K K2 = MarkovChain(normalise(P)).K return (K1-K2)/2*h
def main(): print("===============================================================") start_time = time.time() ####simulation#### #Initialize n = 4 P = np.ones((n, n)) / n MC = MarkovChain('Courtois') P = MC.P r = 2 #0,1,..,n-1 The state for which the stationary distribution will be maximized c = 1 #0, 1, .., n-2 Theta = toTheta(P) epsilon = 0.2 gamma = 0.001 #at M = 10/gamma (=100) the effect of M to the objective function will now start to become noticable N = 500 #Number of iterations #Start print(np.round(P, 3)) print(np.round(MC.pi, 3)) simulation(P, r, epsilon, gamma, N) #Running time print('Running time of the programme:', time.time() - start_time) print("===============================================================")
def derivativeM(P, r, c): ''' Purpose: Calculate the derivative of the mean first passage time matrix of P(Theta) w.r.t. Theta(r,c) Input: P - a (nxn)-matrix (numpy array) of transition probabilities r - indicates the row of the theta element we differentiate to, i.e., for Theta(r,c). c - indicates the column of the theta element we differentiate to, i.e., for Theta(r,c). Output: derivM - (nxn)-matrix (numpy array) of the derivative of the M matrix ''' n, _ = P.shape Ones = np.ones((n, n)) MC = MarkovChain(P) pi = MC.pi[0] Pi = MC.Pi D = MC.D dg_Pi = diag(Pi) dg_D = diag(D) derivD = derivativeD(P, r, c) dg_derivD = diag(derivD) #derivative of the inverse of dg_Pi deriv_dg_Pi_inv = np.zeros((n, n)) deriv_dg_Pi_inv[r, r] = -1 * (derivative_pi(P, r, c)[r]) / (pi[r]**2) #using derivative of M formula derivM = (-derivD + Ones @ dg_derivD) @ np.linalg.inv( dg_Pi) + deriv_dg_Pi_inv @ ( np.identity(n) - D + Ones @ dg_D ) #Column r and row r are (for most) nonzero, does this make sense? return derivM
def Kemeny(P): with torch.no_grad(): tmp = P.clone().detach().numpy() tmp = torch.from_numpy(tmp) K = torch.FloatTensor(MarkovChain(tmp).K) return K
def derivativeM(Theta, r, c, Delta): ''' Purpose: Calculate the derivative of the mean first passage time matrix of P(Theta) w.r.t. Theta(r,c) Input: Theta - a (nx(n-1))-matrix (numpy array) with angles r - indicates the row of the theta element we differentiate to, i.e., for Theta(r,c). c - indicates the column of the theta element we differentiate to, i.e., for Theta(r,c). Output: derivM - (nxn)-matrix (numpy array) of the derivative of the M matrix ''' n, _ = Theta.shape Ones = np.ones((n, n)) MC = MarkovChain(toP(Theta)) pi = MC.pi[0] Pi = MC.Pi D = MC.D dg_Pi = diag(Pi) dg_D = diag(D) #derivD = derivativeD(Theta,r,c) #derivD = derivativeD_SPSA(Theta,r,c, Delta) derivD = derivativeD_H(Theta, r, c) dg_derivD = diag(derivD) #derivative of the inverse of dg_Pi deriv_dg_Pi_inv = np.zeros((n, n)) deriv_dg_Pi_inv[r, r] = -1 * (derivative_pi(Theta, r, c)[r]) / (pi[r]**2) #using derivative of M formula derivM = (-derivD + Ones @ dg_derivD) @ np.linalg.inv( dg_Pi) + deriv_dg_Pi_inv @ (np.identity(n) - D + Ones @ dg_D) return derivM
def Kemeny(indices, values, size, tol=1e-8): """Calculate the Kemeny constant.""" values[abs(values) < tol] = 0.0 #cast values close to zero to zero. #NOTE: this is no longer needed if we do not allow values to drop below eps! P = torch.sparse.FloatTensor(indices, values, size) MC = MarkovChain(P.detach().to_dense().numpy()) return torch.FloatTensor([MC.K])
def start(): print("===============================================================") start_time = time.time() ####simulation#### #Initialize n = 8 MC = MarkovChain('Courtois') P = MC.P #P = np.ones((n,n))/n #MC = MarkovChain(P) r = 0 #0,1,..,n-1 The state for which the stationary distribution will be maximized #c = 5 #0, 1, .., n-2 Theta = toTheta(P) epsilon = 0.1 gamma = 0.0005 N = 300 #Number of iterations #Start print('###########') print('First stationary probabilities:\n', MC.pi[0]) print('First Objective value:', objJ(Theta, r, gamma)) print('First transition probabilities:\n', np.round(MC.P, 3)) #print ('First M value:\n', np.round(MC.M*gamma,2)) simulation(Theta, r, epsilon, gamma, N) #Running time print('Running time of the programme:', time.time() - start_time) print("===============================================================")
def start(): print("===============================================================") start_time = time.time() ####simulation#### #Initialize n = 4 MC = MarkovChain('Courtois') P = MC.P #P = np.ones((n,n))/n #MC = MarkovChain(P) r = 1 #0,1,..,n-1 The state for which the stationary distribution will be maximized #c = 5 #0, 1, .., n-2 Theta = toTheta(P) epsilon = 0.1 #Epsilon = 0.005 and update (>200) epsilon decreasing factor with factor 10 worked for all r in example Courtois gamma = 10**-6 N = 300 #Number of iterations subset = [0] psi = 3 simulation(Theta, r, epsilon, subset, gamma, psi, N) #Running time print('Running time of the programme:', time.time() - start_time) print("===============================================================")
def Kemeny_spsa(indices, values, size, eta): """Calculate gradient of Kemeny constant using SPSA.""" #NOTE: the gradients are only calculated for the edges in the network. n = values.shape[0] delta = np.random.choice([-1, 1], n) #is this implementation oke? Using clamp(0,1), but also using SPSA for not all params. #Use one of the normalisation functions! values1 = (values + eta * delta).clamp(0, 1) values2 = (values - eta * delta).clamp(0, 1) P1 = torch.sparse.FloatTensor(indices, values1, size) P2 = torch.sparse.FloatTensor(indices, values2, size) K1 = MarkovChain(P1.to_dense().numpy()).K K2 = MarkovChain(P2.to_dense().numpy()).K grads = torch.FloatTensor([(K1 - K2) / 2 * eta * delta]) return grads
def simulation(Theta, r, epsilon, gamma, N): ''' Purpose: perform a simulation to optimize pi[r] ''' P = toP(Theta) P_old = P n, _ = P.shape mTheta_changes = np.zeros((N, n - 1)) mStatDist_changes = np.zeros((N, n)) vObj_changes = np.zeros(N) vPr_changes = np.zeros(N) pi_old = stationaryDist(Theta, None) pi_new = pi_old for step_nr in range(N): if step_nr > 150: epsilon = 1 / (step_nr) if step_nr > 300: epsilon = 1 / (2 * step_nr) if step_nr > 450: epsilon = 1 / (4 * step_nr) if step_nr > 600: epsilon = 1 / (8 * step_nr) if step_nr > 750: epsilon = 1 / (16 * step_nr) if step_nr > 900: epsilon = 1 / (32 * step_nr) v_gradJ = np.zeros(n - 1) #store derivJ for all Theta[r,.] angles for c in range(n - 1): #c represents angle index derivJ = deriv_objJ(Theta, r, c, gamma) v_gradJ[c] = derivJ #Update row Theta[r] #Theta[r] = Theta[r] + epsilon*v_gradJ #Update singel Theta[r,c_max] c_max = np.argmax(abs(v_gradJ)) Theta[r, c_max] = Theta[r, c_max] + epsilon * v_gradJ[c_max] #Update MC measures P = toP(Theta) pi = stationaryDist(Theta, None) MC = MarkovChain(P) #Track changes mStatDist_changes[step_nr] = pi vObj_changes[step_nr] = objJ(Theta, r, gamma) mTheta_changes[step_nr] = Theta[r] vPr_changes[step_nr] = P[r, r] print('==========') print('theta[r]=', np.round(Theta, 3)[r]) print('Gradient Obj=', np.round(v_gradJ, 3)) print('Gradient Obj x Epsilon=', np.round(v_gradJ * epsilon, 3)) print('epsilon=', epsilon) print('iteration=', step_nr) print('pi_new', np.round(pi_new, 4)) print('P[r]=', np.round(P[r], 3)) print('Obj=', np.round(vObj_changes[step_nr], 3)) print('==========') print('Maximum M value=', np.max(MC.M)) print('Final self probability=', round(P[r, r], 3)) plotThetaprocess(mTheta_changes, mStatDist_changes, vObj_changes, vPr_changes, step_nr, r)
def derivativeD_SPSA(Theta, r, c, Delta): ''' Purpose: Estimate the derivative w.r.t. Theta(r,c) of the deviation matrix using SPSA Input: Theta - a (nx(n-1))-matrix (numpy array) with angles r - indicates the row of the theta element we differentiate to, i.e., for Theta(r,c). c - indicates the column of the theta element we differentiate to, i.e., for Theta(r,c). Output: derivD - (nxn)-matrix (numpy array) of the derivative of the D matrix ''' n, _ = Theta.shape nabla = 0.0001 Theta1 = copy.deepcopy(Theta) Theta2 = copy.deepcopy(Theta) Theta1[r] = Theta[r] - nabla * Delta Theta2[r] = Theta[r] + nabla * Delta D1 = MarkovChain(toP(Theta1)).D D2 = MarkovChain(toP(Theta2)).D derivD = (D2 - D1) / (2 * nabla * Delta[c]) return derivD
def simulation(P, r, epsilon, gamma, N): ''' Purpose: perform a simulation to optimize pi[r] ''' Theta = toTheta(P) n, _ = P.shape mTheta_changes = np.zeros((N, n - 1)) mStatDist_changes = np.zeros((N, n)) for step_nr in range(N): v_derivJ = np.zeros(n - 1) #store derivJ for all Theta[r,.] angles for c in range(n - 1): #c represents angle index derivJ = deriv_objJ(P, r, c, gamma) v_derivJ[c] = derivJ #find angle with highest effect on the objective value c_max = np.argmax(abs(v_derivJ)) #c_change = random.randint(0,n-2) #Update angle Theta[r,c_max], we are maximizing Theta[r, c_max] = Theta[r, c_max] + epsilon * v_derivJ[c_max] mTheta_changes[step_nr] = Theta[r] #New P matrix P = toP(Theta) MC = MarkovChain(P) mStatDist_changes[step_nr] = MC.pi[0] #stop condition for reducibility if (MC.bUniChain == False): print( 'The Markov chain described by P is no longer uni-chain! This should not happen.' ) exit() #print current progress print('r=', r) print('gradient obj=', v_derivJ) print('P leaving state r=', P[r]) print('P entering state r=', P[:, r]) print('pi=', MC.pi[0]) print('\n') #print Final progress (again) print('\n\n\n') print('Final stationary probabilities:', MC.pi[0]) print('Final Objective value:', objJ(P, r, gamma)) print('Final transition probabilities:\n', np.round(P, 3)) print('Final M value:\n', MC.M) plotThetaprocess(mTheta_changes, mStatDist_changes)
def objJ(Theta, r): ''' Purpose: Compute the value of the objective function. The objective is to maximize pi[r] under the condition that the Markov chain remains irreducible, for this the (relevant) values of M are used as penalty in the objective function. Input: Theta - a (nxn)-matrix (numpy array) of transition probabilities r - indicates the row of the theta element we differentiate to, i.e., for Theta(r,c). Output: J - real number, the objective value ''' MC = MarkovChain(toP(Theta)) pi = MC.pi[0] return pi[r]
def objJ(P, r, gamma): ''' Purpose: Compute the value of the objective function. The objective is to maximize pi[r] under the condition that the Markov chain remains irreducible, for this the (relevant) values of M are used as penalty in the objective function. Input: P - a (nxn)-matrix (numpy array) of transition probabilities r - indicates the row of the theta element we differentiate to, i.e., for Theta(r,c). gamma - real number, the penalty factor for M Output: J - real number, the objective value ''' MC = MarkovChain(P) pi = MC.pi[0] M = MC.M return pi[r] #+ gamma*(sum(M[r]) + sum(M[:,r]))
def plot(): n = 500 Theta = toTheta(MarkovChain('Courtois').P) theta_rnd = np.array([0] + list(np.sort(np.random.rand(n - 2) * np.pi / 2)) + [np.pi / 2]) mObj = np.zeros((n, 2)) vP = np.zeros(n) gamma = 0.0000005 r = 7 ''' for i in range(n): Theta_tmp1 = Theta Theta_tmp1[r,6] = theta_rnd[i] mObj[i,0] = objJ(Theta_tmp1,r,gamma) vP[i] = toP(Theta_tmp1)[r,r] #Theta_tmp2 = Theta #Theta_tmp2[r,5] = theta_rnd[i] #Theta_tmp3 = Theta #Theta_tmp3[r,4] = theta_rnd[i] #Theta_tmp4 = Theta #Theta_tmp4[r,3] = theta_rnd[i] #Theta_tmp5 = Theta #Theta_tmp5[r,2] = theta_rnd[i] #Theta_tmp6 = Theta #Theta_tmp6[r,1] = theta_rnd[i] #Theta_tmp7 = Theta #Theta_tmp7[r,0] = theta_rnd[i] # mObj[i,1] = objJ(Theta_tmp2,r,gamma) plt.plot(theta_rnd, vP, label ='P[r,r]') plt.legend() plt.show() plt.plot(theta_rnd, mObj[:,0], label='J1') plt.legend() plt.show() ''' mDerivJ = np.zeros((n, 7)) for i in range(n): Theta_tmp1 = Theta Theta_tmp1[r, 0] = theta_rnd[i] for c in range(7): mDerivJ[i, c] = deriv_objJ(Theta_tmp1, r, c, gamma) for c in range(7): plt.plot(theta_rnd, mDerivJ[:, c], label=c) plt.legend() plt.show()
def simulation(Theta, r, epsilon, gamma, N): ''' Purpose: perform a simulation to optimize pi[r] ''' P = toP(Theta) n, _ = P.shape mTheta_changes = np.zeros((N, n - 1)) mStatDist_changes = np.zeros((N, n)) mObj_changes = np.zeros(N) for step_nr in range(N): if (step_nr > 100): epsilon = 1 / (step_nr) v_derivJ = np.zeros(n - 1) #store derivJ for all Theta[r,.] angles for c in range(n - 1): #c represents angle index derivJ = deriv_objJ(Theta, r, c, gamma) v_derivJ[c] = derivJ #find angle with highest effect on the objective value c_max = np.argmax(abs(v_derivJ)) #c_max = np.random.randint(n-1) #Update angle Theta[r,c_max], we are maximizing Theta[r, c_max] = Theta[r, c_max] + epsilon * v_derivJ[c_max] mTheta_changes[step_nr] = Theta[r] #New P matrix P = toP(Theta) MC = MarkovChain(P) mStatDist_changes[step_nr] = MC.pi[0] mObj_changes[step_nr] = objJ(Theta, r, gamma) print('==========') print('iteration=', step_nr) ''' print ('epsilon=', epsilon) print ('P[r]=', np.round(P[r],3)) print ('theta[r]=', np.round(mTheta_changes[step_nr],3)) print ('pi=', np.round(mStatDist_changes[step_nr],3)) print ('Derivative Obj=', np.round(v_derivJ,3)) print ('c max=', c_max) print ('Obj=', np.round(mObj_changes[step_nr],3)) print ('==========') ''' #print and plot final results print('Final Objective value:', objJ(Theta, r, gamma)) print('Final transition probabilities:\n', np.round(MC.P, 3)) print('Final stationary probabilities:\n', MC.pi[0]) #print ('Final M value:\n', np.round(MC.M*gamma,2)) plotThetaprocess(mTheta_changes, mStatDist_changes, mObj_changes)
def stationaryDist(Theta, mu): ''' Purpose: Determine the stationary distribution of the Markov chain described by the transition matrix P Input: Theta - a (nx(n-1))-matrix (numpy array) with angles mu - a (nx1)-vector (numpy array) for the Markov chain's 'start values' or None Output: pi - a (nx1)-vector (numpy array) with the (unique) steady state probabilities ''' MC = MarkovChain(toP(Theta)) if MC.bUniChain: #P is a uni-chain pi = MC.pi[0] else: #P is a multi-chain Pi = MC.Pi pi = mu @ Pi return pi
def objJ(Theta, r, subset, gamma, psi, Theta_ini): ''' Purpose: Compute the value of the objective function. The objective is to maximize pi[r] under the condition that the Markov chain remains irreducible, for this the (relevant) values of M are used as penalty in the objective function. Input: Theta - a (nxn)-matrix (numpy array) of transition probabilities r - indicates the row of the theta element we differentiate to, i.e., for Theta(r,c). gamma - real number, the penalty factor for M Output: J - real number, the objective value ''' MC = MarkovChain(toP(Theta)) pi = MC.pi[0] #M = MC.M P_first = toP(Theta_ini) return pi[r] - psi * pen_subset(P_first, MC.P, subset, r)
def derivative_pi(P, r, c): ''' Purpose: Determine the derivative of the stationary distribution w.r.t. Theta(r,c) Input: P - a (nxn)-matrix (numpy array) of transition probabilities r - indicates the row of the theta element we differentiate to, i.e., for Theta(r,c). c - indicates the column of the theta element we differentiate to, i.e., for Theta(r,c). Output: deriv_pi - (nx1)-vector (numpy array) the derivative of pi ''' n, _ = P.shape MC = MarkovChain(P) pi = MC.pi[0] D = MC.D deriv_P = derivativeP(P, r, c) deriv_pi = pi @ deriv_P @ D return deriv_pi
def derivativeD_H(Theta, r, c): ''' Purpose: Calculate the derivative of the deviation matrix of P w.r.t. Theta(r,c) Input: Theta - a (nx(n-1))-matrix (numpy array) with angles r - indicates the row of the theta element we differentiate to, i.e., for Theta(r,c). c - indicates the column of the theta element we differentiate to, i.e., for Theta(r,c). Output: derivD - (nxn)-matrix (numpy array) of the derivative of the D matrix ''' n, _ = Theta.shape P = toP(Theta) MC = MarkovChain(P) Pi = MC.Pi D = MC.D derivP = derivativeP(Theta, r, c) derivPi = derivativePi(Theta, r, c) derivD = -derivPi @ D + D @ derivP @ D return derivD
def derivativeD(Theta, r, c): ''' Purpose: Calculate the derivative of the deviation matrix of P w.r.t. Theta(r,c) Input: Theta - a (nx(n-1))-matrix (numpy array) with angles r - indicates the row of the theta element we differentiate to, i.e., for Theta(r,c). c - indicates the column of the theta element we differentiate to, i.e., for Theta(r,c). Output: derivD - (nxn)-matrix (numpy array) of the derivative of the D matrix ''' n, _ = Theta.shape P = toP(Theta) MC = MarkovChain(P) Pi = MC.Pi derivP = derivativeP(Theta, r, c) derivPi = derivativePi(Theta, r, c) inv_term = np.linalg.inv((np.identity(n) - P + Pi)) derivD = -inv_term @ (-derivP + derivPi) @ inv_term - derivPi return derivD
def derivativePi(Theta, r, c): ''' Purpose: Calculate the derivative of the ergodic projector of P(Theta) w.r.t. Theta(r,c) Input: Theta - a (nx(n-1))-matrix (numpy array) with angles r - indicates the row of the theta element we differentiate to, i.e., for Theta(r,c). c - indicates the column of the theta element we differentiate to, i.e., for Theta(r,c). Output: derivPi - (nxn)-matrix (numpy array) of the derivative of the Pi matrix ''' n, _ = Theta.shape MC = MarkovChain(toP(Theta)) Pi = MC.Pi if (MC.bUniChain == False): print('The MC is not a unichain, derivative not yet determined') exit() #We have uni-chain return np.array([derivative_pi(Theta, r, c)] * n)
def derivative_pi(Theta, r, c): ''' Purpose: Determine the derivative of the stationary distribution w.r.t. Theta(r,c) Input: Theta - a (nx(n-1))-matrix (numpy array) with angles r - indicates the row of the theta element we differentiate to, i.e., for Theta(r,c). c - indicates the column of the theta element we differentiate to, i.e., for Theta(r,c). Output: deriv_pi - (nx1)-vector (numpy array) the derivative of pi ''' n, _ = Theta.shape MC = MarkovChain(toP(Theta)) pi = MC.pi[0] D = MC.D deriv_P = derivativeP(Theta, r, c) #deriv_P = np.zeros((n,n)) #deriv_P[r,c] = 1.0 deriv_pi = pi @ deriv_P @ D return deriv_pi
def derivativeD(P, r, c): ''' Purpose: Calculate the derivative of the deviation matrix of P w.r.t. Theta(r,c) Input: P - a (nxn)-matrix (numpy array) of transition probabilities r - indicates the row of the theta element we differentiate to, i.e., for Theta(r,c). c - indicates the column of the theta element we differentiate to, i.e., for Theta(r,c). Output: derivD - (nxn)-matrix (numpy array) of the derivative of the D matrix ''' n, _ = P.shape MC = MarkovChain(P) Pi = MC.Pi derivP = derivativeP(P, r, c) derivPi = derivativePi(P, r, c) inv_term = np.linalg.inv((np.identity(n) - P + Pi)) derivD = -inv_term @ ( -derivP + derivPi ) @ inv_term - derivPi #Column r and row r are (for most) nonzero, does this make sense? return derivD
def start(r): #print ("===============================================================") start_time = time.time() ####simulation#### #Initialize n=8 MC = MarkovChain('Courtois') P = MC.P #P = np.ones((n,n))/n #MC = MarkovChain(P) #r = 2 #0,1,..,n-1 The state for which the stationary distribution will be maximized #c = 5 #0, 1, .., n-2 Theta = toTheta(P) epsilon = 0.0005 N = 1500 #Number of iterations #Start pi = simulation(Theta,r,epsilon,N) print ('Final stationary distribution for r=%d:\n' % r, np.round(pi,3)) #Running time print ('Running time of the programme:', time.time() - start_time) print ("===============================================================")
def start(): print("===============================================================") start_time = time.time() ####simulation#### #Initialize n = 8 MC = MarkovChain('Courtois') P = MC.P #P = np.ones((n,n))/n #MC = MarkovChain(P) r = 1 #0,1,..,n-1 The state for which the stationary distribution will be maximized #c = 5 #0, 1, .., n-2 Theta = toTheta(P) gamma = 0.01 epsilon = 0.1 N = 5000 #Number of iterations #Start simulation(Theta, r, epsilon, gamma, N) #Running time print('Running time of the programme:', time.time() - start_time) print("===============================================================")
def objJ(Theta, r, gamma): ''' Purpose: Compute the value of the objective function. The objective is to maximize pi[r] under the condition that the Markov chain remains irreducible, for this the (relevant) values of M are used as penalty in the objective function. Input: Theta - a (nxn)-matrix (numpy array) of transition probabilities r - indicates the row of the theta element we differentiate to, i.e., for Theta(r,c). gamma - real number, the penalty factor for M Output: J - real number, the objective value ''' n, _ = Theta.shape MC = MarkovChain(toP(Theta)) pi = MC.pi[0] M = MC.M K = 10**5 pen1 = np.max(M[:, r] - np.ones(n) * K, 0) pen2 = np.max(M[r, :] - np.ones(n) * K, 0) return pi[r] - gamma * ( euclidean(pen1, np.zeros(n)) + euclidean(pen2, np.zeros(n)) ) #gamma*sum(sum(M)) #gamma*(max(np.max(M[r]), np.max((M[:,r])), 1))
def spsa(mx, eta): n = mx.shape[0] delta = np.random.choice([-1, 1], (n, n)) P1, P2 = normalise1(mx + eta * delta), normalise1(mx - eta * delta) K1, K2 = MarkovChain(P1).K, MarkovChain(P2).K return (K1 - K2) / 2 * eta * delta
def SPSA_kem_sph(sph, eta): #eta = 1e-4 Delta = np.random.choice([-1, 1], sph.shape) * eta sph0, sph1 = sph + eta * Delta, sph - eta * Delta K0, K1 = MarkovChain(sph_to_mx(sph0)).K, MarkovChain(sph_to_mx(sph1)).K return K0, K1, (K0 - K1) / (2 * eta * Delta)