def calcMultiStateEnergy(ensemble, ensemble_ref, cutoff=12, k=1, U0=None): import prody as pd import scipy as sci import scipy.spatial as sp import numpy as np n_conf = ensemble.numConfs() n_ref = ensemble_ref.numConfs() energies = np.zeros((n_ref, n_conf)) for i, conf_ref in zip(xrange(n_ref), ensemble_ref.iterCoordsets()): gnm = pd.GNM() gnm.buildKirchhoff(conf_ref, cutoff, k) kirchhoff = gnm.getKirchhoff() np.fill_diagonal(kirchhoff, 0) kirchhoff = abs(kirchhoff) eq_dists = sp.distance.squareform(sp.distance.pdist(conf_ref, metric="euclidean")) for j, conf in zip(xrange(n_conf), ensemble.iterCoordsets()): dists = sp.distance.squareform(sp.distance.pdist(conf, metric="euclidean")) springs = (k / 2) * np.multiply(np.square(dists - eq_dists), kirchhoff) springs = sci.triu(springs) energies[i, j] = np.sum(springs) # return np.min(energies, axis=0) return energies
def __call__(self, A, y, sigma, rng=None): m, n = A.shape transpose = self.transpose if transpose is None: # transpose if matrix is fat, but not if sigmas for each neuron transpose = m < n and sigma.size == 1 if transpose: # substitution: x = A'*xbar, G*xbar = b where G = A*A' + lambda*I G = np.dot(A, A.T) b = y else: # multiplication by A': G*x = A'*b where G = A'*A + lambda*I G = np.dot(A.T, A) b = np.dot(A.T, y) # add L2 regularization term 'lambda' = m * sigma**2 np.fill_diagonal(G, G.diagonal() + m * sigma**2) try: import scipy.linalg factor = scipy.linalg.cho_factor(G, overwrite_a=True) x = scipy.linalg.cho_solve(factor, b) except ImportError: L = np.linalg.cholesky(G) L = np.linalg.inv(L.T) x = np.dot(L, np.dot(L.T, b)) x = np.dot(A.T, x) if transpose else x info = {'rmses': rmses(A, x, y)} return x, info
def test_binary_perplexity_stability(): # Binary perplexity search should be stable. # The binary_search_perplexity had a bug wherein the P array # was uninitialized, leading to sporadically failing tests. k = 10 n_samples = 100 random_state = check_random_state(0) distances = random_state.randn(n_samples, 2).astype(np.float32) # Distances shouldn't be negative distances = np.abs(distances.dot(distances.T)) np.fill_diagonal(distances, 0.0) last_P = None neighbors_nn = np.argsort(distances, axis=1)[:, :k].astype(np.int64) for _ in range(100): P = _binary_search_perplexity(distances.copy(), neighbors_nn.copy(), 3, verbose=0) P1 = _joint_probabilities_nn(distances, neighbors_nn, 3, verbose=0) # Convert the sparse matrix to a dense one for testing P1 = P1.toarray() if last_P is None: last_P = P last_P1 = P1 else: assert_array_almost_equal(P, last_P, decimal=4) assert_array_almost_equal(P1, last_P1, decimal=4)
def cholesky(A, y, sigma, transpose=None): """Solve the least-squares system using the Cholesky decomposition.""" m, n = A.shape if transpose is None: # transpose if matrix is fat, but not if we have sigmas for each neuron transpose = m < n and sigma.size == 1 if transpose: # substitution: x = A'*xbar, G*xbar = b where G = A*A' + lambda*I G = np.dot(A, A.T) b = y else: # multiplication by A': G*x = A'*b where G = A'*A + lambda*I G = np.dot(A.T, A) b = np.dot(A.T, y) # add L2 regularization term 'lambda' = m * sigma**2 np.fill_diagonal(G, G.diagonal() + m * sigma**2) try: import scipy.linalg factor = scipy.linalg.cho_factor(G, overwrite_a=True) x = scipy.linalg.cho_solve(factor, b) except ImportError: L = np.linalg.cholesky(G) L = np.linalg.inv(L.T) x = np.dot(L, np.dot(L.T, b)) x = np.dot(A.T, x) if transpose else x info = {'rmses': npext.rms(y - np.dot(A, x), axis=0)} return x, info
def test_gradient(): # Test gradient of Kullback-Leibler divergence. random_state = check_random_state(0) n_samples = 50 n_features = 2 n_components = 2 alpha = 1.0 distances = random_state.randn(n_samples, n_features).astype(np.float32) distances = np.abs(distances.dot(distances.T)) np.fill_diagonal(distances, 0.0) X_embedded = random_state.randn(n_samples, n_components).astype(np.float32) P = _joint_probabilities(distances, desired_perplexity=25.0, verbose=0) def fun(params): return _kl_divergence(params, P, alpha, n_samples, n_components)[0] def grad(params): return _kl_divergence(params, P, alpha, n_samples, n_components)[1] assert_almost_equal(check_grad(fun, grad, X_embedded.ravel()), 0.0, decimal=5)
def build_bucket_correlation(pos_delta, params, margin): risk_class = pos_delta.RiskClass.unique()[0] g = 0 if risk_class == 'IR': all_curr = pos_delta.Group.unique() g = np.ones((len(all_curr), len(all_curr))) if not margin == 'Curvature': for i in range(len(all_curr)): for j in range(len(all_curr)): CRi = pos_delta.iloc[[i]].CR.values[0] CRj = pos_delta.iloc[[j]].CR.values[0] g[i][j] = min(CRi, CRj) / max(CRi, CRj) g = g * params.IR_Gamma elif risk_class == 'CreditQ': g = params.CreditQ_Corr elif risk_class == 'CreditNonQ': g = params.CreditNonQ_Corr elif risk_class == 'Equity': g = params.Equity_Corr elif risk_class == 'Commodity': g = params.Commodity_Corr if margin == 'Curvature': g = pow(g, 2) g = np.mat(g) np.fill_diagonal(g, 0) return g
def calc_lik_bin(j,L): i = recursive[j] Q = Q_list[Q_index[i]] r_vec=r_vec_list[Q_index[i]] t=delta_t[i] r_ind= r_vec_indexes[i] sign= sign_list[i] rho_vec= np.prod(abs(sign-r_vec[r_ind]),axis=1) # Pt = np.ones((4,4)) # Pt= linalg.expm(Q.T *(t)) w, vl = scipy.linalg.eig(Q,left=True, right=False) # w = eigenvalues # vl = eigenvectors vl_inv = np.linalg.inv(vl) d= exp(w*t) m1 = np.zeros((4,4)) np.fill_diagonal(m1,d) Pt1 = np.dot(vl,m1) Pt = np.dot(Pt1,vl_inv) #print vl, m1, vl_inv PvDes_temp = L[j,:] condLik_temp= np.dot(PvDes_temp,Pt) PvDes= condLik_temp *rho_vec L[j+1,:]= PvDes
def test_boolean_inputs(self): w = np.ones((135,135), dtype=np.bool) np.fill_diagonal(w, False) la = LinearAssignment(w) #if the input doesn't get converted to a float, the masking #doesn't work properly self.assertEqual(la.orig_c.dtype, np.float64)
def expm(A, n_factors=None, normalize=False): """Simple matrix exponential to replace Scipy's matrix exponential This just uses a recursive (factored) version of the Taylor series, and is not as good as Scipy (which uses Pade approximants). The hard part with this implementation is choosing the length of the Taylor series. A longer series is generally needed for a matrix with a larger eigenvalues, but I'm not exactly sure how these relate. I'm using a heuristic based on the matrix norm, since this is kind-of related to the size of eigenvalues, though for larger norms the function becomes inaccurate no matter the length of the series. This function is mostly intended for use in `filter_design`, where the matrices should be small, both in dimensions and norm. """ if A.ndim != 2 or A.shape[0] != A.shape[1]: raise ValueError("Argument must be a square matrix") a = np.linalg.norm(A) if normalize: a = int(a) A = A / float(a) if n_factors is None: n_factors = 20 if normalize else max(20, int(a)) Y = np.zeros_like(A) for i in range(n_factors, 0, -1): Y = np.dot(A / float(i), Y) np.fill_diagonal(Y, Y.diagonal() + 1) # add identity matrix return np.linalg.matrix_power(Y, a) if normalize else Y
def integration(MA,systemByNode): ''' INTEGRATION Integration coefficient I = INTEGRATION(MA,systemByNode) calculates the integration coefficient for each node of the network. The integration coefficient of a region corresponds to the average probability that this region is in the same network community as regions from other systems. Inputs: MA, Module Allegiance matrix, where element (i,j) represents the probability that nodes i and j belong to the same community systemByNode, vector or cell array containing the system assignment for each node Outputs: I, integration coefficient for each node _______________________________________________ Marcelo G Mattar (08/21/2014) ''' # Initialize output I = np.zeros(shape=(len(systemByNode),1), dtype = np.double) # Make sure the diagonal of the module allegiance is all nan MA = np.double(MA) np.fill_diagonal(MA, np.nan) # Calculate the integration for each node for i in range(systemByNode.size): thisSystem = systemByNode[i] I[i] = np.nanmean(MA[i,systemByNode!=thisSystem]) return I
def random_cov(d, diff=None): """Generate random covariance matrix. Generates a random covariance matrix, or two dependent covariance matrices if the argument `diff` is given. """ S = 0.8*np.random.randn(d,d) copy_triu_to_tril(S) np.fill_diagonal(S,0) mineig = linalg.eigvalsh(S, eigvals=(0,0))[0] drand = 0.8*np.random.randn(d) if mineig < 0: S += np.diag(np.exp(drand)-mineig) else: S += np.diag(np.exp(drand)) if not diff: return S.T S2 = S * np.random.randint(2, size=(d,d))*np.exp(diff*np.random.randn(d,d)) copy_triu_to_tril(S2) np.fill_diagonal(S2,0) mineig = linalg.eigvalsh(S2, eigvals=(0,0))[0] drand += diff*np.random.randn(d) if mineig < 0: S2 += np.diag(np.exp(drand)-mineig) else: S2 += np.diag(np.exp(drand)) return S.T, S2.T
def detect_duplicates(file_name, dist_thr=0.1, FOV=(512, 512)): """ Removes duplicate ROIs from file file_name Parameters: ----------- file_name: .zip file with all rois dist_thr: distance threshold for duplicate detection FOV: dimensions of the FOV Returns: -------- duplicates : list of indeces with duplicate entries ind_keep : list of kept indeces """ rois = nf_read_roi_zip(file_name, FOV) cm = [scipy.ndimage.center_of_mass(mm) for mm in rois] sp_rois = scipy.sparse.csc_matrix( np.reshape(rois, (rois.shape[0], np.prod(FOV))).T) D = distance_masks([sp_rois, sp_rois], [cm, cm], 10)[0] np.fill_diagonal(D, 1) indeces = np.where(D < dist_thr) # pairs of duplicate indeces ind = list(np.unique(indeces[1][indeces[1] > indeces[0]])) ind_keep = list(set(range(D.shape[0])) - set(ind)) duplicates = list(np.unique(np.concatenate((indeces[0], indeces[1])))) return duplicates, ind_keep
def recruitment(MA,systemByNode): ''' RECRUITMENT Recruitment coefficient R = RECRUITMENT(MA,systemByNode) calculates the recruitment coefficient for each node of the network. The recruitment coefficient of a region corresponds to the average probability that this region is in the same network community as other regions from its own system. Inputs: MA, Module Allegiance matrix, where element (i,j) represents the probability that nodes i and j belong to the same community systemByNode, vector or cell array containing the system assignment for each node Outputs: R, recruitment coefficient for each node _______________________________________________ Marcelo G Mattar (08/21/2014) ''' # Initialize output R = np.zeros(shape=(systemByNode.size,1), dtype = np.double); # Make sure the diagonal of the module allegiance is all nan MA = np.double(MA) np.fill_diagonal(MA, np.nan) # Calculate the recruitment for each node for i in range(systemByNode.size): thisSystem = systemByNode[i] R[i] = np.nanmean(MA[i,systemByNode==thisSystem]) return R
def condense_matrix(matrice, smallest_index, method='upgma'): """Matrice condensation in the next iteration Smallest index is returned from find_smallest_index. For both leaf at i and j a new distance is calculated from the average of the corresponding row or the corresponding columns We then replace the first index (row and column) by the average vector obtained and the second index by an array with large numbers so that it is never chosen again with find_smallest_index. Now the new regroupement distance value is at the first position! (on row and column) """ first_index, second_index = smallest_index # get the rows and make a new vector by updating distance rows = np.take(matrice, smallest_index, 1) # default we use upgma if(method.lower() == 'nj'): new_vector = ( np.sum(rows, 1) - matrice[first_index, second_index]) * 0.5 else: new_vector = np.average(rows, 1) # replace info in the row and column for first index with new_vector matrice[second_index] = new_vector matrice[:, second_index] = new_vector np.fill_diagonal(matrice, 0) # replace the info in the row and column for the second index with # high numbers so that it is ignored return remove_ij(matrice, first_index, first_index)
def resample_mu_and_Sig(self, A, W): """ Resample p given observations of the weights """ Abool = A.astype(np.bool) for c1 in xrange(self.C): for c2 in xrange(self.C): mask = self._get_mask(Abool, c1, c2) self._gaussians[c1][c2].resample(W[mask]) # Resample self connection if self.special_case_self_conns: mask = np.eye(self.N, dtype=np.bool) & Abool self._self_gaussian.resample(W[mask]) # Resample covariance A_offdiag = Abool.copy() np.fill_diagonal(A_offdiag, False) W_cent = (W - self.Mu)[A_offdiag] self._cov_model.resample(W_cent) # Update gaussians for c1 in xrange(self.C): for c2 in xrange(self.C): self._gaussians[c1][c2].sigma = self._cov_model.sigma
def T_classic(data, sigma=1000.): ''' Implementation of diffusion transition matrix calculation from Haghverdi et al http://biorxiv.org/content/early/2016/03/02/041384 ''' n, G = data.shape d2 = np.zeros((n, n)) for g in range(G): d2_g = np.subtract.outer(data[:, g], data[:, g]) ** 2 d2 += d2_g W = np.exp(-d2 / (2. * sigma ** 2)) D = W.sum(0) q = np.multiply.outer(D, D) np.fill_diagonal(W, 0.) H = W / q D_ = np.diag(H.sum(1)) D_inv = np.diag(1. / np.diag(D_)) T = D_inv.dot(H) phi0 = np.diag(D_) / np.diag(D_).sum() return T, phi0
def Gaussian_elim_pp(A): ''' Perform LU factorization by Gaussian elimination with Partial Pivoting(pp). A is unchanged in the process. Use the LU = PA expression so L is a lower triangular matrix. Return P, L and U. In the process L also need to be permutated. ''' n = A.shape[0] L = np.zeros((n,n)) # Left diagonal 0 because of the permutation later P = np.identity(n) U = np.copy(A) if n == 1: print 'Cannot be used for scalar' return for k in xrange(0,n - 1): # The original index should be +k max_idex = np.argmax(np.abs(U[:,k][k:])) + k if max_idex != k: row_index = np.arange(n) row_index[max_idex], row_index[k] = k, max_idex U = U[row_index,:] # Permutate U for partial pivoting L = L[row_index,:] # Permutate L too P = P[row_index,:] # Form the total permutation matrix if U[k,k] == 0: print 'ERROR: pivot is zero' continue for i in xrange(k + 1,n): L[i,k] = U[i,k] / U[k,k] U[i,k] = 0 for j in xrange(k + 1,n): for i in xrange(k + 1, n): U[i,j] = U[i,j] - L[i,k] * U[k,j] np.fill_diagonal(L,1) # Fill the diagonal of L in the end return P, L, U
def load_vocabulary(): """ Unpickle and return the content of the file `free_associations_vocabulary` generated by `process_data.py`. Output ------ W: association matrix id2voc: dictionary, keys are word ids and values words voc2id: dictionary, keys are words and values word ids """ # absolute path to the free association norms data path = '../../data/associationmatrices/' filename = 'association_norms_symm' try: with open(path+filename, 'rb') as f: id2voc = pickle.load(f) voc2id = pickle.load(f) Wsparse = pickle.load(f) except IOError: raise IOError('Association matrix "' + filename + '" not found' + ' in ' + path + '. To generate the matrix run ' + 'generate_association_matrix.py') # convert to dense matrix (stored as sparse for memory reasons) W = np.asarray(Wsparse.todense()) # normalize weights to [0-1] interval W /= 2 np.fill_diagonal(W, 1.) return W, id2voc, voc2id
def fit(self, X, y): """ X: data matrix, (n x d) y: scalar labels, (n) """ X, labels = check_X_y(X, y) n, d = X.shape num_dims = self.num_dims if num_dims is None: num_dims = d # Initialize A to a scaling matrix A = np.zeros((num_dims, d)) np.fill_diagonal(A, 1./(np.maximum(X.max(axis=0)-X.min(axis=0), EPS))) # Run NCA dX = X[:,None] - X[None] # shape (n, n, d) tmp = np.einsum('...i,...j->...ij', dX, dX) # shape (n, n, d, d) masks = labels[:,None] == labels[None] for it in xrange(self.max_iter): for i, label in enumerate(labels): mask = masks[i] Ax = A.dot(X.T).T # shape (n, num_dims) softmax = np.exp(-((Ax[i] - Ax)**2).sum(axis=1)) # shape (n) softmax[i] = 0 softmax /= softmax.sum() t = softmax[:, None, None] * tmp[i] # shape (n, d, d) d = softmax[mask].sum() * t.sum(axis=0) - t[mask].sum(axis=0) A += self.learning_rate * A.dot(d) self.X_ = X self.A_ = A self.n_iter_ = it return self
def calculate_couplings_levine(dt: float, w_jk: Matrix, w_kj: Matrix) -> Matrix: """ Compute the non-adiabatic coupling according to: `Evaluation of the Time-Derivative Coupling for Accurate Electronic State Transition Probabilities from Numerical Simulations`. Garrett A. Meek and Benjamin G. Levine. dx.doi.org/10.1021/jz5009449 | J. Phys. Chem. Lett. 2014, 5, 2351−2356 """ # Orthonormalize the Overlap matrices w_jk = np.linalg.qr(w_jk)[0] w_kj = np.linalg.qr(w_kj)[0] # Diagonal matrix w_jj = np.diag(np.diag(w_jk)) w_kk = np.diag(np.diag(w_kj)) # remove the values from the diagonal np.fill_diagonal(w_jk, 0) np.fill_diagonal(w_kj, 0) # Components A + B acos_w_jj = np.arccos(w_jj) asin_w_jk = np.arcsin(w_jk) a = acos_w_jj - asin_w_jk b = acos_w_jj + asin_w_jk A = - np.sin(np.sinc(a)) B = np.sin(np.sinc(b)) # Components C + D acos_w_kk = np.arccos(w_kk) asin_w_kj = np.arcsin(w_kj) c = acos_w_kk - asin_w_kj d = acos_w_kk + asin_w_kj C = np.sin(np.sinc(c)) D = np.sin(np.sinc(d)) # Components E w_lj = np.sqrt(1 - (w_jj ** 2) - (w_kj ** 2)) w_lk = -(w_jk * w_jj + w_kk * w_kj) / w_lj asin_w_lj = np.arcsin(w_lj) asin_w_lk = np.arcsin(w_lk) asin_w_lj2 = asin_w_lj ** 2 asin_w_lk2 = asin_w_lk ** 2 t1 = w_lj * w_lk * asin_w_lj x1 = np.sqrt((1 - w_lj ** 2) * (1 - w_lk ** 2)) - 1 t2 = x1 * asin_w_lk t = t1 + t2 E_nonzero = 2 * asin_w_lj * t / (asin_w_lj2 - asin_w_lk2) # Check whether w_lj is different of zero E1 = np.where(np.abs(w_lj) > 1e-8, E_nonzero, np.zeros(A.shape)) E = np.where(np.isclose(asin_w_lj2, asin_w_lk2), w_lj ** 2, E1) cte = 1 / (2 * dt) return cte * (np.arccos(w_jj) * (A + B) + np.arcsin(w_kj) * (C + D) + E)
def test_get_vox_dims(): # setup affine = np.eye(4) np.fill_diagonal(affine, [-3, 3, 3]) output_dir = os.path.join(OUTPUT_DIR, inspect.stack()[0][3]) if not os.path.exists(output_dir): os.makedirs(output_dir) # 3D vol vol = create_random_image(affine=affine) np.testing.assert_array_equal(get_vox_dims(vol), [3, 3, 3]) # 3D image file saved_img_filename = os.path.join(output_dir, "vol.nii.gz") nibabel.save(vol, saved_img_filename) np.testing.assert_array_equal(get_vox_dims(vol), [3, 3, 3]) # 4D niimg film = create_random_image(n_scans=10, affine=affine) np.testing.assert_array_equal(get_vox_dims(film), [3, 3, 3]) # 4D image file film = create_random_image(n_scans=10, affine=affine) saved_img_filename = os.path.join(output_dir, "4D.nii.gz") nibabel.save(film, saved_img_filename) np.testing.assert_array_equal(get_vox_dims(film), [3, 3, 3])
def add_batch(self, X, T, wc=None): """Add a batch of training data to an iterative solution, weighted if neeed. The batch is processed as a whole, the training data is splitted in `ELM.add_data()` method. With parameters HH_out, HT_out, the output will be put into these matrices instead of model. Args: X (matrix): input data matrix size (N * `inputs`) T (matrix): output data matrix size (N * `outputs`) wc (vector): vector of weights for data samples, one weight per sample, size (N * 1) HH_out, HT_out (matrix, optional): output matrices to add batch result into, always given together """ H = self._project(X) T = T.astype(self.precision) if wc is not None: # apply weights if given w = np.array(wc**0.5, dtype=self.precision)[:, None] # re-shape to column matrix H *= w T *= w if self.HH is None: # initialize space for self.HH, self.HT self.HH = np.zeros((self.L, self.L), dtype=self.precision) self.HT = np.zeros((self.L, self.outputs), dtype=self.precision) np.fill_diagonal(self.HH, self.norm) self.HH += np.dot(H.T, H) self.HT += np.dot(H.T, T)
def _get_abs_corr_mat(self, X_filled, tolerance=1e-6): """Get absolute correlation matrix between features. Parameters ---------- X_filled : ndarray, shape (n_samples, n_features) Input data with the most recent imputations. tolerance : float, optional (default=1e-6) ``abs_corr_mat`` can have nans, which will be replaced with ``tolerance``. Returns ------- abs_corr_mat : ndarray, shape (n_features, n_features) Absolute correlation matrix of ``X`` at the beginning of the current round. The diagonal has been zeroed out and each feature's absolute correlations with all others have been normalized to sum to 1. """ n_features = X_filled.shape[1] if (self.n_nearest_features is None or self.n_nearest_features >= n_features): return None abs_corr_mat = np.abs(np.corrcoef(X_filled.T)) # np.corrcoef is not defined for features with zero std abs_corr_mat[np.isnan(abs_corr_mat)] = tolerance # ensures exploration, i.e. at least some probability of sampling np.clip(abs_corr_mat, tolerance, None, out=abs_corr_mat) # features are not their own neighbors np.fill_diagonal(abs_corr_mat, 0) # needs to sum to 1 for np.random.choice sampling abs_corr_mat = normalize(abs_corr_mat, norm='l1', axis=0, copy=False) return abs_corr_mat
def knn_initialize( X, missing_mask, verbose=False, min_dist=1e-6, max_dist_multiplier=1e6): """ Fill X with NaN values if necessary, construct the n_samples x n_samples distance matrix and set the self-distance of each row to infinity. Returns contents of X laid out in row-major, the distance matrix, and an "effective infinity" which is larger than any entry of the distance matrix. """ X_row_major = X.copy("C") if missing_mask.sum() != np.isnan(X_row_major).sum(): # if the missing values have already been zero-filled need # to put NaN's back in the data matrix for the distances function X_row_major[missing_mask] = np.nan D = all_pairs_normalized_distances(X_row_major) D_finite_flat = D[np.isfinite(D)] if len(D_finite_flat) > 0: max_dist = max_dist_multiplier * max(1, D_finite_flat.max()) else: max_dist = max_dist_multiplier # set diagonal of distance matrix to a large value since we don't want # points considering themselves as neighbors np.fill_diagonal(D, max_dist) D[D < min_dist] = min_dist # prevents 0s D[D > max_dist] = max_dist # prevents infinities return X_row_major, D, max_dist
def test_mean(): """tests various ways to compute mean - collapsing different combination of axes""" data = np.arange(100).reshape(10,10) ts_1 = TimeSeriesX.create(data, None, dims=['x','y'], coords={'x':np.arange(10)*2, 'y':np.arange(10), 'samplerate': 1}) grand_mean = ts_1.mean() assert grand_mean == 49.5 x_mean = ts_1.mean(dim='x') assert (x_mean == np.arange(45,55,1, dtype=np.float)).all() # checking axes assert(ts_1.y == x_mean.y).all() y_mean = ts_1.mean(dim='y') assert (y_mean == np.arange(4.5,95,10, dtype=np.float)).all() # checking axes assert (y_mean.x == ts_1.x).all() # test mean NaN data_2 = np.arange(100, dtype=np.float).reshape(10,10) np.fill_diagonal(data_2,np.NaN) # data_2[9,9] = 99 ts_2 = TimeSeriesX.create(data_2, None, dims=['x','y'], coords={'x':np.arange(10)*2, 'y':np.arange(10), 'samplerate': 1}) grand_mean = ts_2.mean(skipna=True) assert grand_mean == 49.5
def question_1(): # Adjacency matrix. A = numpy.matrix([ [0, 0, 1, 0, 0, 1, 0, 0], [0, 0, 0, 0, 1, 0, 0, 1], [1, 0, 0, 1, 0, 1, 0, 0], [0, 0, 1, 0, 1, 0, 1, 0], [0, 1, 0, 1, 0, 0, 0, 1], [1, 0, 1, 0, 0, 0, 1, 0], [0, 0, 0, 1, 0, 1, 0, 1], [0, 1, 0, 0, 1, 0, 1, 0] ]) rn, cn = A.shape # Degree matrix. D = numpy.asmatrix(numpy.zeros((rn, cn), int)) numpy.fill_diagonal(D, sum(A)) # Laplacian matrix. L = D - A sum_a = A.sum() sum_d = D.sum() sum_l = L.sum() nonzero_a = numpy.count_nonzero(A) nonzero_d = numpy.count_nonzero(D) nonzero_l = numpy.count_nonzero(L) print('A: sum={} #nonzero={}'.format(sum_a, nonzero_a)) print('D: sum={} #nonzero={}'.format(sum_d, nonzero_d)) print('L: sum={} #nonzero={}'.format(sum_l, nonzero_l))
def build_kernels(self): """ Build the synaptic connectivity matrices """ n = self.n # Compute all the possible distances dist = [self.build_distances(n, 0.917, 0.0, 1.0), self.build_distances(n, 0.083, 0.0, 1.0), self.build_distances(n, 0.912, 0.83, 1.0)] # Create a temporary vector containing gaussians g = np.empty((len(self.K), n, n)) for j in range(len(self.K)): for i in range(n): # g[j, i] = self.K[j] * self.gaussian(dist[i], self.S[j]) g[j, i] = self.K[j] * self.g(dist[j][i], self.S[j]) g[j, self.m:self.k] = 0.0 # GPe to STN connections W12 = np.zeros((n, n)) W12[:self.m, self.k:] = g[0, self.k:, self.k:] # STN to GPe connections W21 = np.zeros((n, n)) W21[self.k:, :self.m] = g[1, :self.m, :self.m] # GPe to GPe connections W22 = np.zeros((n, n)) W22[self.k:, self.k:] = g[2, self.k:, self.k:] np.fill_diagonal(W22, 0.0) return W12, W21, W22, dist
def test_wcs_dropping(): wcs = WCS(naxis=4) wcs.wcs.pc = np.zeros([4, 4]) np.fill_diagonal(wcs.wcs.pc, np.arange(1, 5)) pc = wcs.wcs.pc # for later use below dropped = wcs.dropaxis(0) assert np.all(dropped.wcs.get_pc().diagonal() == np.array([2, 3, 4])) dropped = wcs.dropaxis(1) assert np.all(dropped.wcs.get_pc().diagonal() == np.array([1, 3, 4])) dropped = wcs.dropaxis(2) assert np.all(dropped.wcs.get_pc().diagonal() == np.array([1, 2, 4])) dropped = wcs.dropaxis(3) assert np.all(dropped.wcs.get_pc().diagonal() == np.array([1, 2, 3])) wcs = WCS(naxis=4) wcs.wcs.cd = pc dropped = wcs.dropaxis(0) assert np.all(dropped.wcs.get_pc().diagonal() == np.array([2, 3, 4])) dropped = wcs.dropaxis(1) assert np.all(dropped.wcs.get_pc().diagonal() == np.array([1, 3, 4])) dropped = wcs.dropaxis(2) assert np.all(dropped.wcs.get_pc().diagonal() == np.array([1, 2, 4])) dropped = wcs.dropaxis(3) assert np.all(dropped.wcs.get_pc().diagonal() == np.array([1, 2, 3]))
def __call__(self, A, Y, rng=None, E=None): # form Gram matrix so we can add regularization sigma = self.reg * A.max() G = np.dot(A.T, A) Y = np.dot(A.T, Y) np.fill_diagonal(G, G.diagonal() + sigma) return super(NnlsL2, self).__call__(G, Y, rng=rng, E=E)
def testBijector(self): x = np.float32(np.random.randn(3, 4, 4)) y = x.copy() for i in range(x.shape[0]): np.fill_diagonal(y[i, :, :], np.exp(np.diag(x[i, :, :]))) exp = tfb.Exp() b = tfb.TransformDiagonal(diag_bijector=exp) y_ = self.evaluate(b.forward(x)) self.assertAllClose(y, y_) x_ = self.evaluate(b.inverse(y)) self.assertAllClose(x, x_) fldj = self.evaluate(b.forward_log_det_jacobian(x, event_ndims=2)) ildj = self.evaluate(b.inverse_log_det_jacobian(y, event_ndims=2)) self.assertAllEqual( fldj, self.evaluate(exp.forward_log_det_jacobian( np.array([np.diag(x_mat) for x_mat in x]), event_ndims=1))) self.assertAllEqual( ildj, self.evaluate(exp.inverse_log_det_jacobian( np.array([np.diag(y_mat) for y_mat in y]), event_ndims=1)))
def _srm(self, data): """Expectation-Maximization algorithm for fitting the probabilistic SRM. Parameters ---------- data : list of 2D arrays, element i has shape=[voxels_i, samples] Each element in the list contains the fMRI data of one subject. Returns ------- w : list of array, element i has shape=[voxels_i, features] The orthogonal transforms (mappings) :math:`W_i` for each subject. s : array, shape=[features, samples] The shared response. """ subjects = len(data) self.random_state_ = np.random.RandomState(self.rand_seed) random_states = [ np.random.RandomState( self.random_state_.randint(2 ** 32 - 1, dtype=np.int64) ) for i in range(len(data)) ] # Initialization step: initialize the outputs with initial values, # voxels with the number of voxels in each subject. w, _ = _init_w_transforms(data, self.features, random_states) shared_response = self._compute_shared_response(data, w) if logger.isEnabledFor(logging.INFO): # Calculate the current objective function value objective = self._objective_function(data, w, shared_response) logger.info("Objective function %f" % objective) # Main loop of the algorithm for iteration in range(self.n_iter): logger.info("Iteration %d" % (iteration + 1)) # Update each subject's mapping transform W_i: for subject in range(subjects): a_subject = data[subject].dot(shared_response.T) perturbation = np.zeros(a_subject.shape) np.fill_diagonal(perturbation, 0.001) u_subject, _, v_subject = np.linalg.svd( a_subject + perturbation, full_matrices=False ) w[subject] = u_subject.dot(v_subject) # Update the shared response: shared_response = self._compute_shared_response(data, w) if logger.isEnabledFor(logging.INFO): # Calculate the current objective function value objective = self._objective_function(data, w, shared_response) logger.info("Objective function %f" % objective) return w, shared_response
def estimate_shift2D(self, reference='current', correlation_threshold=None, chunk_size=30, roi=None, normalize_corr=False, sobel=True, medfilter=True, hanning=True, plot=False, dtype='float', show_progressbar=None, sub_pixel_factor=1): """Estimate the shifts in a image using phase correlation This method can only estimate the shift by comparing bidimensional features that should not change position between frames. To decrease the memory usage, the time of computation and the accuracy of the results it is convenient to select a region of interest by setting the roi keyword. Parameters ---------- reference : {'current', 'cascade' ,'stat'} If 'current' (default) the image at the current coordinates is taken as reference. If 'cascade' each image is aligned with the previous one. If 'stat' the translation of every image with all the rest is estimated and by performing statistical analysis on the result the translation is estimated. correlation_threshold : {None, 'auto', float} This parameter is only relevant when reference='stat'. If float, the shift estimations with a maximum correlation value lower than the given value are not used to compute the estimated shifts. If 'auto' the threshold is calculated automatically as the minimum maximum correlation value of the automatically selected reference image. chunk_size : {None, int} If int and reference='stat' the number of images used as reference are limited to the given value. roi : tuple of ints or floats (left, right, top, bottom) Define the region of interest. If int(float) the position is given axis index(value). Note that ROIs can be used in place of a tuple. sobel : bool apply a sobel filter for edge enhancement medfilter : bool apply a median filter for noise reduction hanning : bool Apply a 2d hanning filter plot : bool or 'reuse' If True plots the images after applying the filters and the phase correlation. If 'reuse', it will also plot the images, but it will only use one figure, and continuously update the images in that figure as it progresses through the stack. dtype : str or dtype Typecode or data-type in which the calculations must be performed. %s sub_pixel_factor : float Estimate shifts with a sub-pixel accuracy of 1/sub_pixel_factor parts of a pixel. Default is 1, i.e. no sub-pixel accuracy. Returns ------- shifts : list of array List of estimated shifts Notes ----- The statistical analysis approach to the translation estimation when using reference='stat' roughly follows [*]_ . If you use it please cite their article. References ---------- .. [*] Schaffer, Bernhard, Werner Grogger, and Gerald Kothleitner. “Automated Spatial Drift Correction for EFTEM Image Series.” Ultramicroscopy 102, no. 1 (December 2004): 27–36. """ if show_progressbar is None: show_progressbar = preferences.General.show_progressbar self._check_signal_dimension_equals_two() if roi is not None: # Get the indices of the roi yaxis = self.axes_manager.signal_axes[1] xaxis = self.axes_manager.signal_axes[0] roi = tuple([xaxis._get_index(i) for i in roi[2:]] + [yaxis._get_index(i) for i in roi[:2]]) ref = None if reference == 'cascade' else \ self.__call__().copy() shifts = [] nrows = None images_number = self.axes_manager._max_index + 1 if plot == 'reuse': # Reuse figure for plots plot = plt.figure() if reference == 'stat': nrows = images_number if chunk_size is None else \ min(images_number, chunk_size) pcarray = ma.zeros(( nrows, self.axes_manager._max_index + 1, ), dtype=np.dtype([('max_value', np.float), ('shift', np.int32, (2, ))])) nshift, max_value = estimate_image_shift( self(), self(), roi=roi, sobel=sobel, medfilter=medfilter, hanning=hanning, normalize_corr=normalize_corr, plot=plot, dtype=dtype, sub_pixel_factor=sub_pixel_factor) np.fill_diagonal(pcarray['max_value'], max_value) pbar_max = nrows * images_number else: pbar_max = images_number # Main iteration loop. Fills the rows of pcarray when reference # is stat with progressbar(total=pbar_max, disable=not show_progressbar, leave=True) as pbar: for i1, im in enumerate(self._iterate_signal()): if reference in ['current', 'cascade']: if ref is None: ref = im.copy() shift = np.array([0, 0]) nshift, max_val = estimate_image_shift( ref, im, roi=roi, sobel=sobel, medfilter=medfilter, hanning=hanning, plot=plot, normalize_corr=normalize_corr, dtype=dtype, sub_pixel_factor=sub_pixel_factor) if reference == 'cascade': shift += nshift ref = im.copy() else: shift = nshift shifts.append(shift.copy()) pbar.update(1) elif reference == 'stat': if i1 == nrows: break # Iterate to fill the columns of pcarray for i2, im2 in enumerate(self._iterate_signal()): if i2 > i1: nshift, max_value = estimate_image_shift( im, im2, roi=roi, sobel=sobel, medfilter=medfilter, hanning=hanning, normalize_corr=normalize_corr, plot=plot, dtype=dtype, sub_pixel_factor=sub_pixel_factor) pcarray[i1, i2] = max_value, nshift del im2 pbar.update(1) del im if reference == 'stat': # Select the reference image as the one that has the # higher max_value in the row sqpcarr = pcarray[:, :nrows] sqpcarr['max_value'][:] = symmetrize(sqpcarr['max_value']) sqpcarr['shift'][:] = antisymmetrize(sqpcarr['shift']) ref_index = np.argmax(pcarray['max_value'].min(1)) self.ref_index = ref_index shifts = (pcarray['shift'] + pcarray['shift'][ref_index, :nrows][:, np.newaxis]) if correlation_threshold is not None: if correlation_threshold == 'auto': correlation_threshold = \ (pcarray['max_value'].min(0)).max() _logger.info("Correlation threshold = %1.2f", correlation_threshold) shifts[ pcarray['max_value'] < correlation_threshold] = ma.masked shifts.mask[ref_index, :] = False shifts = shifts.mean(0) else: shifts = np.array(shifts) del ref return shifts
R = np.zeros((31, 31)) for i in range(31): for j in range(31): #対角成分 if j == i: #kが奇数の場合 if k[i] % 2 != 0: R[i][j] = 1.0 #kが偶数の場合 else: R[i][j] = 4.0 #非対角成分 else: R[i][j] = 0 #推定値x_hatの初期値(3*1) x_hat_init = np.zeros(3) #推定誤差共分散行列Pの初期値(3*3) Pk_init = np.zeros((3, 3)) #対角成分を初期値を10^6にする(変更すると推定精度が変わる) np.fill_diagonal(Pk_init, 1000000) #プラント雑音共分散行列qの初期値(3*3) Qk_init = np.zeros((3, 3)) #対角成分を初期値を10^6にする(変更すると推定精度が変わる) np.fill_diagonal(Qk_init, 0) print('Start Kalman Filtering') KalmanFiltering(x_hat_init, Pk_init, Z, A, R)
print( cross_val_score(sgd_clf, X_train_scaled, y_train, cv=3, scoring="accuracy")) # 에러 분석 y_train_pred = cross_val_predict(sgd_clf, X_train, y_train, cv=3) conf_mx = confusion_matrix(y_train, y_train_pred) print(conf_mx) plt.matshow(conf_mx, cmap=plt.cm.gray) plt.show() row_sums = conf_mx.sum(axis=1, keepdims=True) norm_conf_mx = conf_mx / row_sums np.fill_diagonal(norm_conf_mx, 0) plt.matshow(norm_conf_mx, cmap=plt.cm.gray) plt.show() y_train_large = (y_train >= 7) y_train_odd = (y_train % 2 == 1) y_multilabel = np.c_[y_train_large, y_train_odd] knn_clf = KNeighborsClassifier() knn_clf.fit(X_train, y_multilabel) print(knn_clf.predict([some_digit])) y_train_knn_pred = cross_val_predict(knn_clf, X_train, y_multilabel, cv=3) print(f1_score(y_multilabel, y_train_knn_pred, average="macro"))
(idx + 1)] = NodeAttribute(ts_) npy_adjacency_matrix[:all_node_num, (all_node_num + n_expanded) * idx:(all_node_num + n_expanded) * idx + all_node_num] = get_adjacency_matrix( ts_, L, 'all') # t+1 node_attribute[:pred_attribute[c_idx][ts].shape[0], attribute_dim * (L - 1):] = pred_attribute[c_idx][ts] npy_adjacency_matrix[:, (all_node_num + n_expanded) * (L - 1):] = pred_adjacency_matrix[ts][c_idx] lil_adjacency_matrix = lil_matrix(npy_adjacency_matrix) lil_node_attribute = lil_matrix(node_attribute) mmwrite(OutputDir[c_idx] + "/input/node_attribute/" + str(ts), lil_node_attribute) mmwrite(OutputDir[c_idx] + "/input/adjacency/" + str(ts), lil_adjacency_matrix) label = np.zeros( (all_node_num + n_expanded, all_node_num + n_expanded)) label[:all_node_num, :all_node_num] = get_adjacency_matrix( ts_test, L, "disappeared") mmwrite(OutputDir[c_idx] + "/label/" + str(ts), lil_matrix(label)) exist_matrix = get_exist_matrix(ts_train[-1]) np.fill_diagonal(exist_matrix, 0) mask = np.zeros((all_node_num + n_expanded, all_node_num + n_expanded)) mask[:all_node_num, :all_node_num] = get_adjacency_matrix( ts_train[-1], L, 'all') mmwrite(OutputDir[c_idx] + "/mask/" + str(ts), lil_matrix(mask))
def InitializeFlatGraph(n, min_weight, max_weight, dropout=0, seed=-1, graph_type="complete", intercalation=None, modulo=None): if graph_type == "complete": if seed == -1: W = np.random.uniform(low=min_weight, high=max_weight, size=n) else: local_state = np.random.RandomState(seed) W = local_state.uniform(low=min_weight, high=max_weight, size=n) if dropout > 0: connected = False while connected == False: if seed == -1: D = np.random.choice([0, 1], size=n, replace=True, p=[dropout, 1 - dropout]) else: D = local_state.choice([0, 1], size=n, replace=True, p=[dropout, 1 - dropout]) # Check if the dropped out matrix is connected # Make it a symmetric matrix D_sym = SymmetricMatrix(D) # Check if it resembles a connected graph D_sym = nx.from_numpy_matrix(D_sym) if nx.is_connected(D_sym): connected = True # Once we have a conencted graph, add weights W = np.multiply(W, D) elif graph_type == "cycle": if seed == -1: weights = np.random.uniform(low=min_weight, high=max_weight, size=n) else: local_state = np.random.RandomState(seed) weights = local_state.uniform(low=min_weight, high=max_weight, size=n) W = np.zeros((n, n)) i = 0 j = 1 for k in range(n - 1): W[i, j] = weights[k] i += 1 j += 1 # Add last weight W[0, n - 1] = weights[n - 1] i_lower = np.tril_indices(n, -1) W[i_lower] = W.T[i_lower] # Make sure diagonal is empty np.fill_diagonal(W, 0) W = np.copy(W[np.triu_indices_from(W, k=1)]) elif graph_type == "regular_log(n)": degseq = np.repeat(np.floor(np.log(n)), n) adjMatr = AdjacencyMatrixDegree(degseq) if seed == -1: W = np.random.uniform(low=min_weight, high=max_weight, size=n) else: local_state = np.random.RandomState(seed) W = local_state.uniform(low=min_weight, high=max_weight, size=n) W = np.multiply(W, adjMatr) W = np.copy(W[np.triu_indices_from(W, k=1)]) elif graph_type == "intercalate": W = InitializeGraph(n, min_weight, max_weight, dropout, seed, graph_type, intercalation) W = np.copy(W[np.triu_indices_from(W, k=1)]) elif graph_type == "modulo": W = InitializeGraph(n, min_weight, max_weight, dropout, seed, graph_type, None, modulo) W = np.copy(W[np.triu_indices_from(W, k=1)]) return W
def InitializeGraph(n, min_weight, max_weight, dropout, seed=-1, graph_type="complete", intercalation=None, modulo=None): # Randomness if graph_type == "complete": if seed == -1: W = np.random.uniform(low=min_weight, high=max_weight, size=(n, n)) else: local_state = np.random.RandomState(seed) W = local_state.uniform(low=min_weight, high=max_weight, size=(n, n)) # Dropout with connected if dropout > 0: connected = False while not connected: if seed == -1: D = np.random.choice([0, 1], size=(n, n), replace=True, p=[dropout, 1 - dropout]) else: D = local_state.choice([0, 1], size=(n, n), replace=True, p=[dropout, 1 - dropout]) W_dropped = np.multiply(W, D) # Check for connectivity i_lower = np.tril_indices(n, -1) W_dropped[i_lower] = W_dropped.T[i_lower] # Make sure diagonal is empty np.fill_diagonal(W_dropped, 0) W_sym = nx.from_numpy_matrix(W_dropped) if nx.is_connected(W_sym): connected = True W = np.copy(W_dropped) elif graph_type == "cycle": if seed == -1: weights = np.random.uniform(low=min_weight, high=max_weight, size=n) else: local_state = np.random.RandomState(seed) weights = local_state.uniform(low=min_weight, high=max_weight, size=(n, n)) W = np.zeros((n, n)) i = 0 j = 1 for k in range(n - 1): W[i, j] = weights[k] i += 1 j += 1 # Add last weight W[0, n - 1] = weights[n - 1] elif graph_type == "regular_log(n)": degseq = np.repeat(np.floor(np.log(n)), n) adjMatr = AdjacencyMatrixDegree(degseq) if seed == -1: W = np.random.uniform(low=min_weight, high=max_weight, size=n) else: local_state = np.random.RandomState(seed) W = local_state.uniform(low=min_weight, high=max_weight, size=n) W = np.multiply(W, adjMatr) elif graph_type == "intercalate": if seed == -1: weights = np.random.uniform(low=min_weight, high=max_weight, size=n) else: local_state = np.random.RandomState(seed) weights = local_state.uniform(low=min_weight, high=max_weight, size=(n, n)) adjMatr = np.zeros((n, n)) # Fill diagonals above the main diagonal for k in range(intercalation): for i in range(n - k - 1): adjMatr[i, i + k + 1] = 1 # Fill upper right corner for j in range(intercalation): for i in range(j + 1): adjMatr[j - i, n - i - 1] = 1 W = np.multiply(weights, adjMatr) elif graph_type == "modulo": if seed == -1: weights = np.random.uniform(low=min_weight, high=max_weight, size=n) else: local_state = np.random.RandomState(seed) weights = local_state.uniform(low=min_weight, high=max_weight, size=(n, n)) adjMatr = np.zeros((n, n)) # Create circle for i in range(n - 1): adjMatr[i, i + 1] = 1 # Fill upper right corner adjMatr[0, n - 1] = 1 # Add other connections # Fill diagonals above the main diagonal for i in range(n - modulo): adjMatr[i, i + modulo] = 1 # Fill upper right corner for i in range(modulo): adjMatr[modulo - 1 - i, n - i - 1] = 1 W = np.multiply(weights, adjMatr) # Stochastic Block Model elif graph_type == "assortative": communities = np.floor(n / 3) sizes = 3 * [int(communities)] # Check if we have all nodes: if 3 * communities < n: sizes[0] += 1 if 3 * communities < n - 1: sizes[1] += 1 probs = [[0.4, 0.02, 0.02], [0.02, 0.4, 0.02], [0.02, 0.02, 0.4]] g = nx.stochastic_block_model(sizes, probs, seed=0) W = nx.to_numpy_matrix(g) elif graph_type == "disassortative": communities = np.floor(n / 3) sizes = 3 * [int(communities)] # Check if we have all nodes: if 3 * communities < n: sizes[0] += 1 if 3 * communities < n - 1: sizes[1] += 1 probs = [[0.02, 0.08, 0.08], [0.08, 0.02, 0.08], [0.08, 0.08, 0.2]] g = nx.stochastic_block_model(sizes, probs, seed=0) W = nx.to_numpy_matrix(g) elif graph_type == "ordered": communities = np.floor(n / 3) sizes = 3 * [int(communities)] # Check if we have all nodes: if 3 * communities < n: sizes[0] += 1 if 3 * communities < n - 1: sizes[1] += 1 probs = [[0, 0.15, 0.15], [0.15, 0, 0.15], [0.15, 0.15, 0]] g = nx.stochastic_block_model(sizes, probs, seed=0) W = nx.to_numpy_matrix(g) elif graph_type == "core-periphery": communities = np.floor(n / 3) sizes = 3 * [int(communities)] # Check if we have all nodes: if 3 * communities < n: sizes[0] += 1 if 3 * communities < n - 1: sizes[1] += 1 probs = [[0.4, 0.15, 0.03], [0.15, 0.15, 0.03], [0.03, 0.03, 0.03]] g = nx.stochastic_block_model(sizes, probs, seed=0) W = nx.to_numpy_matrix(g) # Copy upper diagonal to lower diagonal (only the upper perturbation counts) i_lower = np.tril_indices(n, -1) W[i_lower] = W.T[i_lower] # Make sure diagonal is empty np.fill_diagonal(W, 0) return W
# 把测试集比例调整为0.8 from sklearn.metrics import precision_score print(precision_score(y_test, y_predict, average='micro')) #0.93115438108484 from sklearn.metrics import confusion_matrix print(confusion_matrix(y_test, y_predict)) # [[147 0 1 0 0 1 0 0 0 0] # [ 0 123 1 2 0 0 0 3 4 10] # [ 0 0 134 1 0 0 0 0 1 0] # [ 0 0 0 138 0 5 0 1 5 0] # [ 2 5 0 0 139 0 0 3 0 1] # [ 1 3 1 0 0 146 0 0 1 0] # [ 0 2 0 0 0 1 131 0 2 0] # [ 0 0 0 1 0 0 0 132 1 2] # [ 1 9 2 3 2 4 0 0 115 4] # [ 0 1 0 5 0 3 0 2 2 134]] cfm = confusion_matrix(y_test, y_predict) plt.matshow(cfm, cmap=plt.cm.gray) plt.show() #可看出有很多‘1’预测成了‘9’,很多‘8’预测成了‘1’ row_sums = np.sum(cfm, axis=1) err_matrix = cfm / row_sums np.fill_diagonal(err_matrix, 0) plt.matshow(err_matrix, cmap=plt.cm.gray) plt.show()
def run(self): self.reg_cov = 1e-6 * np.identity(len(self.X[0])) x, y = np.meshgrid(np.sort(self.X[:, 0]), np.sort(self.X[:, 1])) self.XY = np.array([x.flatten(), y.flatten()]).T """ 1. Set the initial mu, covariance and pi values""" self.mu = np.random.randint( min(self.X[:, 0]), max(self.X[:, 0]), size=(self.number_of_sources, len(self.X[0])) ) # This is a nxm matrix since we assume n sources (n Gaussians) where each has m dimensions self.cov = np.zeros( (self.number_of_sources, len(X[0]), len(X[0])) ) # We need a nxmxm covariance matrix for each source since we have m features --> We create symmetric covariance matrices with ones on the digonal for dim in range(len(self.cov)): np.fill_diagonal(self.cov[dim], 5) self.pi = np.ones( self.number_of_sources) / self.number_of_sources # Are "Fractions" log_likelihoods = [ ] # In this list we store the log likehoods per iteration and plot them in the end to check if # if we have converged """Plot the initial state""" fig = plt.figure(figsize=(10, 10)) ax0 = fig.add_subplot(111) ax0.scatter(self.X[:, 0], self.X[:, 1]) ax0.set_title('Initial state') for m, c in zip(self.mu, self.cov): c += self.reg_cov multi_normal = multivariate_normal(mean=m, cov=c) ax0.contour(np.sort(self.X[:, 0]), np.sort(self.X[:, 1]), multi_normal.pdf(self.XY).reshape( len(self.X), len(self.X)), colors='black', alpha=0.3) ax0.scatter(m[0], m[1], c='grey', zorder=10, s=100) for i in range(self.iterations): """E Step""" r_ic = np.zeros((len(self.X), len(self.cov))) for m, co, p, r in zip(self.mu, self.cov, self.pi, range(len(r_ic[0]))): co += self.reg_cov mn = multivariate_normal(mean=m, cov=co) r_ic[:, r] = p * mn.pdf(self.X) / np.sum([ pi_c * multivariate_normal(mean=mu_c, cov=cov_c).pdf(X) for pi_c, mu_c, cov_c in zip(self.pi, self.mu, self.cov + self.reg_cov) ], axis=0) """ The above calculation of r_ic is not that obvious why I want to quickly derive what we have done above. First of all the nominator: We calculate for each source c which is defined by m,co and p for every instance x_i, the multivariate_normal.pdf() value. For each loop this gives us a 100x1 matrix (This value divided by the denominator is then assigned to r_ic[:,r] which is in the end a 100x3 matrix). Second the denominator: What we do here is, we calculate the multivariate_normal.pdf() for every instance x_i for every source c which is defined by pi_c, mu_c, and cov_c and write this into a list. This gives us a 3x100 matrix where we have 100 entrances per source c. Now the formula wants us to add up the pdf() values given by the 3 sources for each x_i. Hence we sum up this list over axis=0. This gives us then a list with 100 entries. What we have now is FOR EACH LOOP a list with 100 entries in the nominator and a list with 100 entries in the denominator where each element is the pdf per class c for each instance x_i (nominator) respectively the summed pdf's of classes c for each instance x_i. Consequently we can now divide the nominator by the denominator and have as result a list with 100 elements which we can then assign to r_ic[:,r] --> One row r per source c. In the end after we have done this for all three sources (three loops) and run from r==0 to r==2 we get a matrix with dimensionallity 100x3 which is exactly what we want. If we check the entries of r_ic we see that there mostly one element which is much larger than the other two. This is because every instance x_i is much closer to one of the three gaussians (that is, much more likely to come from this gaussian) than it is to the other two. That is practically speaing, r_ic gives us the fraction of the probability that x_i belongs to class c over the probability that x_i belonges to any of the classes c (Probability that x_i occurs given the 3 Gaussians). """ """M Step""" # Calculate the new mean vector and new covariance matrices, based on the probable membership of the single x_i to classes c --> r_ic self.mu = [] self.cov = [] self.pi = [] log_likelihood = [] for c in range(len(r_ic[0])): m_c = np.sum(r_ic[:, c], axis=0) mu_c = (1 / m_c) * np.sum( self.X * r_ic[:, c].reshape(len(self.X), 1), axis=0) self.mu.append(mu_c) # Calculate the covariance matrix per source based on the new mean self.cov.append(((1 / m_c) * np.dot( (np.array(r_ic[:, c]).reshape(len(self.X), 1) * (self.X - mu_c)).T, (self.X - mu_c))) + self.reg_cov) # Calculate pi_new which is the "fraction of points" respectively the fraction of the probability assigned to each source self.pi.append( m_c / np.sum(r_ic) ) # Here np.sum(r_ic) gives as result the number of instances. This is logical since we know # that the columns of each row of r_ic adds up to 1. Since we add up all elements, we sum up all # columns per row which gives 1 and then all rows which gives then the number of instances (rows) # in X --> Since pi_new contains the fractions of datapoints, assigned to the sources c, # The elements in pi_new must add up to 1 """Log likelihood""" log_likelihoods.append( np.log( np.sum([ k * multivariate_normal(self.mu[i], self.cov[j]).pdf(X) for k, i, j in zip(self.pi, range(len(self.mu)), range(len(self.cov))) ]))) print(log_likelihoods) """ This process of E step followed by a M step is now iterated a number of n times. In the second step for instance, we use the calculated pi_new, mu_new and cov_new to calculate the new r_ic which are then used in the second M step to calculat the mu_new2 and cov_new2 and so on.... """ fig2 = plt.figure(figsize=(10, 10)) ax1 = fig2.add_subplot(111) ax1.set_title('Log-Likelihood') ax1.plot(range(0, self.iterations, 1), log_likelihoods)
def _srm(self, data): """Expectation-Maximization algorithm for fitting the probabilistic SRM. Parameters ---------- data : list of 2D arrays, element i has shape=[voxels_i, samples] Each element in the list contains the fMRI data of one subject. Returns ------- sigma_s : array, shape=[features, features] The covariance :math:`\\Sigma_s` of the shared response Normal distribution. w : list of array, element i has shape=[voxels_i, features] The orthogonal transforms (mappings) :math:`W_i` for each subject. mu : list of array, element i has shape=[voxels_i] The voxel means :math:`\\mu_i` over the samples for each subject. rho2 : array, shape=[subjects] The estimated noise variance :math:`\\rho_i^2` for each subject s : array, shape=[features, samples] The shared response. """ samples = min([d.shape[1] for d in data if d is not None], default=sys.maxsize) subjects = len(data) self.random_state_ = np.random.RandomState(self.rand_seed) random_states = [ np.random.RandomState( self.random_state_.randint(2 ** 32 - 1, dtype=np.int64) ) for i in range(len(data)) ] # Initialization step: initialize the outputs with initial values, # voxels with the number of voxels in each subject, and trace_xtx with # the ||X_i||_F^2 of each subject. w, voxels = _init_w_transforms(data, self.features, random_states) x, mu, rho2, trace_xtx = self._init_structures(data, subjects) shared_response = np.zeros((self.features, samples)) sigma_s = np.identity(self.features) # Main loop of the algorithm (run for iteration in range(self.n_iter): logger.info("Iteration %d" % (iteration + 1)) # E-step: # Sum the inverted the rho2 elements for computing W^T * Psi^-1 * W rho0 = (1 / rho2).sum() # Invert Sigma_s using Cholesky factorization (chol_sigma_s, lower_sigma_s) = scipy.linalg.cho_factor( sigma_s, check_finite=False ) inv_sigma_s = scipy.linalg.cho_solve( (chol_sigma_s, lower_sigma_s), np.identity(self.features), check_finite=False, ) # Invert (Sigma_s + rho_0 * I) using Cholesky factorization sigma_s_rhos = inv_sigma_s + np.identity(self.features) * rho0 chol_sigma_s_rhos, lower_sigma_s_rhos = scipy.linalg.cho_factor( sigma_s_rhos, check_finite=False ) inv_sigma_s_rhos = scipy.linalg.cho_solve( (chol_sigma_s_rhos, lower_sigma_s_rhos), np.identity(self.features), check_finite=False, ) # Compute the sum of W_i^T * rho_i^-2 * X_i, and the sum of traces # of X_i^T * rho_i^-2 * X_i wt_invpsi_x = np.zeros((self.features, samples)) trace_xt_invsigma2_x = 0.0 for subject in range(subjects): if data[subject] is not None: wt_invpsi_x += (w[subject].T.dot(x[subject])) / rho2[subject] trace_xt_invsigma2_x += trace_xtx[subject] / rho2[subject] log_det_psi = np.sum(np.log(rho2) * voxels) # Update the shared response shared_response = sigma_s.dot( np.identity(self.features) - rho0 * inv_sigma_s_rhos ).dot(wt_invpsi_x) # M-step # Update Sigma_s and compute its trace sigma_s = ( inv_sigma_s_rhos + shared_response.dot(shared_response.T) / samples ) trace_sigma_s = samples * np.trace(sigma_s) # Update each subject's mapping transform W_i and error variance # rho_i^2 for subject in range(subjects): if x[subject] is not None: a_subject = x[subject].dot(shared_response.T) perturbation = np.zeros(a_subject.shape) np.fill_diagonal(perturbation, 0.001) u_subject, s_subject, v_subject = np.linalg.svd( a_subject + perturbation, full_matrices=False ) w[subject] = u_subject.dot(v_subject) rho2[subject] = trace_xtx[subject] rho2[subject] += -2 * np.sum(w[subject] * a_subject).sum() rho2[subject] += trace_sigma_s rho2[subject] /= samples * voxels[subject] else: rho2[subject] = 0 if logger.isEnabledFor(logging.INFO): # Calculate and log the current log-likelihood for checking # convergence loglike = self._likelihood( chol_sigma_s_rhos, log_det_psi, chol_sigma_s, trace_xt_invsigma2_x, inv_sigma_s_rhos, wt_invpsi_x, samples, ) logger.info("Objective function %f" % loglike) return sigma_s, w, mu, rho2, shared_response
# Nodal Admitance Matrix Ybus = np.zeros((len(buses.keys()), len(buses.keys())), dtype=complex) # Shunt Elements Vector Bshunt = np.zeros(len(buses), dtype=complex) for key in lines.keys(): Ybus[lines[key].origin - 1][lines[key].destiny - 1] = -1 / (lines[key].R + 1j * lines[key].X) Bshunt[lines[key].origin - 1] += 1j * lines[key].B / 2 Bshunt[lines[key].destiny - 1] += 1j * lines[key].B / 2 Ybus += Ybus.T np.fill_diagonal(Ybus, Bshunt - np.sum(Ybus, axis=1)) # Create Jacobian Matrix H = np.zeros((len(measures.keys()), len(buses.keys()))) # Create measurement vector z = np.zeros((len(measures.keys()), 1)) # Create weight matrix W = np.zeros((len(measures.keys()), len(measures.keys()))) # Fill Jacobian matrix, measurement vector and weight matrix aux = 0 for key in measures.keys(): # Fill Jacobian matrix
import scipy.sparse.linalg as splg N = 32 h = 1 / N """Create A""" Ih = np.identity(N + 1) Ih[0, 0] = 0 Ih[N, N] = 0 vdiag = 4 * np.ones(N - 1) voffs = -1 * np.ones(N - 2) zeros = np.zeros((1, N - 1)) Th = np.diag(vdiag) np.fill_diagonal(Th[:, 1:], voffs) np.fill_diagonal(Th[1:], voffs) Th = np.block([[h**2, zeros, 0], [zeros.T, Th, zeros.T], [0, zeros, h**2]]) #print(Th) zN = np.zeros((N + 1, (N + 1)**2 - (N + 1))) zShort = np.zeros((N + 1, N + 1)) #print(np.shape(zN)) I = h**2 * np.identity(N + 1) block1 = np.block([[I, zN]]) block2 = np.block([[zShort, Th, -Ih, zN[:, 2 * (N + 1):]]]) A = np.block([[block1], [block2]]) for i in range(3, N):
def calc_grid_probs(grid, grid_n_x, grid_n_y, Suscept, kernel_function, \ d_perimeter_x, d_perimeter_y): """ Calculate the probability of infection from one grid square to the next. (quicker than version above) Notes ----- Grid and Suscept MUST be ordered according to grid number! Could also use pairwise chebyshev distances (scipy.spatial.distance.cdist) instead of using ogrid etc (below) if the x/y lengths of the grid cells are the same. Parameters ---------- grid: np.array grid_n_x: np.array grid_n_y: np.array Suscept: np.array kernel_function: function d_perimeter_x : float d_perimeter_y : float Returns ------- MaxRate Num first_ last_ max_sus Dist2all KDist2all Example ------- # Calculate grid-level probabilities of infection for the UK data import epi_speed_test.py.fmd as fmd kernel = fmd.UKKernel out = calc_grid_probs(grid, 100, 100, np.random.rand(len(grid)), kernel, \ np.ptp(demo.easting.values), np.ptp(demo.northing.values)) Updates ------- # 6th Oct 2014 Changed use of grid.max()+1 to grid_n_x*grid_n_y, WP # Feb 2017 Changed to use np.ogrid etc, WP """ NG = grid_n_x*grid_n_y # Find the farm indices per grid square fpg = [np.where(grid == i)[0] for i in range(NG)] # Perhaps be made quicker by only looking at the grids with farms in them. # Maximum susceptibility of a feedlot per grid square (0 for empty grid) max_sus = np.asarray([np.max(Suscept[x]) if len(x) > 0 else 0 for x in fpg]) # Find the number of farms per grid square Num = np.array([len(i) for i in fpg]) # First farm in grid, -1 if the list is empty first_ = np.asarray([x[0] if len(x) > 0 else -1 for x in fpg]) # Last farm in grid, -2 if the list is empty last_ = np.asarray([x[-1] if len(x) > 0 else -2 for x in fpg]) NCOL = float(grid_n_x); NROW = float(grid_n_y) HEIGHT = d_perimeter_y/NROW; WIDTH = d_perimeter_x/NCOL # Calculate the minimum sq-distance between each grid square # X and Y position of each grid square Y, X = np.ogrid[0:NROW*NCOL, 0:NROW*NCOL] HDIST = np.abs(Y%NCOL - X%NCOL) - 1 VDIST = np.abs(Y/NCOL - X/NCOL) - 1 HDIST[HDIST < 0] = 0; VDIST[VDIST < 0] = 0 Dist2all = np.add((HDIST*WIDTH)**2, (VDIST*HEIGHT)**2) KDist2all = kernel_function(Dist2all) # very slow # Could perhaps use Numpy broadcasting for this ... MaxRate = np.multiply(KDist2all, np.tile(max_sus, (grid_n_x*grid_n_y, 1))) # All grids with no farms should have infinite rate of input infection # All grids with no farms should have infinite rate of susceptibility MaxRate[Num == 0] = np.inf MaxRate[:, Num == 0] = np.inf # Set rate from one grid to itself to infinity. np.fill_diagonal(MaxRate, np.inf) return(MaxRate, Num, first_, last_, max_sus, Dist2all, KDist2all)
cm = confusion_matrix(y_train, y_train_pred) cm plt.matshow(cm, cmap=plt.cm.gray) plt.show() # %% """ Normalize the confusion matrix """ row_sums = cm.sum(axis=1, keepdims=True) norm_cm = cm / row_sums # Remove the diagnols to keep only the errors np.fill_diagonal(norm_cm, 0) plt.matshow(norm_cm, cmap=plt.cm.gray) plt.show() # %% """ Multilable classification KNeighborsClassifier f1_score -- assumes all labels are equally important and hence simple average other option average="weighted" -- to give each label a weight equal to its support (i.e., the number of instances with that target label) """ from sklearn.neighbors import KNeighborsClassifier y_train_large = (y_train >= 7)
x = np.array([[1, 3], [2, 4]]) y = np.array([[1, 0], [0, 1]]) z = np.array([[x, x], [y, y]]) #2 a @ a.T np.add(b, a.T) A @ A.T A**3 A @ B #3 A = np.random.randn(10, 5) B = np.random.randn(5, 10) C = A @ B np.fill_diagonal(C, 1) #4 D = np.tile(A, (2, 3)) E = np.tile(B, (3, 2)) F = D @ E F[F <= 0.5] = 0 #5 np.reshape(D, (3, 10, 10)) np.reshape(E, (3, 10, 10)) #6 x = np.arange(100.0) np.reshape(x, (10, 10)) np.reshape(x, (20, 5))
#remove the small streamlines streamlines_cut = lambda: (sl for sl in streamlines if len(sl) > 1) streamlines = Streamlines(streamlines_cut()) print runno + ' streamlines paparation finished' # In[5]: #build the connectivity matrix M = utils.connectivity_matrix(streamlines, labels_, affine=affine_labels, return_mapping=False, mapping_as_streamlines=False) #fill dialgonal with 0 np.fill_diagonal(M, 0) print runno + ' connectivity matrix building finished' # In[20]: M = M[1:, 1:] # In[22]: plt.imshow(M) # In[11]: np.diag(M)
def sampler(self, w_limit, tau_limit, file): start = time.time() # ------------------- initialize MCMC testsize = self.testdata.shape[0] trainsize = self.traindata.shape[0] samples = self.samples self.sgd_depth = 1 learn_rate = 0.5 x_test = np.linspace(0, 1, num=testsize) x_train = np.linspace(0, 1, num=trainsize) Netlist = [None] * 10 netsize = np.zeros(self.subtasks, dtype=np.int) for n in xrange(0, self.subtasks): module = self.mtaskNet[n] trdata = self.taskdata( self.traindata, module[0], module[2]) # make the partitions for task data testdata = self.taskdata(self.testdata, module[0], module[2]) Netlist[n] = Network(self.mtaskNet[n], trdata, testdata, learn_rate) # print("Size: "+str(n)+" " + str(trdata)+ " " + str(module[0])) # print trdata for n in xrange(0, self.subtasks): netw = Netlist[n].Top netsize[n] = self.net_size(netw) # num of weights and bias # print(netsize[n]) y_test = self.testdata[:, netw[0]:] y_train = self.traindata[:, netw[0]:] pos_w = np.ones( (samples, self.subtasks, netsize[self.subtasks - 1])) # posterior of all weights and bias over all samples pos_tau = np.ones((samples, self.subtasks)) fxtrain_samples = np.ones( (samples, self.subtasks, trainsize, netw[2])) # fx of train data over all samples fxtest_samples = np.ones((samples, self.subtasks, testsize, netw[2])) # fx of test data over all samples rmse_train = np.zeros((samples, self.subtasks)) rmse_test = np.zeros((samples, self.subtasks)) w = np.random.randn(self.subtasks, netsize[self.subtasks - 1]) w_proposal = np.random.randn(self.subtasks, netsize[self.subtasks - 1]) w_gd = np.random.randn(self.subtasks, netsize[self.subtasks - 1]) w_prop_gd = np.random.randn(self.subtasks, netsize[self.subtasks - 1]) # step_w = 0.05; # defines how much variation you need in changes to w # step_eta = 0.2; # exp 0 step_w = w_limit # defines how much variation you need in changes to w step_eta = tau_limit # exp 1 # --------------------- Declare FNN and initialize pred_train = Netlist[0].evaluate_proposal(self.traindata, w[0, :netsize[0]], self.mtaskNet, 0) pred_test = Netlist[0].evaluate_proposal(self.testdata, w[0, :netsize[0]], self.mtaskNet, 0) rmsetrain = np.zeros(self.subtasks) rmsetest = np.zeros(self.subtasks) eta = np.log(np.var(pred_train - y_train)) tau_pro = np.exp(eta) pred_train = np.zeros((self.subtasks, trainsize, Netlist[0].Top[2])) pred_test = np.zeros((self.subtasks, testsize, Netlist[0].Top[2])) sigma_squared = 25 nu_1 = 0 nu_2 = 0 sigma_diagmat = np.zeros( (self.subtasks, netsize[self.subtasks - 1], netsize[self.subtasks - 1])) # for Equation 9 in Ref [Chandra_ICONIP2017] for s in xrange(self.subtasks): np.fill_diagonal(sigma_diagmat[s], step_w) delta_likelihood = 0.5 # an arbitrary position likelihood = np.zeros(self.subtasks) likelihood_proposal = np.zeros(self.subtasks) likelihood_ignore = np.zeros(self.subtasks) prior_current = np.zeros(self.subtasks) prior_pro = np.zeros(self.subtasks) for n in xrange(0, self.subtasks): prior_current[n] = self.prior_likelihood( sigma_squared, nu_1, nu_2, w[n, :netsize[0]], tau_pro, Netlist[n].Top) # takes care of the gradients mh_prob = np.zeros(self.subtasks) for s in xrange(0, self.subtasks - 1): [likelihood[s], fxtrain_samples[0, s, :], rmse_train[0, s] ] = self.likelihood_func(Netlist[s], self.traindata, w[s, :netsize[s]], tau_pro, s) [likelihood_ignore, fxtest_samples[0, s, :], rmse_test[0, s]] = self.likelihood_func(Netlist[s], self.testdata, w[s, :netsize[s]], tau_pro, s) w[s + 1, :netsize[s]] = w[s, :netsize[s]] s = self.subtasks - 1 [likelihood[s], fxtrain_samples[0, s, :], rmse_train[0, s]] = self.likelihood_func(Netlist[s], self.traindata, w[s, :netsize[s]], tau_pro, s) # print likelihood naccept = 0 # print 'begin sampling using mcmc random walk' # plt.plot(x_train, y_train) # plt.plot(x_train, pred_train) # plt.title("Plot of Data vs Initial Fx") # plt.savefig('mcmcresults/begin.png') # plt.clf() # plt.plot(x_train, y_train) diff_prop = np.zeros(self.subtasks) diff = np.zeros(self.subtasks) prior_prop = np.zeros(self.subtasks) for i in range(1, samples - 1): # print i for s in xrange(self.subtasks): # print("B2:") # print(Netlist[s].B2.shape) # print(netsize[s], Netlist[s].Top) w_gd[s, :netsize[s]] = Netlist[s].langevin_gradient( self.traindata, w[s, :netsize[s]].copy(), self.sgd_depth, self.mtaskNet, s) # Eq 8 w_proposal[ s, :netsize[s]] = w_gd[s, :netsize[s]] + np.random.normal( 0, step_w, netsize[s]) # Eq 7 w_prop_gd[s, :netsize[s]] = Netlist[s].langevin_gradient( self.traindata, w_proposal[s].copy(), self.sgd_depth, self.mtaskNet, s) # print(multivariate_normal.pdf(w, w_prop_gd, sigma_diagmat),multivariate_normal.pdf(w_proposal, w_gd, sigma_diagmat)) for s in xrange(self.subtasks): diff_prop[s] = np.log( multivariate_normal.pdf( w[s, :netsize[s]], w_prop_gd[s, :netsize[s]], sigma_diagmat[s, :netsize[s], :netsize[s]]) - np.log( multivariate_normal.pdf( w_proposal[s, :netsize[s]], w_gd[s, :netsize[s]], sigma_diagmat[s, :netsize[s], :netsize[s]]))) eta_pro = eta + np.random.normal(0, step_eta, 1) tau_pro = math.exp(eta_pro) for s in xrange(self.subtasks - 1): [likelihood_proposal[s], pred_train[s, :], rmsetrain[s] ] = self.likelihood_func(Netlist[s], self.traindata, w_proposal[s, :netsize[s]], tau_pro, s) [_, pred_test[s, :], rmsetest[s] ] = self.likelihood_func(Netlist[s], self.testdata, w_proposal[s, :netsize[s]], tau_pro, s) w_proposal[s + 1, :netsize[s]] = w_proposal[s, :netsize[s]] s = self.subtasks - 1 [likelihood_proposal[s], pred_train[s, :], rmsetrain[s]] = self.likelihood_func(Netlist[s], self.traindata, w_proposal[s, :netsize[s]], tau_pro, s) [_, pred_test[s, :], rmsetest[s]] = self.likelihood_func(Netlist[s], self.testdata, w_proposal[s, :netsize[s]], tau_pro, s) # likelihood_ignore refers to parameter that will not be used in the alg. for s in xrange(self.subtasks): prior_prop[s] = self.prior_likelihood( sigma_squared, nu_1, nu_2, w_proposal[s, :netsize[s]], tau_pro, Netlist[s].Top) # takes care of the gradients diff_prior = prior_prop - prior_current diff_likelihood = likelihood_proposal - likelihood for s in xrange(self.subtasks): diff[s] = min( 700, diff_prior[s] + diff_likelihood[s] + diff_prop[s]) mh_prob[s] = min(1, math.exp(diff[s])) # print() # print(diff, i) # print(mh_prob) u = random.uniform(0, 1) if u < mh_prob[s]: # Update position # print i, ' is accepted sample' naccept += 1 likelihood[s] = likelihood_proposal[s] prior_current[s] = prior_prop[s] w[s, :netsize[s]] = w_proposal[s, :netsize[s]] eta = eta_pro elapsed_time = ":".join( covert_time(int(time.time() - start))) # sys.stdout.write('\r' + file + ' : ' + str(round(float(i) / (samples - 1) * 100, 2)) + '% complete....'+" time elapsed: " + elapsed_time) # print likelihood, prior_current, diff_prop, rmsetrain, rmsetest, w, 'accepted' # print w_proposal, 'w_proposal' # print w_gd, 'w_gd' # print w_prop_gd, 'w_prop_gd' pos_w[i + 1, s, :netsize[s]] = w_proposal[s, :netsize[s]] pos_tau[i + 1] = tau_pro fxtrain_samples[i + 1, s, :] = pred_train[s, :] fxtest_samples[i + 1, s, :] = pred_test[s, :] rmse_train[i + 1, s] = rmsetrain[s] rmse_test[i + 1, s] = rmsetest[s] print i, 'accepted' # plt.plot(x_train, pred_train) else: pos_w[i + 1, s, :netsize[s]] = pos_w[i, s, :netsize[s]] pos_tau[i + 1, s] = pos_tau[i, s] fxtrain_samples[i + 1, s, :] = fxtrain_samples[i, s, :] fxtest_samples[i + 1, s, :] = fxtest_samples[i, s, :] rmse_train[i + 1, s] = rmse_train[i, s] rmse_test[i + 1, s] = rmse_test[i, s] print i, 'rejected and retained' sys.stdout.write('\r' + file + ' : 100% ..... Total Time: ' + ":".join(covert_time(int(time.time() - start)))) # print naccept, ' num accepted' # print naccept / (samples * 1.0), '% was accepted' accept_ratio = naccept / (samples * self.subtasks * 1.0) * 100 # plt.title("Plot of Accepted Proposals") # plt.savefig('mcmcresults/proposals.png') # plt.savefig('mcmcresults/proposals.svg', format='svg', dpi=600) # plt.clf() return (pos_w, pos_tau, fxtrain_samples, fxtest_samples, x_train, x_test, rmse_train, rmse_test, accept_ratio)
def compute_cim_coefficients(self) -> None: """Compute the coefficients of the matrix _cim by using the following equality q_xx' = M[x, x'] / T[x]. The class member ``_cim`` will contain the computed cim """ np.fill_diagonal(self._cim, self._cim.diagonal() * -1) self._cim = ((self._cim.T + 1) / (self._state_residence_times + 1)).T
mmc = conf_in.mean() ece, mce = get_calib(py_in, targets) save_res_ood(tab_ood['MNIST - MNIST'], mmc) save_res_cal(tab_cal['MAP'], ece, mce) print( f'[In, MAP] Time: NA/{time_pred:.1f}s; Accuracy: {acc_in:.3f}; ECE: {ece:.3f}; MCE: {mce:.3f}; MMC: {mmc:.3f}' ) # ---------------------------------------------------------------------- # Make a binary classification dataset if args.generate_confmat: from sklearn.metrics import confusion_matrix C = confusion_matrix(targets, py_in.argmax(1)) print(C) np.fill_diagonal(C, -1) max_idx = np.unravel_index(C.argmax(), shape=C.shape) max_val = C.max() print(f'Most confused index: {max_idx}, value: {max_val}') sys.exit() # ---------------------------------------------------------------------- # Out-distribution - EMNIST py_out = predict(test_loader_EMNIST, model).cpu().numpy() conf_emnist = get_confidence(py_out) mmc = conf_emnist.mean() auroc = get_auroc(py_in, py_out) save_res_ood(tab_ood['MNIST - EMNIST'], mmc, auroc) print(f'[Out-EMNIST, MAP] MMC: {mmc:.3f}; AUROC: {auroc:.3f}')
data['gabamat'] = np.zeros((data['n_n'], data['n_n'])) n_n = data['n_n'] l_n = int(0.25 * n_n) p_n = int(0.75 * n_n) ach_mat = np.zeros((n_n, n_n)) ach_mat[p_n:, :p_n] = np.random.choice([0., 1.], size=(l_n, p_n), p=(1 - data['PNLN'], data['PNLN'])) # PN->LN ach_mat[:p_n, :p_n] = np.random.choice([0., 1.], size=(p_n, p_n), p=(1 - data['PNPN'], data['PNPN'])) # PN->PN np.fill_diagonal(ach_mat, 0.) data['achmat'] = ach_mat gaba_mat = np.zeros((n_n, n_n)) gaba_mat[:p_n, p_n:] = np.random.choice([0., 1.], size=(p_n, l_n), p=(1 - data['LNPN'], data['LNPN'])) # LN->PN gaba_mat[p_n:, p_n:] = np.random.choice([0., 1.], size=(l_n, l_n), p=(1 - data['LNLN'], data['LNLN'])) # LN->LN np.fill_diagonal(gaba_mat, 0.) data['gabamat'] = gaba_mat with open('model.pkl', 'wb') as fp:
def get_neighb_matrix(coords, resolution=500, size=1): D = cdist(coords, coords, 'euclidean') lim = np.ceil(size*resolution*np.sqrt(2)) NM = (D < lim).astype(int) np.fill_diagonal(NM, 0) return NM
def diagadd(m, v): """Add a vector to the diagonal of a matrix""" np.fill_diagonal(m, m.diagonal() + v)
def _linprog_simplex(c, A_ub=None, b_ub=None, A_eq=None, b_eq=None, bounds=None, maxiter=1000, disp=False, callback=None, tol=1.0E-12, bland=False, **unknown_options): """ Solve the following linear programming problem via a two-phase simplex algorithm. maximize: c^T * x subject to: A_ub * x <= b_ub A_eq * x == b_eq Parameters ---------- c : array_like Coefficients of the linear objective function to be maximized. A_ub : array_like 2-D array which, when matrix-multiplied by x, gives the values of the upper-bound inequality constraints at x. b_ub : array_like 1-D array of values representing the upper-bound of each inequality constraint (row) in A_ub. A_eq : array_like 2-D array which, when matrix-multiplied by x, gives the values of the equality constraints at x. b_eq : array_like 1-D array of values representing the RHS of each equality constraint (row) in A_eq. bounds : array_like The bounds for each independent variable in the solution, which can take one of three forms:: None : The default bounds, all variables are non-negative. (lb, ub) : If a 2-element sequence is provided, the same lower bound (lb) and upper bound (ub) will be applied to all variables. [(lb_0, ub_0), (lb_1, ub_1), ...] : If an n x 2 sequence is provided, each variable x_i will be bounded by lb[i] and ub[i]. Infinite bounds are specified using -np.inf (negative) or np.inf (positive). callback : callable If a callback function is provide, it will be called within each iteration of the simplex algorithm. The callback must have the signature `callback(xk, **kwargs)` where xk is the current solution vector and kwargs is a dictionary containing the following:: "tableau" : The current Simplex algorithm tableau "nit" : The current iteration. "pivot" : The pivot (row, column) used for the next iteration. "phase" : Whether the algorithm is in Phase 1 or Phase 2. "bv" : A structured array containing a string representation of each basic variable and its current value. Options ------- maxiter : int The maximum number of iterations to perform. disp : bool If True, print exit status message to sys.stdout tol : float The tolerance which determines when a solution is "close enough" to zero in Phase 1 to be considered a basic feasible solution or close enough to positive to to serve as an optimal solution. bland : bool If True, use Bland's anti-cycling rule [3] to choose pivots to prevent cycling. If False, choose pivots which should lead to a converged solution more quickly. The latter method is subject to cycling (non-convergence) in rare instances. Returns ------- A scipy.optimize.OptimizeResult consisting of the following fields:: x : ndarray The independent variable vector which optimizes the linear programming problem. slack : ndarray The values of the slack variables. Each slack variable corresponds to an inequality constraint. If the slack is zero, then the corresponding constraint is active. success : bool Returns True if the algorithm succeeded in finding an optimal solution. status : int An integer representing the exit status of the optimization:: 0 : Optimization terminated successfully 1 : Iteration limit reached 2 : Problem appears to be infeasible 3 : Problem appears to be unbounded nit : int The number of iterations performed. message : str A string descriptor of the exit status of the optimization. Examples -------- Consider the following problem: Minimize: f = -1*x[0] + 4*x[1] Subject to: -3*x[0] + 1*x[1] <= 6 1*x[0] + 2*x[1] <= 4 x[1] >= -3 where: -inf <= x[0] <= inf This problem deviates from the standard linear programming problem. In standard form, linear programming problems assume the variables x are non-negative. Since the variables don't have standard bounds where 0 <= x <= inf, the bounds of the variables must be explicitly set. There are two upper-bound constraints, which can be expressed as dot(A_ub, x) <= b_ub The input for this problem is as follows: >>> from scipy.optimize import linprog >>> c = [-1, 4] >>> A = [[-3, 1], [1, 2]] >>> b = [6, 4] >>> x0_bnds = (None, None) >>> x1_bnds = (-3, None) >>> res = linprog(c, A, b, bounds=(x0_bnds, x1_bnds)) >>> print(res) fun: -22.0 message: 'Optimization terminated successfully.' nit: 1 slack: array([ 39., 0.]) status: 0 success: True x: array([ 10., -3.]) References ---------- .. [1] Dantzig, George B., Linear programming and extensions. Rand Corporation Research Study Princeton Univ. Press, Princeton, NJ, 1963 .. [2] Hillier, S.H. and Lieberman, G.J. (1995), "Introduction to Mathematical Programming", McGraw-Hill, Chapter 4. .. [3] Bland, Robert G. New finite pivoting rules for the simplex method. Mathematics of Operations Research (2), 1977: pp. 103-107. """ _check_unknown_options(unknown_options) status = 0 messages = { 0: "Optimization terminated successfully.", 1: "Iteration limit reached.", 2: "Optimization failed. Unable to find a feasible" " starting point.", 3: "Optimization failed. The problem appears to be unbounded.", 4: "Optimization failed. Singular matrix encountered." } have_floor_variable = False cc = np.asarray(c) # The initial value of the objective function element in the tableau f0 = 0 # The number of variables as given by c n = len(c) # Convert the input arguments to arrays (sized to zero if not provided) Aeq = np.asarray(A_eq) if A_eq is not None else np.empty([0, len(cc)]) Aub = np.asarray(A_ub) if A_ub is not None else np.empty([0, len(cc)]) beq = np.ravel(np.asarray(b_eq)) if b_eq is not None else np.empty([0]) bub = np.ravel(np.asarray(b_ub)) if b_ub is not None else np.empty([0]) # Analyze the bounds and determine what modifications to be made to # the constraints in order to accommodate them. L = np.zeros(n, dtype=np.float64) U = np.ones(n, dtype=np.float64) * np.inf if bounds is None or len(bounds) == 0: pass elif len(bounds) == 2 and not hasattr(bounds[0], '__len__'): # All bounds are the same a = bounds[0] if bounds[0] is not None else -np.inf b = bounds[1] if bounds[1] is not None else np.inf L = np.asarray(n * [a], dtype=np.float64) U = np.asarray(n * [b], dtype=np.float64) else: if len(bounds) != n: status = -1 message = ("Invalid input for linprog with method = 'simplex'. " "Length of bounds is inconsistent with the length of c") else: try: for i in range(n): if len(bounds[i]) != 2: raise IndexError() L[i] = bounds[i][0] if bounds[i][0] is not None else -np.inf U[i] = bounds[i][1] if bounds[i][1] is not None else np.inf except IndexError: status = -1 message = ("Invalid input for linprog with " "method = 'simplex'. bounds must be a n x 2 " "sequence/array where n = len(c).") if np.any(L == -np.inf): # If any lower-bound constraint is a free variable # add the first column variable as the "floor" variable which # accommodates the most negative variable in the problem. n = n + 1 L = np.concatenate([np.array([0]), L]) U = np.concatenate([np.array([np.inf]), U]) cc = np.concatenate([np.array([0]), cc]) Aeq = np.hstack([np.zeros([Aeq.shape[0], 1]), Aeq]) Aub = np.hstack([np.zeros([Aub.shape[0], 1]), Aub]) have_floor_variable = True # Now before we deal with any variables with lower bounds < 0, # deal with finite bounds which can be simply added as new constraints. # Also validate bounds inputs here. for i in range(n): if (L[i] > U[i]): status = -1 message = ("Invalid input for linprog with method = 'simplex'. " "Lower bound %d is greater than upper bound %d" % (i, i)) if np.isinf(L[i]) and L[i] > 0: status = -1 message = ("Invalid input for linprog with method = 'simplex'. " "Lower bound may not be +infinity") if np.isinf(U[i]) and U[i] < 0: status = -1 message = ("Invalid input for linprog with method = 'simplex'. " "Upper bound may not be -infinity") if np.isfinite(L[i]) and L[i] > 0: # Add a new lower-bound (negative upper-bound) constraint Aub = np.vstack([Aub, np.zeros(n)]) Aub[-1, i] = -1 bub = np.concatenate([bub, np.array([-L[i]])]) L[i] = 0 if np.isfinite(U[i]): # Add a new upper-bound constraint Aub = np.vstack([Aub, np.zeros(n)]) Aub[-1, i] = 1 bub = np.concatenate([bub, np.array([U[i]])]) U[i] = np.inf # Now find negative lower bounds (finite or infinite) which require a # change of variables or free variables and handle them appropriately for i in range(0, n): if L[i] < 0: if np.isfinite(L[i]) and L[i] < 0: # Add a change of variables for x[i] # For each row in the constraint matrices, we take the # coefficient from column i in A, # and subtract the product of that and L[i] to the RHS b beq = beq - Aeq[:, i] * L[i] bub = bub - Aub[:, i] * L[i] # We now have a nonzero initial value for the objective # function as well. f0 = f0 - cc[i] * L[i] else: # This is an unrestricted variable, let x[i] = u[i] - v[0] # where v is the first column in all matrices. Aeq[:, 0] = Aeq[:, 0] - Aeq[:, i] Aub[:, 0] = Aub[:, 0] - Aub[:, i] cc[0] = cc[0] - cc[i] if np.isinf(U[i]): if U[i] < 0: status = -1 message = ("Invalid input for linprog with " "method = 'simplex'. Upper bound may not be -inf.") # The number of upper bound constraints (rows in A_ub and elements in b_ub) mub = len(bub) # The number of equality constraints (rows in A_eq and elements in b_eq) meq = len(beq) # The total number of constraints m = mub + meq # The number of slack variables (one for each of the upper-bound constraints) n_slack = mub # The number of artificial variables (one for each lower-bound and equality # constraint) n_artificial = meq + np.count_nonzero(bub < 0) try: Aub_rows, Aub_cols = Aub.shape except ValueError: raise ValueError("Invalid input. A_ub must be two-dimensional") try: Aeq_rows, Aeq_cols = Aeq.shape except ValueError: raise ValueError("Invalid input. A_eq must be two-dimensional") if Aeq_rows != meq: status = -1 message = ("Invalid input for linprog with method = 'simplex'. " "The number of rows in A_eq must be equal " "to the number of values in b_eq") if Aub_rows != mub: status = -1 message = ("Invalid input for linprog with method = 'simplex'. " "The number of rows in A_ub must be equal " "to the number of values in b_ub") if Aeq_cols > 0 and Aeq_cols != n: status = -1 message = ("Invalid input for linprog with method = 'simplex'. " "Number of columns in A_eq must be equal " "to the size of c") if Aub_cols > 0 and Aub_cols != n: status = -1 message = ("Invalid input for linprog with method = 'simplex'. " "Number of columns in A_ub must be equal to the size of c") if status != 0: # Invalid inputs provided raise ValueError(message) # Create the tableau T = np.zeros([m + 2, n + n_slack + n_artificial + 1]) # Insert objective into tableau T[-2, :n] = cc T[-2, -1] = f0 b = T[:-2, -1] if meq > 0: # Add Aeq to the tableau T[:meq, :n] = Aeq # Add beq to the tableau b[:meq] = beq if mub > 0: # Add Aub to the tableau T[meq:meq + mub, :n] = Aub # At bub to the tableau b[meq:meq + mub] = bub # Add the slack variables to the tableau np.fill_diagonal(T[meq:m, n:n + n_slack], 1) # Further set up the tableau. # If a row corresponds to an equality constraint or a negative b (a lower # bound constraint), then an artificial variable is added for that row. # Also, if b is negative, first flip the signs in that constraint. slcount = 0 avcount = 0 basis = np.zeros(m, dtype=int) r_artificial = np.zeros(n_artificial, dtype=int) for i in range(m): if i < meq or b[i] < 0: # basic variable i is in column n+n_slack+avcount basis[i] = n + n_slack + avcount r_artificial[avcount] = i avcount += 1 if b[i] < 0: b[i] *= -1 T[i, :-1] *= -1 T[i, basis[i]] = 1 T[-1, basis[i]] = 1 else: # basic variable i is in column n+slcount basis[i] = n + slcount slcount += 1 # Make the artificial variables basic feasible variables by subtracting # each row with an artificial variable from the Phase 1 objective for r in r_artificial: T[-1, :] = T[-1, :] - T[r, :] nit1, status = _solve_simplex(T, n, basis, phase=1, callback=callback, maxiter=maxiter, tol=tol, bland=bland) # if pseudo objective is zero, remove the last row from the tableau and # proceed to phase 2 if abs(T[-1, -1]) < tol: # Remove the pseudo-objective row from the tableau T = T[:-1, :] # Remove the artificial variable columns from the tableau T = np.delete(T, np.s_[n + n_slack:n + n_slack + n_artificial], 1) else: # Failure to find a feasible starting point status = 2 if status != 0: message = messages[status] if disp: print(message) return OptimizeResult(x=np.nan, fun=-T[-1, -1], nit=nit1, status=status, message=message, success=False) # Phase 2 nit2, status = _solve_simplex(T, n, basis, maxiter=maxiter - nit1, phase=2, callback=callback, tol=tol, nit0=nit1, bland=bland) solution = np.zeros(n + n_slack + n_artificial) solution[basis[:m]] = T[:m, -1] x = solution[:n] slack = solution[n:n + n_slack] # For those variables with finite negative lower bounds, # reverse the change of variables masked_L = np.ma.array(L, mask=np.isinf(L), fill_value=0.0).filled() x = x + masked_L # For those variables with infinite negative lower bounds, # take x[i] as the difference between x[i] and the floor variable. if have_floor_variable: for i in range(1, n): if np.isinf(L[i]): x[i] -= x[0] x = x[1:] # Optimization complete at this point obj = -T[-1, -1] if status in (0, 1): if disp: print(messages[status]) print(" Current function value: {0: <12.6f}".format(obj)) print(" Iterations: {0:d}".format(nit2)) else: if disp: print(messages[status]) print(" Iterations: {0:d}".format(nit2)) return OptimizeResult(x=x, fun=obj, nit=int(nit2), status=status, slack=slack, message=messages[status], success=(status == 0))
def _check_counts_matrix(counts, lengths, ploidy, exclude_zeros=False, chrom_subset_index=None): """Check counts dimensions, reformat, & excise selected chromosomes. """ if chrom_subset_index is not None and len(chrom_subset_index) / max( counts.shape) not in (1, 2): raise ValueError( "chrom_subset_index size (%d) does not fit counts" " shape (%d, %d)." % (len(chrom_subset_index), counts.shape[0], counts.shape[1])) if len(counts.shape) != 2: raise ValueError( "Counts matrix must be two-dimensional, current shape = (%s)" % ', '.join([str(x) for x in counts.shape])) if any([x > lengths.sum() * ploidy for x in counts.shape]): raise ValueError( "Counts matrix shape (%d, %d) is greater than number" " of beads (%d) in %s genome." % (counts.shape[0], counts.shape[1], lengths.sum() * ploidy, { 1: "haploid", 2: "diploid" }[ploidy])) if any([x / lengths.sum() not in (1, 2) for x in counts.shape]): raise ValueError( "Counts matrix shape (%d, %d) does not match lenghts" " (%s)" % (counts.shape[0], counts.shape[1], ",".join(map(str, lengths)))) empty_val = 0 torm = np.full((max(counts.shape)), False) if not exclude_zeros: empty_val = np.nan torm = find_beads_to_remove(counts, max(counts.shape)) counts = counts.astype(float) if sparse.issparse(counts) or isinstance(counts, CountsMatrix): counts = counts.toarray() if not isinstance(counts, np.ndarray): counts = np.array(counts) if not np.array_equal(counts[~np.isnan(counts)], counts[~np.isnan(counts)].round()): warn("Counts matrix must only contain integers or NaN") if counts.shape[0] == counts.shape[1]: counts[np.tril_indices(counts.shape[0])] = empty_val counts[torm, :] = empty_val counts[:, torm] = empty_val if chrom_subset_index is not None: counts = counts[ chrom_subset_index[:counts. shape[0]], :][:, chrom_subset_index[:counts. shape[1]]] elif min(counts.shape) * 2 == max(counts.shape): homo1 = counts[:min(counts.shape), :min(counts.shape)] homo2 = counts[counts.shape[0] - min(counts.shape):, counts.shape[1] - min(counts.shape):] if counts.shape[0] == min(counts.shape): homo1 = homo1.T homo2 = homo2.T np.fill_diagonal(homo1, empty_val) np.fill_diagonal(homo2, empty_val) homo1[:, torm[:min(counts.shape)] | torm[min(counts.shape):]] = empty_val homo2[:, torm[:min(counts.shape)] | torm[min(counts.shape):]] = empty_val # axis=0 is vertical concat counts = np.concatenate([homo1, homo2], axis=0) counts[torm, :] = empty_val if chrom_subset_index is not None: counts = counts[ chrom_subset_index[:counts. shape[0]], :][:, chrom_subset_index[:counts. shape[1]]] else: raise ValueError("Input counts matrix is - %d by %d. Counts must be" " n-by-n or n-by-2n or 2n-by-2n." % (counts.shape[0], counts.shape[1])) if exclude_zeros: counts[np.isnan(counts)] = 0 counts = sparse.coo_matrix(counts) return counts
def implementCircuitModel(self, initConditions, circuitParams, model, time, dx, eps, frequency, delta_phi0): epsilonR = circuitParams.epsilonR epsilonOx = circuitParams.epsilonOx epsilonStern = circuitParams.epsilonStern C0 = circuitParams.C0 Cox = circuitParams.Cox Cstern = circuitParams.Cstern #CB=circuitParams.CB R = circuitParams.R cInit = initConditions.cInit c0 = cInit[0] / 2 q0Init = initConditions.q0Init v0Init = initConditions.v0Init vOxInit = initConditions.vOxInit vSternInit = initConditions.vSternInit vBInit = initConditions.vBInit c0Init = initConditions.c0Init R0Init = initConditions.R0Init simulationType = model.simulationType Normal = model.NormalCircuit Reduced = model.ReducedCircuit circuitModel = model.circuitModel LinCapModel = model.LinCapModel NonLinCapModel = model.NonLinCapModel NonLinCapVarResModel = model.NonLinCapVarResModel NonLinCapInhomResModel = model.NonLinCapInhomResModel NonLinCapVarResROxModel = model.NonLinCapVarResROxModel if circuitModel == LinCapModel: iInit = ([ 0, (delta_phi0 - (v0Init + vOxInit + vSternInit)) / R, v0Init, vOxInit, vSternInit, delta_phi0 - (v0Init + vOxInit + vSternInit) ]) i = odeint(circuitModel_LinCap, iInit, time, args=(R, C0, Cox, Cstern, frequency, delta_phi0)) charge = i[:, 0] current = i[:, 1] vEdl = i[:, 2] vOx = i[:, 3] vStern = i[:, 4] vBulk = i[:, -1] m = 1. / dx c = np.ones((len(vBulk), m)) c = 2 * c cTot = (0.5 * (c[:, 0] + c[:, -1]) + np.sum(c[:, 1:-1], axis=1)) * dx R = R * np.ones(np.shape(vBulk)) timeBulk = time elif circuitModel == NonLinCapModel: iInit = ([ vOxInit * Cox, (delta_phi0 - 2 * (v0Init + vOxInit + vSternInit)) / R, v0Init, vOxInit, vSternInit, delta_phi0 - 2 * (v0Init + vOxInit + vSternInit) ]) i = odeint(circuitModel_NonLinCap, iInit, time, args=(R, C0, Cox, Cstern, frequency, delta_phi0)) charge = i[:, 0] current = i[:, 1] vEdl = i[:, 2] vOx = i[:, 3] vStern = i[:, 4] vBulk = i[:, -1] m = 1. / dx c = cInit[0] * np.ones((len(vBulk), m)) c = 2 * c cTot = (0.5 * (c[:, 0] + c[:, -1]) + np.sum(c[:, 1:-1], axis=1)) * dx R = R * np.ones(np.shape(vBulk)) timeBulk = time elif circuitModel == NonLinCapVarResModel: # #nFreq=len(frequency) # iInit=([q0Init,(vBInit-(v0Init+vOxInit+vSternInit))/R,v0Init,vOxInit,vSternInit,c0,R,vBInit-(v0Init+vOxInit+vSternInit)]) # iInit=np.append(q0Init,(vBInit-(v0Init+vOxInit+vSternInit))/R)#([0,delta_phi0/R,0,R,cInit,eps,delta_phi0]) # iInit=np.append(iInit,v0Init) # iInit=np.append(iInit,vOxInit) # iInit=np.append(iInit,vSternInit) # iInit=np.append(iInit,c0Init) # iInit=np.append(iInit,R0Init) # iInit=np.append(iInit,vBInit-(v0Init+vOxInit+vSternInit)) iInit = ([ q0Init, (vBInit - (v0Init + vOxInit + vSternInit)) / R, v0Init, vOxInit, vSternInit, c0, R, vBInit - (v0Init + vOxInit + vSternInit) ]) # charge,current,vEdl,vOx,vStern,R,c,vBulk=odeint(circuitModel_NonLinCap_VarRes,iInit,time,args=(C0,Cox,Cstern,frequency,delta_phi0)); i = odeint(circuitModel_NonLinCap_VarRes, iInit, time, args=(C0, Cox, Cstern, frequency, delta_phi0)) #print("i",i,np.shape(i)) [charge, current, vEdl, vOx, vStern, c, R, vBulk] = [i[:, m] for m in range(len(iInit))] # charge=i[:,0] # current=i[:,1] # vEdl=i[:,2] # vOx=i[:,3] # vStern=i[:,4] # R=i[:,5] # c=i[:,6] # vBulk=i[:,-1] # m=int(1./dx); # c=np.multiply(np.reshape(c,(len(vBulk),1)),np.matrix(np.ones((len(vBulk),m)))) # c=2*c # cTot=(0.5*(c[:,0]+c[:,-1])+np.sum(c[:,1:-1],axis=1))*dx c = 2 * c cTot = np.array([2.0]) #print("time",time,len(time)) timeBulk = time #plt.plot(time,R) # plt.plot(time,vBulk) elif circuitModel == NonLinCapVarResROxModel: ROx = circuitParams.ROx iInit = ([ q0Init, (vBInit - (v0Init + vOxInit + vSternInit)) / R, v0Init, vOxInit, vSternInit, c0, R, vBInit - (v0Init + vOxInit + vSternInit) ]) #iInit=([0,(vBInit-(v0Init+vOxInit+vSternInit))/R,v0Init,vOxInit,vSternInit,c0,R,vBInit-(v0Init+vOxInit+vSternInit)]) i = odeint(circuitModelNonLinCapROxVarRes, iInit, time, args=(C0, Cox, Cstern, ROx, frequency, delta_phi0)) [charge, current, vEdl, vOx, vStern, c, R, vBulk] = [i[:, m] for m in range(len(iInit))] # charge=i[:,0] # current=i[:,1] # vEdl=i[:,2] # vOx=i[:,3] # vStern=i[:,4] # R=i[:,5] # c=i[:,6] # vBulk=i[:,-1] # m=int(1./dx); # c=np.multiply(np.reshape(c,(len(vBulk),1)),np.matrix(np.ones((len(vBulk),m)))) c = 2 * c cTot = np.array([2.0]) #(0.5*(c[:,0]+c[:,-1])+np.sum(c[:,1:-1],axis=1))*dx timeBulk = time elif circuitModel == NonLinCapInhomResModel: m = int(1.0 / dx) n = int(1.0 / dx) cInit = (2 * c0 * np.ones((m, 1))) x = np.linspace(0, 1, m) diags = np.asarray([1.0, 2.0, 1.0]) A = np.zeros((n, m), diags.dtype) b = np.zeros((n, 1), diags.dtype) for z, v in enumerate((1.0, -2.0, 1.0)): np.fill_diagonal(A[1:-1, z:], v) A[0, 0] = -2.0 A[0, 1] = 2.0 A[-1, -1] = -2.0 A[-1, -2] = 2.0 A = A / (dx**2) A = np.matrix(A) b[0] = -2 b[-1] = -2 iInit = np.append(0, (vBInit - (v0Init + vOxInit + vSternInit)) / R) #([0,delta_phi0/R,0,R,cInit,eps,delta_phi0]) iInit = np.append(iInit, v0Init) iInit = np.append(iInit, vOxInit) iInit = np.append(iInit, vSternInit) iInit = np.append(iInit, R) iInit = np.append(iInit, cInit) iInit = np.append(iInit, vBInit - (v0Init + vOxInit + vSternInit)) #print iInit i = odeint(circuitModel_NonLinCap_InhomResPython, iInit, time, args=(dx, eps, C0, Cox, Cstern, frequency, delta_phi0, A, b)) charge = i[:, 0] current = i[:, 1] vEdl = i[:, 2] vOx = i[:, 3] vStern = i[:, 4] R = i[:, 5] vBulk = i[:, -1] c = i[:, 6:-1] cTot = (0.5 * (c[:, 0] + c[:, -1]) + np.sum(c[:, 1:-1], axis=1)) * dx timeBulk = time return [ charge, current, vEdl, vOx, vStern, c, cTot, R, vBulk, timeBulk ]
# Q2 # Jacobi iteration method A = np.array([[8,-1,3,-1], [-1,6,2,0], [3,2,9,1], [-1,0,1,7]], dtype = float) ## Sign Function import numpy as np def sign(x): if x >= 0: return 1 else: return -1 while True: mask = np.ones(A.shape, dtype = bool) np.fill_diagonal(mask, 0) idx = np.where(abs(A) == abs(A[mask]).max()) p = idx[0][0] q = idx[0][1] rho = 0.5*(A[q][q] - A[p][p])/A[p][q] t = sign(rho)/(abs(rho) + np.sqrt(rho*rho + 1)) c = 1./(np.sqrt(1 + t*t)) s = c*t ## Constructing the matrix D D[:] = A[:] D[p][q] = 0 D[q][p] = 0 D[p][p] = c*c*A[p][p] + s*s*A[q][q] - 2*c*s*A[p][q] D[q][q] = s*s*A[p][p] + c*c*A[q][q] + 2*c*s*A[p][q]
atlas = datasets.fetch_atlas_aal() # Loading atlas image stored in 'maps' atlas_filename = atlas['maps'] # Loading atlas data stored in 'labels' labels = atlas['labels'] labels = brain_roi_labels_ ############################################################################ # Build and display a correlation matrix correlation_measure = ConnectivityMeasure(kind='correlation') correlation_matrix = correlation_measure.fit_transform([time_series_])[0] plt.figure(figsize=(10, 10)) # Mask out the major diagonal numpy.fill_diagonal(correlation_matrix, 0) plt.imshow(correlation_matrix, interpolation="nearest", cmap="RdBu_r", vmax=0.8, vmin=-0.8) plt.colorbar() # And display the labels x_ticks = plt.xticks(range(len(labels)), labels, rotation=90) y_ticks = plt.yticks(range(len(labels)), labels) #coords = atlas.region_coords # We threshold to keep only the 20% of edges with the highest value # because the graph is very dense plotting.plot_connectome(correlation_matrix,
def createMPLengthThreeV2(pDict, pDir, verbosity): # Get the list of the primary path matrices pNames = list(pDict.keys()) pNames.sort() pNum = len(pNames) # Get the list of all metapaths created thus far mDict = readKeyFilePP(pDir) # Set the number from which to start naming matrices mNum = NChooseK(pNum, 2) + (2 * pNum) # Check if all expected 3-step paths were created # exit if they are mNumExpected = math.pow(pNum, 3) mp3Count = 0 for mpName in list(mDict.keys()): if mpName.count('-') == 2: mp3Count += 1 # end loop if mp3Count == mNumExpected: if verbosity: print("All 3-length paths already computed ...") return elif mp3Count >= mNumExpected: print("ERROR: Uh-oh, more 3-length paths than expected already exist!") sys.exit() # end if # Get the matrix dimensions from genes.txt file sizeOf = 0 pDirAlt = pDir if pDirAlt.endswith("_MetaPaths/"): pDirAlt = pDirAlt[:-11] + '/' with open(pDirAlt + 'genes.txt') as fin: for line in fin: sizeOf += 1 # end with # Build list of 2-step path names mp2Names = list() for mpName in list(mDict.keys()): if mpName.count('-') == 1: mp2Names.append(mpName) # end loop mp2Names.sort() # Multiply each matrix pair pNameSet = set() for p1 in pNames: # Load the first matrix mFileA = str(pDict[p1]).zfill(FNAME_ZPAD) + MATRIX_EXTENSION matrixA = getPathMatrixV2(mFileA, pDir, sizeOf) sizeGB = matrixA.nbytes / 1.0e9 if verbosity > 1: print("-- loaded {}, size: {:.3f} GBytes".format(p1, sizeGB)) for p2 in mp2Names: # The name of this path name = p1 + '-' + p2 # The name of the reversed path p2v = p2.split('-') nameRev = p2v[1] + '-' + p2v[0] + '-' + p1 # Optionally skipping consecutive edges if not KEEP_DOUBLE: if p1 == p2: continue # end if if name in pNameSet: continue # end if # Create new matrix if file doesn't already exist newMFName = str(mNum).zfill(FNAME_ZPAD) + MATRIX_EXTENSION if not os.path.isfile(pDir + newMFName): # Save a placeholder (allows multiple threads, to skip this one) fakeMx = np.ones((2, 2)) saveMatrixNumpyV2(fakeMx, newMFName, pDir, name, MX_SAVE_AS_INT) # Load the second matrix mFileB = str(mDict[p2][0]).zfill(FNAME_ZPAD) + MATRIX_EXTENSION matrixB = getPathMatrixV2(mFileB, pDir, sizeOf) # eliminate loops by removing self-connections np.fill_diagonal(matrixA, 0) np.fill_diagonal(matrixB, 0) # NOTE: Final saved matrix will contain self-connections # Multiply the two matrices if verbosity > 1: print("-- -- creating matrix {}".format(name)) newM = np.dot(matrixA, matrixB) if verbosity > 1: print("-- -- -- saving matrix {}".format(newMFName)) saveMatrixNumpyV2(newM, newMFName, pDir, name, MX_SAVE_AS_INT) # end if # end if # Add the matrix name & number to mDict if name == nameRev: # (ie: typeA-typeA) # Then add just this matrix to the list mDict[name] = [mNum, False] pNameSet.add(name) else: # Add this path & note the reverse path mDict[name] = [mNum, False] pNameSet.add(name) # Reverse path uses transpose mDict[nameRev] = [mNum, True] pNameSet.add(nameRev) # end if mNum += 1 # end loop saveKeyFile(mDict, pDir) return
def compute_distance_matrix(X): dists = np.sum(np.square(X), axis = 1) \ + np.transpose([np.sum(np.square(X), axis = 1)]) -2*np.dot(X, X.T) dists = np.sqrt(abs(dists)) np.fill_diagonal(dists, 0) return dists
def __init__(self, **kwargs): params = { 'max_steps': 50, 'dim': 2, 'num': 1, 'radius': [1.0], 'mass': [1.0], 'pp_collision_law': 'pp_specular', 'gamma': 'uniform' } params.update(kwargs) if (params['gamma'] == 'uniform'): params['gamma'] = np.sqrt(2 / (2 + params['dim'])) elif (params['gamma'] == 'shell'): params['gamma'] = np.sqrt(2 / params['dim']) elif (params['gamma'] == 'point'): params['gamma'] = 0 # Each parameter list must be num_particles long. If not, this will extend by filling with the last entry constants = ['radius', 'mass', 'gamma'] for const in constants: c = listify(params[const]) #listify defined at bottom of this file for p in range(len(c), params['num']): c.append(c[-1]) params[const] = np.asarray(c).astype(float) for key, val in params.items(): if isinstance(val, list): val = np.asarray(val) # converts lists to arrays setattr(self, key, val) self.mom_inert = self.mass * (self.gamma * self.radius)**2 self.get_mesh() self.pp_gap_min = cross_subtract(self.radius, -self.radius) np.fill_diagonal(self.pp_gap_min, -1) # no gap between a particle and itself self.wp_dt = np.zeros([len(wall), self.num], dtype='float') self.wp_mask = self.wp_dt.copy().astype(bool) if self.pp_collision_law == 'pp_ignore': self.pp_dt = np.array([np.inf]) else: self.pp_dt = np.zeros([self.num, self.num], dtype='float') self.pp_mask = self.pp_dt.copy().astype(bool) self.t = 0.0 self.cell_offset = np.zeros( [self.num, self.dim], dtype=int) # tracks which cell the particle is in self.col = {} self.t_hist = [] self.col_hist = [] self.pos_hist = [] self.vel_hist = [] self.spin_hist = [] # Color particles (helpful for the future when we have many particles) cm = plt.cm.gist_rainbow idx = np.linspace(0, cm.N - 1, self.num).round().astype(int) self.clr = [cm(i) for i in idx]