def __init__(self, value): self.value = value self.value_ = np.linalg.cholesky(value)[np.tril_indices(value.shape[0])] self.shape = value.shape self.size = value.size self.free = np.resize(True, self.value_.shape) self.to_external = lambda val: np.linalg.cholesky(val)[np.tril_indices(self.shape[0])]
def actor_critic_update(X0, A, R, X1, gamma=0.99, learnrate=0.0001): global WP1, WP2, WV_A, WV_b, WV global grad_WP1, grad_WP2, grad_N global grad_WP1s, grad_WP2s global noise_cov S0 = XtoS(X0) S1 = XtoS(X1) SS0 = np.outer(S0, S0)[np.tril_indices(7)] SS1 = np.outer(S1, S1)[np.tril_indices(7)] deltaV = SS0 - gamma * SS1 WV_A += np.outer(deltaV, deltaV) WV_b += R * deltaV WV = np.linalg.solve(WV_A, WV_b) advantage = R + gamma * np.dot(SS1, WV) - np.dot(SS0, WV) g1 = -0.5*np.outer(A, A) * advantage grad_WP1 += g1 grad_WP1s += g1*g1 g2 = np.outer(A, SS0) * advantage grad_WP2 += g2 grad_WP2s += g2*g2 grad_N += 1 # if X0[4] > 4: # print 'V(s0)', np.dot(SS0, WV), 'V(s1)', np.dot(SS1, WV), 'R', R # print 'A', A, 'adv', advantage # , 'g_wp1\n', grad_WP1, 'g_wp2\n', grad_WP2 noise_cov = np.linalg.inv(np.linalg.cholesky( WP1 + 1e-2 + np.eye(2))).T
def test_frozen(self): # Test that the frozen and non-frozen inverse Wishart gives the same # answers # Construct an arbitrary positive definite scale matrix dim = 4 scale = np.diag(np.arange(dim)+1) scale[np.tril_indices(dim, k=-1)] = np.arange(dim*(dim-1)/2) scale = np.dot(scale.T, scale) # Construct a collection of positive definite matrices to test the PDF X = [] for i in range(5): x = np.diag(np.arange(dim)+(i+1)**2) x[np.tril_indices(dim, k=-1)] = np.arange(dim*(dim-1)/2) x = np.dot(x.T, x) X.append(x) X = np.array(X).T # Construct a 1D and 2D set of parameters parameters = [ (10, 1, np.linspace(0.1, 10, 5)), # 1D case (10, scale, X) ] for (df, scale, x) in parameters: iw = invwishart(df, scale) assert_equal(iw.var(), invwishart.var(df, scale)) assert_equal(iw.mean(), invwishart.mean(df, scale)) assert_equal(iw.mode(), invwishart.mode(df, scale)) assert_allclose(iw.pdf(x), invwishart.pdf(x, df, scale))
def amplitudes_to_cisdvec(c0, c1, c2): nocc, nvir = c1.shape ooidx = numpy.tril_indices(nocc, -1) vvidx = numpy.tril_indices(nvir, -1) c2tril = lib.take_2d(c2.reshape(nocc**2,nvir**2), ooidx[0]*nocc+ooidx[1], vvidx[0]*nvir+vvidx[1]) return numpy.hstack((c0, c1.ravel(), c2tril.ravel()))
def find_stationary_var(amat=None, bmat=None, cmat=None): """Find fixed point of H = CC' + AHA' + BHB' given A, B, C. Parameters ---------- amat, bmat, cmat : (nstocks, nstocks) arrays Parameter matrices Returns ------- (nstocks, nstocks) array Unconditional variance matrix """ nstocks = amat.shape[0] kwargs = {'amat': amat, 'bmat': bmat, 'ccmat': cmat.dot(cmat.T)} fun = partial(ParamGeneric.fixed_point, **kwargs) try: with np.errstate(divide='ignore', invalid='ignore'): hvar = np.eye(nstocks) sol = sco.fixed_point(fun, hvar[np.tril_indices(nstocks)]) hvar[np.tril_indices(nstocks)] = sol hvar[np.triu_indices(nstocks, 1)] \ = hvar.T[np.triu_indices(nstocks, 1)] return hvar except RuntimeError: # warnings.warn('Could not find stationary varaince!') return None
def test_tril_indices(): # indices without and with offset il1 = tril_indices(4) il2 = tril_indices(4, 2) a = np.array([[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12], [13, 14, 15, 16]]) # indexing: yield (assert_array_equal, a[il1], array([ 1, 5, 6, 9, 10, 11, 13, 14, 15, 16]) ) # And for assigning values: a[il1] = -1 yield (assert_array_equal, a, array([[-1, 2, 3, 4], [-1, -1, 7, 8], [-1, -1, -1, 12], [-1, -1, -1, -1]]) ) # These cover almost the whole array (two diagonals right of the main one): a[il2] = -10 yield (assert_array_equal, a, array([[-10, -10, -10, 4], [-10, -10, -10, -10], [-10, -10, -10, -10], [-10, -10, -10, -10]]) )
def test_equivalence(self): """ The Equivalence covariance structure can represent an exchangeable covariance structure. Here we check that the results are identical using the two approaches. """ np.random.seed(3424) endog = np.random.normal(size=20) exog = np.random.normal(size=(20, 2)) exog[:, 0] = 1 groups = np.kron(np.arange(5), np.ones(4)) groups[12:] = 3 # Create unequal size groups # Set up an Equivalence covariance structure to mimic an # Exchangeable covariance structure. pairs = {} start = [0, 4, 8, 12] for k in range(4): pairs[k] = {} # Diagonal values (variance parameters) if k < 3: pairs[k][0] = (start[k] + np.r_[0, 1, 2, 3], start[k] + np.r_[0, 1, 2, 3]) else: pairs[k][0] = (start[k] + np.r_[0, 1, 2, 3, 4, 5, 6, 7], start[k] + np.r_[0, 1, 2, 3, 4, 5, 6, 7]) # Off-diagonal pairs (covariance parameters) if k < 3: a, b = np.tril_indices(4, -1) pairs[k][1] = (start[k] + a, start[k] + b) else: a, b = np.tril_indices(8, -1) pairs[k][1] = (start[k] + a, start[k] + b) ex = sm.cov_struct.Exchangeable() model1 = sm.GEE(endog, exog, groups, cov_struct=ex) result1 = model1.fit() for return_cov in False, True: ec = sm.cov_struct.Equivalence(pairs, return_cov=return_cov) model2 = sm.GEE(endog, exog, groups, cov_struct=ec) result2 = model2.fit() # Use large atol/rtol for the correlation case since there # are some small differences in the results due to degree # of freedom differences. if return_cov == True: atol, rtol = 1e-6, 1e-6 else: atol, rtol = 1e-3, 1e-3 assert_allclose(result1.params, result2.params, atol=atol, rtol=rtol) assert_allclose(result1.bse, result2.bse, atol=atol, rtol=rtol) assert_allclose(result1.scale, result2.scale, atol=atol, rtol=rtol)
def full_to_unique(y_full, feedmap, feedmask=None): if feedmask is None: feedmask = np.ones(feedmap.shape, dtype=np.bool) y_full[np.tril_indices(feedmap.shape[0])] = y_full[np.tril_indices(feedmap.shape[0])].conj() y_unique = y_full[np.where(feedmask)][np.unique(feedmap[np.where(feedmask)], return_index=True)[1]] return y_unique
def tril_index_matrix(self): n = self.global_size num_tril_entries = self.num_tril_entries tril_index_matrix = np.zeros([n, n], dtype=int) tril_index_matrix[np.tril_indices(n)] = np.arange(num_tril_entries) tril_index_matrix[ np.tril_indices(n)[::-1] ] = np.arange(num_tril_entries) return tril_index_matrix
def unique_to_full(y_unique, feedmap, feedmask=None): y_full = y_unique[feedmap] y_full[np.tril_indices(feedmap.shape[0])] = y_full[np.tril_indices(feedmap.shape[0])].conj() if feedmask is not None: y_full[np.where(np.logical_not(feedmask))] = 0.0 return y_full
def scrape_args(self, records, scale=1, guide_tree=None, niters=10, keep_topology=False): # local lists distances = [] variances = [] headers = [] for rec in records: distances.append(rec.parameters.partitions.distances) variances.append(rec.parameters.partitions.variances) headers.append(rec.get_names()) num_matrices = len(records) label_set = reduce(lambda x, y: x.union(y), (set(l) for l in headers)) labels_len = len(label_set) # labels string can be built straight away labels_string = '{0}\n{1}\n'.format(labels_len, ' '.join(label_set)) # distvar and genome_map need to be built up distvar_list = [str(num_matrices)] genome_map_list = ['{0} {1}'.format(num_matrices, labels_len)] # build up lists to turn into strings for i in range(num_matrices): labels = headers[i] dim = len(labels) dmatrix = np.array(distances[i]) vmatrix = np.array(variances[i]) matrix = np.zeros(dmatrix.shape) matrix[np.triu_indices(len(dmatrix), 1)] = dmatrix[np.triu_indices(len(dmatrix), 1)] matrix[np.tril_indices(len(vmatrix), -1)] = vmatrix[np.tril_indices(len(vmatrix), -1)] if scale: matrix[np.triu_indices(dim, 1)] *= scale matrix[np.tril_indices(dim, -1)] *= scale * scale if isinstance(matrix, np.ndarray): matrix_string = '\n'.join([' '.join(str(x) for x in row) for row in matrix]) + '\n' else: matrix_string = matrix distvar_list.append('{0} {0} {1}\n{2}'.format(dim, i + 1, matrix_string)) genome_map_entry = ' '.join((str(labels.index(lab) + 1) if lab in labels else '-1') for lab in label_set) genome_map_list.append(genome_map_entry) distvar_string = '\n'.join(distvar_list) genome_map_string = '\n'.join(genome_map_list) if guide_tree is None: guide_tree = Tree.new_iterative_rtree(labels_len, names=label_set, rooted=True) tree_string = guide_tree.scale(scale).newick.replace('\'', '') return distvar_string, genome_map_string, labels_string, tree_string, niters, keep_topology
def test_cl_ldl(AA): """ Test the CL implentation of LDL algorithm. This tests a series (cl_size) of matrices against the Python implementation. """ # Convert to single float AA = AA.astype(DTYPE) # First calculate the Python based values for each matrix in AA py_ldl_D = np.empty((AA.shape[0], AA.shape[2]), dtype=AA.dtype) py_ldl_L = np.empty(AA.shape, dtype=AA.dtype) for i in range(AA.shape[2]): py_ldl_D[..., i], py_ldl_L[..., i] = ldl(AA[..., i]) # Setup CL context import pyopencl as cl from pycllp.ldl import cl_krnl_ldl ctx = cl.create_some_context() queue = cl.CommandQueue(ctx) # Result arrays m, n, cl_size = AA.shape L = np.empty(cl_size*m*(m+1)/2, dtype=DTYPE) D = np.empty(cl_size*m, dtype=DTYPE) mf = cl.mem_flags A_g = cl.Buffer(ctx, mf.READ_ONLY | mf.COPY_HOST_PTR, hostbuf=AA) # Create and compile kernel prg = cl_krnl_ldl(ctx) L_g = cl.Buffer(ctx, mf.READ_WRITE, L.nbytes) D_g = cl.Buffer(ctx, mf.READ_WRITE, D.nbytes) # Test normal LDL (unmodified) prg.ldl(queue, (cl_size,), None, np.int32(m), np.int32(n), A_g, L_g, D_g) cl.enqueue_copy(queue, L, L_g) cl.enqueue_copy(queue, D, D_g) # Compare each matrix decomposition with the python equivalent. for i in range(cl_size): np.testing.assert_allclose(py_ldl_D[..., i], D[i::cl_size], rtol=1e-6, atol=1e-7) np.testing.assert_allclose(py_ldl_L[..., i][np.tril_indices(m)], L[i::cl_size], rtol=1e-6, atol=1e-7) # Now test the modified algorithm ... beta = np.sqrt(np.amax(AA)) prg.modified_ldl(queue, (cl_size,), None, np.int32(m), np.int32(n), A_g, L_g, D_g, DTYPE(beta), DTYPE(1e-6)) cl.enqueue_copy(queue, L, L_g) cl.enqueue_copy(queue, D, D_g) # Compare each matrix decomposition with the python equivalent. for i in range(cl_size): np.testing.assert_allclose(py_ldl_D[..., i], D[i::cl_size], rtol=1e-6, atol=1e-7) np.testing.assert_allclose(py_ldl_L[..., i][np.tril_indices(m)], L[i::cl_size], rtol=1e-6, atol=1e-7)
def impute_missing_bins(hic_matrix, regions=None, per_chromosome=True, stat=np.ma.mean): """ Impute missing contacts in a Hi-C matrix. For inter-chromosomal data uses the mean of all inter-chromosomal contacts, for intra-chromosomal data uses the mean of intra-chromosomal counts at the corresponding diagonal. :param hic_matrix: A square numpy array :param regions: A list of :class:`~GenomicRegion`s - if omitted, will create a dummy list :param per_chromosome: Do imputation on a per-chromosome basis (recommended) :param stat: The aggregation statistic to be used for imputation, defaults to the mean. """ if regions is None: for i in range(hic_matrix.shape[0]): regions.append(GenomicRegion(chromosome='', start=i, end=i)) chr_bins = dict() for i, region in enumerate(regions): if region.chromosome not in chr_bins: chr_bins[region.chromosome] = [i, i] else: chr_bins[region.chromosome][1] = i n = len(regions) if not hasattr(hic_matrix, "mask"): hic_matrix = masked_matrix(hic_matrix) imputed = hic_matrix.copy() if per_chromosome: for c_start, c_end in chr_bins.itervalues(): # Correcting intrachromoc_startmal contacts by mean contact count at each diagonal for i in range(c_end - c_start): ind = kth_diag_indices(c_end - c_start, -i) diag = imputed[c_start:c_end, c_start:c_end][ind] diag[diag.mask] = stat(diag) imputed[c_start:c_end, c_start:c_end][ind] = diag # Correcting interchromoc_startmal contacts by mean of all contact counts between # each set of chromoc_startmes for other_start, other_end in chr_bins.itervalues(): # Only correct upper triangle if other_start <= c_start: continue inter = imputed[c_start:c_end, other_start:other_end] inter[inter.mask] = stat(inter) imputed[c_start:c_end, other_start:other_end] = inter else: for i in range(n): diag = imputed[kth_diag_indices(n, -i)] diag[diag.mask] = stat(diag) imputed[kth_diag_indices(n, -i)] = diag # Copying upper triangle to lower triangle imputed[np.tril_indices(n)] = imputed.T[np.tril_indices(n)] return imputed
def structure_function(self, bins): """ compute the structure function of the light curve at given time lags """ dt = np.subtract.outer(self.t,self.t)[np.tril_indices(self.t.shape[0], k=-1)] dm = np.subtract.outer(self.y,self.y)[np.tril_indices(self.y.shape[0], k=-1)] sqrsum, bins, _ = binned_statistic(dt, dm**2, bins=bins, statistic='sum') n, _, _ = binned_statistic(dt, dm**2, bins=bins, statistic='count') SF = np.sqrt(sqrsum/n) lags = 0.5*(bins[1:] + bins[:-1]) return lags, SF
def get_grad_tril(mo_coeff_kpts, mo_occ_kpts, fock): if is_khf: grad_kpts = [] for k, mo in enumerate(mo_coeff_kpts): f_mo = reduce(numpy.dot, (mo.T.conj(), fock[k], mo)) nmo = f_mo.shape[0] grad_kpts.append(f_mo[numpy.tril_indices(nmo, -1)]) return numpy.hstack(grad_kpts) else: f_mo = reduce(numpy.dot, (mo_coeff_kpts.T.conj(), fock, mo_coeff_kpts)) nmo = f_mo.shape[0] return f_mo[numpy.tril_indices(nmo, -1)]
def get_y_matrix(self, projectile, daughter): """Returns a ``DIM x DIM`` yield matrix. Args: projectile (int): PDG ID of projectile particle daughter (int): PDG ID of final state daughter/secondary particle Returns: numpy.array: yield matrix Note: In the current version, the matrices have to be multiplied by the bin widths. In later versions they will be stored with the multiplication carried out. """ # TODO: modify yields to include the bin size if not self.band: return self.yields[(projectile, daughter)].dot(self.weights) else: m = self.yields[(projectile, daughter)].dot(self.weights) # set all elements except those inside selected xf band to 0 m[np.tril_indices(self.dim, -2 - self.band[1])] = 0 if self.band[0] < 0: m[np.triu_indices(self.dim, -self.band[0])] = 0 return m
def threePointsToStandard(e, p, q, r): """Return a projective transformation that maps three points on a conic to the conic xy + yz + xz = 0. Keyword arguments: e -- a projective conic p -- the first point on e q -- the second point on e r -- the third point on e """ coeffs = e p, q, r = np.matrix(p), np.matrix(q), np.matrix(r) # Determine a matrix A associated with a projective transformation that # maps P, Q, and R onto [1, 0, 0], [0, 1, 0], and [0, 0, 1], respectively. A = np.linalg.inv(np.vstack([p, q, r])) # Determine the equation bx'y' + fx'z' + gy'z' = 0 of t(E), for some real # numbers b, f, and g. M = sum([coeff * u.T * v for coeff, (u, v) in zip(coeffs, combinations_with_replacement((p, q, r), 2))]) # Get B from M by adding like terms to find b, f, and g and then # constructing a diagonal matrix from the flat [1/g, 1/f, 1/b]. B = np.diagflat([1 / (u + v) for u, v in reversed(zip(np.array(M)[np.triu_indices(3, 1)], np.array(M)[np.tril_indices(3, -1)]))]) return B * A
def update_nogrid(self, params): endog = self.model.endog_li cached_means = self.model.cached_means varfunc = self.model.family.variance dep_params = np.zeros(self.max_lag + 1) dn = np.zeros(self.max_lag + 1) for i in range(self.model.num_group): expval, _ = cached_means[i] stdev = np.sqrt(varfunc(expval)) resid = (endog[i] - expval) / stdev j1, j2 = np.tril_indices(len(expval)) dx = np.abs(self.time[i][j1] - self.time[i][j2]) ii = np.flatnonzero(dx <= self.max_lag) j1 = j1[ii] j2 = j2[ii] dx = dx[ii] vs = np.bincount(dx, weights=resid[ j1] * resid[j2], minlength=self.max_lag + 1) vd = np.bincount(dx, minlength=self.max_lag + 1) ii = np.flatnonzero(vd > 0) dn[ii] += 1 if len(ii) > 0: dep_params[ii] += vs[ii] / vd[ii] dep_params /= dn self.dep_params = dep_params[1:] / dep_params[0]
def multivar_norm_cdf(upper, cov_matrix): """CDF of multivariate Gaussian centered at 0 with covariance matrix cov_matrix. CDF is taken from -inf to u.""" if upper.size == 1: return scipy.stats.norm.cdf(upper[0], 0, numpy.sqrt(cov_matrix[0, 0])) # Standardize the upper bound u using the standard deviation std = numpy.sqrt(numpy.diag(cov_matrix)) std_upper = upper / std # Convert covariance matrix into correlation matrix: http://en.wikipedia.org/wiki/Correlation_and_dependence#Correlation_matrices corr_matrix = cov_matrix / std / std.reshape(upper.size, 1) # standardize -> correlation matrix # Indices for traversing the strict lower triangular elements of corr_matrix in column major, as required by the fortran mvndst function. strict_lower_diag_indices = numpy.tril_indices(upper.size, -1) # Call into the scipy wrapper for the fortran method "mvndst" # Link: http://www.math.wsu.edu/faculty/genz/software/fort77/mvtdstpack.f out = scipy.stats.kde.mvn.mvndst( numpy.zeros(upper.size, dtype=int), # The lower bound of integration. We initialize with 0 because it is ignored (because of the third argument). std_upper, # The upper bound of integration numpy.zeros(upper.size, dtype=int), # For each dim, 0 means -inf for lower bound corr_matrix[strict_lower_diag_indices], # The vector of strict lower triangular correlation coefficients maxpts=self._mvndst_parameters.maxpts_per_dim * upper.size, # Maximum number of iterations for the mvndst function releps=self._mvndst_parameters.releps, # The error allowed relative to actual value abseps=self._mvndst_parameters.abseps, # The absolute error allowed ) return out[1] # Index 1 corresponds to the actual value. 0 has the error, and 2 is a flag denoting whether releps was reached
def vector_from_array(array): """Get triangle of output in vector from a correlation-type array. Parameters ---------- array : np.array Returns ------- vector (np.array) Notes ----- Old Matlab code indexes by column (aka.Fortran-style), so to get the indices of the top triangle, we have to do some reshaping. Otherwise, if the vector made up by rows is OK, then simply : triangle = np.triu_indices(array.size, k=1), out = array[triangle] """ triangle_lower = np.tril_indices(array.shape[0], k=-1) flatten_idx = np.arange(array.size).reshape(array.shape)[triangle_lower] triangle = np.unravel_index(flatten_idx, array.shape, order='F') # triangle = np.triu_indices(array.size, k=1) # out = array[triangle] return array[triangle]
def _getSignificantData(self, sig_lvl): '''Find which edges significant at passed level and set self properties. ''' rows,cols = self.data.shape #rows = cols mask = zeros((rows,cols)) mask[tril_indices(rows,0)] = 1 #preparing mask cvals = unique(self.data[triu_indices(rows,1)]) # cvals is sorted # calculate upper bound, i.e. what value in the distribution of values # has sig_lvl fraction of the data higher than or equal to it. this is # not guaranteed to be precise because of repeated values. for instance # assume the distribution of dissimilarity values is: # [.1, .2, .2, .2, .2, .3, .4, .5, .6, .6, .6, .6, .6, .6, .7] # and you want sig_lvl=.2, i.e. you get 20 percent of the linkages as # significant. this would result in choosing the score .6 since its the # third in the ordered list (of 15 elements, 3/15=.2). but, since there # is no a-priori way to tell which of the multiple .6 linkages are # significant, we select all of them, forcing our lower bound to # encompass 7/15ths of the data. the round call on the ub is to avoid # documented numpy weirdness where it will misassign >= calls for long # floats. ub = round(cvals[-round(sig_lvl*len(cvals))],7) mdata = ma(self.data, mask) self.actual_sig_lvl = \ (mdata >= ub).sum()/float(mdata.shape[0]*(mdata.shape[0]-1)/2) self.sig_edges = where(mdata >= ub, 1, 0).nonzero() self.otu1 = [self.otu_ids[i] for i in self.sig_edges[0]] self.otu2 = [self.otu_ids[i] for i in self.sig_edges[1]] self.sig_otus = list(set(self.otu1+self.otu2)) self.edges = zip(self.otu1, self.otu2) self.cvals = mdata[self.sig_edges[0], self.sig_edges[1]]
def scoring2B_behavior(): t_clusters = np.zeros((600,3)) t_clusters[0:200,0] = 1 t_clusters[200:400,1] = 1 t_clusters[400:,2] = 1 t_ccm = np.dot(t_clusters,t_clusters.T) n_uniq = len(np.triu_indices(t_ccm.shape[0],k=1)[0]) res = [] concentrations = [1000,100,50,25,10,5,3,1] for c in concentrations: for i in range(50): ccm = np.copy(t_ccm) ccm[np.triu_indices(t_ccm.shape[0],k=1)] -= np.random.beta(1,c,n_uniq) #ccm[np.tril_indices(t_ccm.shape[0],k=-1)] = ccm[np.triu_indices(t_ccm.shape[0],k=1)] ccm[np.tril_indices(t_ccm.shape[0],k=-1)] = 0 ccm = ccm + ccm.T np.fill_diagonal(ccm,1) ccm = np.abs(ccm) res.append([c,calculate2(ccm,t_ccm)]) res = [map(str,x) for x in res] res = ['\t'.join(x) for x in res] f = open('scoring2B_beta.tsv', 'w') f.write('\n'.join(res)) f.close()
def test_overlap(xSer): 'test the set overlap among items in a Series \ -return set of all pairwise overlap values' ns = len(xSer) if ns >1: # overlapMtrx = pd.DataFrame(data=np.zeros([ns,ns]), # index=xSer.index, # columns=xSer.index) overlapMtrx = np.zeros([ns,ns]) for i1, ix1 in enumerate(xSer.index): s1 = xSer[ix1] for i2, ix2 in enumerate(xSer.index): s2 = xSer[ix2] nO = len(s1.intersection(s2)) # overlapMtrx.ix[ix1,ix2] = nO # overlapMtrx.ix[ix2,ix1] = nO overlapMtrx[i1,i2] = nO overlapMtrx[i2,i1] = nO iUp = np.tril_indices(ns) overlapMtrx[iUp] = np.nan overlapArray = overlapMtrx.flatten() overlapArray = overlapArray[~np.isnan(overlapArray)] # return overlapArray else: # return pd.DataFrame() return np.zeros(0)
def calculate_forces(clf, numbers, coords, meta=None, h=1e-6): """ A function that uses finite differences to calculate forces of a molecule using the given clf. The default feature vector for this to use is the coulomb matrix. """ if meta is None: meta = [1] vectors = [] for i, coord in enumerate(coords): for j in xrange(3): for sign in [-1, 1]: new_coords = coords.copy() new_coords[i, j] += sign * h / 2 mat = get_coulomb_matrix(numbers, new_coords) mat[mat < 0] = 0 vectors.append(mat[numpy.tril_indices(mat.shape[0])].tolist() + meta) results = clf.predict(numpy.matrix(vectors)) forces = numpy.zeros(coords.shape) for i, coord in enumerate(coords): for j in xrange(3): forces[i, j] = (results[i * len(coord) * 2 + j * 2 + 1] - results[i * len(coord) * 2 + j * 2]) / h return forces
def calculate_surface(clf, numbers, coords, atom_idx, max_displacement=0.5, steps=25, meta=None): """ A function that uses plots the value of the clf as a function of `atom_ix` displacement. """ if meta is None: meta = [1] values = numpy.linspace(-max_displacement, max_displacement, steps) results = numpy.zeros((steps, steps)) for i, x in enumerate(values): for j, y in enumerate(values): new_coords = coords.copy() new_coords[atom_idx, 0] += x new_coords[atom_idx, 1] += y mat = get_coulomb_matrix(numbers, new_coords) mat[mat < 0] = 0 vector = mat[numpy.tril_indices(mat.shape[0])].tolist() + meta results[i, j] = clf.predict(numpy.matrix(vector))[0] extent = [-max_displacement, max_displacement, -max_displacement, max_displacement] get_matrix_plot(results, extent) print results.max(), results.min(), results.std() return results
def pack_tril(mat, axis=-1, out=None): '''flatten the lower triangular part of a matrix. Given mat, it returns mat[...,numpy.tril_indices(mat.shape[0])] Examples: >>> pack_tril(numpy.arange(9).reshape(3,3)) [0 3 4 6 7 8] ''' if mat.size == 0: return numpy.zeros(mat.shape+(0,), dtype=mat.dtype) if mat.ndim == 2: count, nd = 1, mat.shape[0] shape = nd*(nd+1)//2 else: count, nd = mat.shape[:2] shape = (count, nd*(nd+1)//2) if mat.ndim == 2 or axis == -1: mat = numpy.asarray(mat, order='C') out = numpy.ndarray(shape, mat.dtype, buffer=out) if mat.dtype == numpy.double: fn = _np_helper.NPdpack_tril_2d else: fn = _np_helper.NPzpack_tril_2d fn(ctypes.c_int(count), ctypes.c_int(nd), out.ctypes.data_as(ctypes.c_void_p), mat.ctypes.data_as(ctypes.c_void_p)) return out else: # pack the leading two dimension assert(axis == 0) out = mat[numpy.tril_indices(nd)] return out
def test_is_scaled_chisquared(self): # The 2-dimensional Wishart with an arbitrary scale matrix can be # transformed to a scaled chi-squared distribution. # For :math:`S \sim W_p(V,n)` and :math:`\lambda \in \mathbb{R}^p` we have # :math:`\lambda' S \lambda \sim \lambda' V \lambda \times \chi^2(n)` np.random.seed(482974) sn = 500 df = 10 dim = 4 # Construct an arbitrary positive definite matrix scale = np.diag(np.arange(4)+1) scale[np.tril_indices(4, k=-1)] = np.arange(6) scale = np.dot(scale.T, scale) # Use :math:`\lambda = [1, \dots, 1]'` lamda = np.ones((dim,1)) sigma_lamda = lamda.T.dot(scale).dot(lamda).squeeze() w = wishart(df, sigma_lamda) c = chi2(df, scale=sigma_lamda) # Statistics assert_allclose(w.var(), c.var()) assert_allclose(w.mean(), c.mean()) assert_allclose(w.entropy(), c.entropy()) # PDF X = np.linspace(0.1,10,num=10) assert_allclose(w.pdf(X), c.pdf(X)) # rvs rvs = w.rvs(size=sn) args = (df,0,sigma_lamda) alpha = 0.01 check_distribution_rvs('chi2', args, alpha, rvs)
def initialize(self, model): super(GlobalOddsRatio, self).initialize(model) if self.model.weights is not None: warnings.warn("weights not implemented for GlobalOddsRatio " "cov_struct, using unweighted covariance estimate", NotImplementedWarning) # Need to restrict to between-subject pairs cpp = [] for v in model.endog_li: # Number of subjects in this group m = int(len(v) / self._ncut) i1, i2 = np.tril_indices(m, -1) cpp1 = {} for k1 in range(self._ncut): for k2 in range(k1 + 1): jj = np.zeros((len(i1), 2), dtype=np.int64) jj[:, 0] = i1 * self._ncut + k1 jj[:, 1] = i2 * self._ncut + k2 cpp1[(k2, k1)] = jj cpp.append(cpp1) self.cpp = cpp # Initialize the dependence parameters self.crude_or = self.observed_crude_oddsratio() if self.model.update_dep: self.dep_params = self.crude_or
def bic(em_fit_result_dict, LL_all): ''' Compute the Bayesian Information Criterion score Split it, associating the parameters to the number of datapoint they really take care of. ''' # Number of parameters: # - mixt_target_tr: 1 # - mixt_random_tr: 1 # - mixt_nontarget_trk: 1 # - alpha: 1 # - beta: 1 # - gamma: 1 # First count the Loglikelihood bic_tot = -2. * np.nansum(LL_all[np.tril_indices(LL_all.shape[0])]) # Then count alpha, beta and gamma, for all datapoints appropriately K = 3 bic_tot += K * np.log(np.nansum(np.isfinite(LL_all))) # Now do the mixture proportions per condition for nitems_i, nitems in enumerate(em_fit_result_dict['T_space']): for trecall_i, trecall in enumerate(em_fit_result_dict['T_space']): if trecall <= nitems: K = 3 bic_tot += K * np.log(np.nansum(np.isfinite(LL_all[nitems_i, trecall_i]))) return bic_tot
def symmetrize(m, use_triangle='lower'): """Symmetrize a square NumPy array by reflecting one triangular section across the diagonal to the other. """ if use_triangle not in ('lower', 'upper'): raise ValueError if not len(m.shape) == 2: raise ValueError if not (m.shape[0] == m.shape[1]): raise ValueError dim = m.shape[0] lower_indices = numpy.tril_indices(dim, k=-1) upper_indices = numpy.triu_indices(dim, k=1) ms = m.copy() if use_triangle == 'lower': ms[upper_indices] = ms[lower_indices] if use_triangle == 'upper': ms[lower_indices] = ms[upper_indices] return ms
def overlap(cibra, ciket, nmo, nocc, s=None): '''Overlap between two CISD wavefunctions. Args: s : 2D array The overlap matrix of non-orthogonal one-particle basis ''' if s is None: return dot(cibra, ciket, nmo, nocc) DEBUG = True nvir = nmo - nocc nov = nocc * nvir bra0, bra1, bra2 = cisdvec_to_amplitudes(cibra, nmo, nocc) ket0, ket1, ket2 = cisdvec_to_amplitudes(ciket, nmo, nocc) # Sort the ket orbitals to make the orbitals in bra one-one mapt to orbitals # in ket. if ((not DEBUG) and abs(numpy.linalg.det(s[:nocc, :nocc]) - 1) < 1e-2 and abs(numpy.linalg.det(s[nocc:, nocc:]) - 1) < 1e-2): ket_orb_idx = numpy.where(abs(s) > 0.9)[1] s = s[:, ket_orb_idx] oidx = ket_orb_idx[:nocc] vidx = ket_orb_idx[nocc:] - nocc ket1 = ket1[oidx[:, None], vidx] ket2 = ket2[oidx[:, None, None, None], oidx[:, None, None], vidx[:, None], vidx] ooidx = numpy.tril_indices(nocc, -1) vvidx = numpy.tril_indices(nvir, -1) bra2aa = bra2 - bra2.transpose(1, 0, 2, 3) bra2aa = lib.take_2d(bra2aa.reshape(nocc**2, nvir**2), ooidx[0] * nocc + ooidx[1], vvidx[0] * nvir + vvidx[1]) ket2aa = ket2 - ket2.transpose(1, 0, 2, 3) ket2aa = lib.take_2d(ket2aa.reshape(nocc**2, nvir**2), ooidx[0] * nocc + ooidx[1], vvidx[0] * nvir + vvidx[1]) occlist0 = numpy.arange(nocc).reshape(1, nocc) occlists = numpy.repeat(occlist0, 1 + nov + bra2aa.size, axis=0) occlist0 = occlists[:1] occlist1 = occlists[1:1 + nov] occlist2 = occlists[1 + nov:] ia = 0 for i in range(nocc): for a in range(nocc, nmo): occlist1[ia, i] = a ia += 1 ia = 0 for i in range(nocc): for j in range(i): for a in range(nocc, nmo): for b in range(nocc, a): occlist2[ia, i] = a occlist2[ia, j] = b ia += 1 na = len(occlists) if DEBUG: trans = numpy.empty((na, na)) for i, idx in enumerate(occlists): s_sub = s[idx].T.copy() minors = s_sub[occlists] trans[i, :] = numpy.linalg.det(minors) # Mimic the transformation einsum('ab,ap->pb', FCI, trans). # The wavefunction FCI has the [excitation_alpha,excitation_beta] # representation. The zero blocks like FCI[S_alpha,D_beta], # FCI[D_alpha,D_beta], are explicitly excluded. bra_mat = numpy.zeros((na, na)) bra_mat[0, 0] = bra0 bra_mat[0, 1:1 + nov] = bra_mat[1:1 + nov, 0] = bra1.ravel() bra_mat[0, 1 + nov:] = bra_mat[1 + nov:, 0] = bra2aa.ravel() bra_mat[1:1 + nov, 1:1 + nov] = bra2.transpose(0, 2, 1, 3).reshape(nov, nov) ket_mat = numpy.zeros((na, na)) ket_mat[0, 0] = ket0 ket_mat[0, 1:1 + nov] = ket_mat[1:1 + nov, 0] = ket1.ravel() ket_mat[0, 1 + nov:] = ket_mat[1 + nov:, 0] = ket2aa.ravel() ket_mat[1:1 + nov, 1:1 + nov] = ket2.transpose(0, 2, 1, 3).reshape(nov, nov) ovlp = lib.einsum('ab,ap,bq,pq->', bra_mat, trans, trans, ket_mat) else: nov1 = 1 + nov noovv = bra2aa.size bra_SS = numpy.zeros((nov1, nov1)) bra_SS[0, 0] = bra0 bra_SS[0, 1:] = bra_SS[1:, 0] = bra1.ravel() bra_SS[1:, 1:] = bra2.transpose(0, 2, 1, 3).reshape(nov, nov) ket_SS = numpy.zeros((nov1, nov1)) ket_SS[0, 0] = ket0 ket_SS[0, 1:] = ket_SS[1:, 0] = ket1.ravel() ket_SS[1:, 1:] = ket2.transpose(0, 2, 1, 3).reshape(nov, nov) trans_SS = numpy.empty((nov1, nov1)) trans_SD = numpy.empty((nov1, noovv)) trans_DS = numpy.empty((noovv, nov1)) occlist01 = occlists[:nov1] for i, idx in enumerate(occlist01): s_sub = s[idx].T.copy() minors = s_sub[occlist01] trans_SS[i, :] = numpy.linalg.det(minors) minors = s_sub[occlist2] trans_SD[i, :] = numpy.linalg.det(minors) s_sub = s[:, idx].copy() minors = s_sub[occlist2] trans_DS[:, i] = numpy.linalg.det(minors) ovlp = lib.einsum('ab,ap,bq,pq->', bra_SS, trans_SS, trans_SS, ket_SS) ovlp += lib.einsum('ab,a ,bq, q->', bra_SS, trans_SS[:, 0], trans_SD, ket2aa.ravel()) ovlp += lib.einsum('ab,ap,b ,p ->', bra_SS, trans_SD, trans_SS[:, 0], ket2aa.ravel()) ovlp += lib.einsum(' b, p,bq,pq->', bra2aa.ravel(), trans_SS[0, :], trans_DS, ket_SS) ovlp += lib.einsum(' b, p,b ,p ->', bra2aa.ravel(), trans_SD[0, :], trans_DS[:, 0], ket2aa.ravel()) ovlp += lib.einsum('a ,ap, q,pq->', bra2aa.ravel(), trans_DS, trans_SS[0, :], ket_SS) ovlp += lib.einsum('a ,a , q, q->', bra2aa.ravel(), trans_DS[:, 0], trans_SD[0, :], ket2aa.ravel()) # FIXME: whether to approximate the overlap between double excitation coefficients if numpy.linalg.norm(bra2aa) * numpy.linalg.norm(ket2aa) < 1e-4: # Skip the overlap if coefficients of double excitation are small enough pass if (abs(numpy.linalg.det(s[:nocc, :nocc]) - 1) < 1e-2 and abs(numpy.linalg.det(s[nocc:, nocc:]) - 1) < 1e-2): # If the overlap matrix close to identity enough, use the <D|D'> overlap # for orthogonal single-particle basis to approximate the overlap # for non-orthogonal basis. ovlp += numpy.dot(bra2aa.ravel(), ket2aa.ravel()) * trans_SS[0, 0] * 2 else: from multiprocessing import sharedctypes, Process buf_ctypes = sharedctypes.RawArray('d', noovv) trans_ket = numpy.ndarray(noovv, buffer=buf_ctypes) def trans_dot_ket(i0, i1): for i in range(i0, i1): s_sub = s[occlist2[i]].T.copy() minors = s_sub[occlist2] trans_ket[i] = numpy.linalg.det(minors).dot(ket2aa.ravel()) nproc = lib.num_threads() if nproc > 1: seg = (noovv + nproc - 1) // nproc ps = [] for i0, i1 in lib.prange(0, noovv, seg): p = Process(target=trans_dot_ket, args=(i0, i1)) ps.append(p) p.start() [p.join() for p in ps] else: trans_dot_ket(0, noovv) ovlp += numpy.dot(bra2aa.ravel(), trans_ket) * trans_SS[0, 0] * 2 return ovlp
def s2kl_s1(symmetry, eri, norb): idx = numpy.tril_indices(norb) eri1 = numpy.empty((norb, norb, norb, norb)) eri1[:, :, idx[0], idx[1]] = eri.reshape(norb, norb, -1) eri1[:, :, idx[1], idx[0]] = eri.reshape(norb, norb, -1) return eri1
def main(): usage = '''Usage: python %s [-h] [-s samples] [-k clusters] [-d dims] [-p band positions] fileName\n spatial and spectral dimensions are lists, e.g., -d [0,0,400,400]''' % sys.argv[ 0] options, args = getopt.getopt(sys.argv[1:], 'hs:k:d:p:') dims = None pos = None K = 8 m = 1000 for option, value in options: if option == '-h': print usage return elif option == '-d': dims = eval(value) elif option == '-p': pos = eval(value) elif option == '-k': K = eval(value) elif option == '-s': m = eval(value) gdal.AllRegister() infile = args[0] inDataset = gdal.Open(infile, GA_ReadOnly) cols = inDataset.RasterXSize rows = inDataset.RasterYSize bands = inDataset.RasterCount if dims: x0, y0, cols, rows = dims else: x0 = 0 y0 = 0 if pos is not None: bands = len(pos) else: pos = range(1, bands + 1) path = os.path.dirname(infile) basename = os.path.basename(infile) root, ext = os.path.splitext(basename) outfile = path + '/' + root + '_hcl' + ext print '------- Hierarchical clustering ---------' print time.asctime() print 'Input: %s' % infile print 'Clusters: %i' % K print 'Samples: %i' % m start = time.time() GG = np.zeros((cols * rows, bands)) k = 0 for b in pos: band = inDataset.GetRasterBand(b) band = band.ReadAsArray(x0, y0, cols, rows).astype(float) GG[:, k] = np.ravel(band) k += 1 # training data idx = np.random.randint(0, rows * cols, size=m) G = GG[idx, :] ones = np.mat(np.ones(m, dtype=np.int)) mults = np.ones(m, dtype=np.int) Ls = np.array(range(m)) # initial cost array G2 = np.mat(np.sum(G**2, 1)) Delta = G2.T * ones Delta = Delta + Delta.T Delta = Delta - 2 * np.mat(G) * np.mat(G).T idx = np.tril_indices(m) Delta[idx] = 10e10 Delta = Delta.A # begin iteration cost = 0.0 costarray = [] c = m while c > K: bm = best_merge(Delta) j = bm[0] i = bm[1] # j > i costarray.append(cost + Delta[i, j]) # re-label idx = np.where(Ls == j)[0] Ls[idx] = i idx = np.where(Ls > j)[0] Ls[idx] -= 1 # pre-merge multiplicities ni = mults[i] nj = mults[j] # update merge-cost array, k = i+1 ... c-1 if c - i - 1 == 0: k = [i + 1] else: k = i + 1 + range(c - i - 1) nk = mults[k] Dkj = np.minimum(Delta[k, j].ravel(), Delta[j, k].ravel()) idx = np.where(k == j)[0] Dkj[idx] = 0 Delta[i, k] = ((ni + nk) * Delta[i, k] + (nj + nk) * Dkj - nk * Delta[i, j]) / (ni + nj + nk) # update merge-cost array, k = 0 ... i-1 if i == 0: k = [0] else: k = range(i - 1) nk = mults[k] Dkj = np.minimum(Delta[k, j].ravel(), Delta[j, k].ravel()) idx = np.where(k == j)[0] Dkj[idx] = 0 Delta[k, i] = ((ni + nk) * Delta[k, i] + (nj + nk) * Dkj - nk * Delta[i, j]) / (ni + nj + nk) # update multiplicities mults[i] = mults[i] + mults[j] # delete the upper cluster idx = np.ones(c) idx[j] = 0 idx = np.where(idx == 1)[0] mults = mults[idx] Delta = Delta[:, idx] Delta = Delta[idx, :] c -= 1 print 'classifying...' labs = [] for L in Ls: lab = np.zeros(K) lab[L] = 1.0 labs.append(lab) labs = np.array(labs) classifier = sc.Maxlike(G, labs) if classifier.train(): driver = gdal.GetDriverByName('GTiff') outDataset = driver.Create(outfile, cols, rows, 1, GDT_Byte) projection = inDataset.GetProjection() geotransform = inDataset.GetGeoTransform() if geotransform is not None: gt = list(geotransform) gt[0] = gt[0] + x0 * gt[1] gt[3] = gt[3] + y0 * gt[5] outDataset.SetGeoTransform(tuple(gt)) if projection is not None: outDataset.SetProjection(projection) cls, _ = classifier.classify(GG) outBand = outDataset.GetRasterBand(1) outBand.WriteArray(np.reshape(cls, (rows, cols)), 0, 0) outBand.FlushCache() outDataset = None inDataset = None ymax = np.max(costarray) plt.loglog(range(K, m), list(reversed(costarray))) p = plt.gca() p.set_title('Merge cost') p.set_xlabel('Custers') p.set_ylim((1, ymax)) plt.show() print 'result written to: ' + outfile print 'elapsed time: ' + str(time.time() - start) else: print 'classification failed'
def sim_block_study(X, afreq, ldscore, n_study, n_blocks, n_causal_per_block, block_p, pve, effect_distribution, min_r2, max_r2, min_ldscore, max_ldscore): """ each block has a causal snps, each study assigned to a main block tissues within a block share the causal snp tissues out of block have the causal snp with probability block_p min_r2, max_r2: pairwise r2 values allowed for causal snps min_lscore, max_ldscore: ldscore range allowed for causal variants """ n_variants = X.shape[1] n_samples = X.shape[0] n_causal = n_blocks * n_causal_per_block R2 = np.corrcoef(X.T)**2 active = (ldscore > min_ldscore) & (ldscore < max_ldscore) causal_snps = select_causal_snps(R2, n_causal, min_r2, max_r2, active).reshape(n_causal_per_block, -1) # draw block ids and causal snps # ensure each block gets at least one tissue block_id = np.sort( np.concatenate([ np.arange(n_blocks), np.random.choice(n_blocks, n_study - n_blocks) ])) # make block probability matrix causal_p = np.eye(n_blocks) causal_p[causal_p == 0] = block_p results = [] for t in range(n_study): # sample causal snps for tissue causal_idx = np.random.binomial(1, causal_p[block_id[t]]) == 1 causal_in_study = np.concatenate( [cs[causal_idx] for cs in causal_snps]) results.append( sim_expression_single_study(X, afreq, causal_in_study, pve, effect_distribution)) expression = np.atleast_2d(np.array([x[0] for x in results])) true_effects = np.atleast_2d(np.array([x[1] for x in results])) residual_variance = np.array([x[2] for x in results]) # trim down to the causal snps we actually used causal_snps = np.arange(n_variants)[np.any(true_effects != 0, 0)] tril = np.tril_indices(n_study, k=-1) true_coloc = (true_effects @ true_effects.T != 0)[tril] return { 'expression': expression, 'true_effects': true_effects, 'true_coloc': true_coloc, 'residual_variance': residual_variance, 'causal_snps': causal_snps, 'n_causal': causal_snps.size, 'K': int(np.ceil(causal_snps.size / 10) * 10), 'ldscore': ldscore }
def __init__(self, endog, exog=None, order=(1, 0), trend='c', error_cov_type='unstructured', measurement_error=False, enforce_stationarity=True, enforce_invertibility=True, **kwargs): # Model parameters self.error_cov_type = error_cov_type self.measurement_error = measurement_error self.enforce_stationarity = enforce_stationarity self.enforce_invertibility = enforce_invertibility # Save the given orders self.order = order self.trend = trend # Model orders self.k_ar = int(order[0]) self.k_ma = int(order[1]) self.k_trend = int(self.trend == 'c') # Check for valid model if trend not in ['c', 'nc']: raise ValueError('Invalid trend specification.') if error_cov_type not in ['diagonal', 'unstructured']: raise ValueError('Invalid error covariance matrix type' ' specification.') if self.k_ar == 0 and self.k_ma == 0: raise ValueError('Invalid VARMAX(p,q) specification; at least one' ' p,q must be greater than zero.') # Warn for VARMA model if self.k_ar > 0 and self.k_ma > 0: warn( 'Estimation of VARMA(p,q) models is not generically robust,' ' due especially to identification issues.', EstimationWarning) # Exogenous data self.k_exog = 0 if exog is not None: exog_is_using_pandas = _is_using_pandas(exog, None) if not exog_is_using_pandas: exog = np.asarray(exog) # Make sure we have 2-dimensional array if exog.ndim == 1: if not exog_is_using_pandas: exog = exog[:, None] else: exog = pd.DataFrame(exog) self.k_exog = exog.shape[1] # Note: at some point in the future might add state regression, as in # SARIMAX. self.mle_regression = self.k_exog > 0 # We need to have an array or pandas at this point if not _is_using_pandas(endog, None): endog = np.asanyarray(endog) # Model order # Used internally in various places _min_k_ar = max(self.k_ar, 1) self._k_order = _min_k_ar + self.k_ma # Number of states k_endog = endog.shape[1] k_posdef = k_endog k_states = k_endog * self._k_order # By default, initialize as stationary kwargs.setdefault('initialization', 'stationary') # By default, use LU decomposition kwargs.setdefault('inversion_method', INVERT_UNIVARIATE | SOLVE_LU) # Initialize the state space model super(VARMAX, self).__init__(endog, exog=exog, k_states=k_states, k_posdef=k_posdef, **kwargs) # Set as time-varying model if we have time-trend or exog if self.k_exog > 0 or self.k_trend > 1: self.ssm._time_invariant = False # Initialize the parameters self.parameters = OrderedDict() self.parameters['trend'] = self.k_endog * self.k_trend self.parameters['ar'] = self.k_endog**2 * self.k_ar self.parameters['ma'] = self.k_endog**2 * self.k_ma self.parameters['regression'] = self.k_endog * self.k_exog if self.error_cov_type == 'diagonal': self.parameters['state_cov'] = self.k_endog # These parameters fill in a lower-triangular matrix which is then # dotted with itself to get a positive definite matrix. elif self.error_cov_type == 'unstructured': self.parameters['state_cov'] = (int(self.k_endog * (self.k_endog + 1) / 2)) self.parameters['obs_cov'] = self.k_endog * self.measurement_error self.k_params = sum(self.parameters.values()) # Initialize known elements of the state space matrices # If we have exog effects, then the state intercept needs to be # time-varying if self.k_exog > 0: self.ssm['state_intercept'] = np.zeros((self.k_states, self.nobs)) # The design matrix is just an identity for the first k_endog states idx = np.diag_indices(self.k_endog) self.ssm[('design', ) + idx] = 1 # The transition matrix is described in four blocks, where the upper # left block is in companion form with the autoregressive coefficient # matrices (so it is shaped k_endog * k_ar x k_endog * k_ar) ... if self.k_ar > 0: idx = np.diag_indices((self.k_ar - 1) * self.k_endog) idx = idx[0] + self.k_endog, idx[1] self.ssm[('transition', ) + idx] = 1 # ... and the lower right block is in companion form with zeros as the # coefficient matrices (it is shaped k_endog * k_ma x k_endog * k_ma). idx = np.diag_indices((self.k_ma - 1) * self.k_endog) idx = (idx[0] + (_min_k_ar + 1) * self.k_endog, idx[1] + _min_k_ar * self.k_endog) self.ssm[('transition', ) + idx] = 1 # The selection matrix is described in two blocks, where the upper # block selects the all k_posdef errors in the first k_endog rows # (the upper block is shaped k_endog * k_ar x k) and the lower block # also selects all k_posdef errors in the first k_endog rows (the lower # block is shaped k_endog * k_ma x k). idx = np.diag_indices(self.k_endog) self.ssm[('selection', ) + idx] = 1 idx = idx[0] + _min_k_ar * self.k_endog, idx[1] if self.k_ma > 0: self.ssm[('selection', ) + idx] = 1 # Cache some indices if self.trend == 'c' and self.k_exog == 0: self._idx_state_intercept = np.s_['state_intercept', :k_endog] elif self.k_exog > 0: self._idx_state_intercept = np.s_['state_intercept', :k_endog, :] if self.k_ar > 0: self._idx_transition = np.s_['transition', :k_endog, :] else: self._idx_transition = np.s_['transition', :k_endog, k_endog:] if self.error_cov_type == 'diagonal': self._idx_state_cov = (('state_cov', ) + np.diag_indices(self.k_endog)) elif self.error_cov_type == 'unstructured': self._idx_lower_state_cov = np.tril_indices(self.k_endog) if self.measurement_error: self._idx_obs_cov = ('obs_cov', ) + np.diag_indices(self.k_endog) # Cache some slices def _slice(key, offset): length = self.parameters[key] param_slice = np.s_[offset:offset + length] offset += length return param_slice, offset offset = 0 self._params_trend, offset = _slice('trend', offset) self._params_ar, offset = _slice('ar', offset) self._params_ma, offset = _slice('ma', offset) self._params_regression, offset = _slice('regression', offset) self._params_state_cov, offset = _slice('state_cov', offset) self._params_obs_cov, offset = _slice('obs_cov', offset)
def test_gradient(): from brainiak.reprsimil.brsa import BRSA import brainiak.utils.utils as utils import scipy.stats import numpy as np import os.path import numdifftools as nd np.random.seed(100) file_path = os.path.join(os.path.dirname(__file__), "example_design.1D") # Load an example design matrix design = utils.ReadDesign(fname=file_path) # concatenate it by 4 times, mimicking 4 runs of itenditcal timing design.design_used = np.tile(design.design_used[:, 0:17], [4, 1]) design.n_TR = design.n_TR * 4 # start simulating some data n_V = 200 n_C = np.size(design.design_used, axis=1) n_T = design.n_TR noise_bot = 0.5 noise_top = 1.5 noise_level = np.random.rand(n_V) * (noise_top - noise_bot) + noise_bot # noise level is random. # AR(1) coefficient rho1_top = 0.8 rho1_bot = -0.2 rho1 = np.random.rand(n_V) * (rho1_top - rho1_bot) + rho1_bot # generating noise noise = np.zeros([n_T, n_V]) noise[0, :] = np.random.randn(n_V) * noise_level / np.sqrt(1 - rho1**2) for i_t in range(1, n_T): noise[i_t, :] = noise[i_t - 1, :] * rho1 + np.random.randn(n_V) * noise_level # ideal covariance matrix ideal_cov = np.zeros([n_C, n_C]) ideal_cov = np.eye(n_C) * 0.6 ideal_cov[0, 0] = 0.2 ideal_cov[5:9, 5:9] = 0.6 for cond in range(5, 9): ideal_cov[cond, cond] = 1 idx = np.where(np.sum(np.abs(ideal_cov), axis=0) > 0)[0] L_full = np.linalg.cholesky(ideal_cov) # generating signal snr_level = 5.0 # test with high SNR # snr = np.random.rand(n_V)*(snr_top-snr_bot)+snr_bot # Notice that accurately speaking this is not snr. the magnitude of signal depends # not only on beta but also on x. inten = np.random.randn(n_V) * 20.0 # parameters of Gaussian process to generate pseuso SNR tau = 0.8 smooth_width = 5.0 inten_kernel = 1.0 coords = np.arange(0, n_V)[:, None] dist2 = np.square(coords - coords.T) inten_tile = np.tile(inten, [n_V, 1]) inten_diff2 = (inten_tile - inten_tile.T)**2 K = np.exp(-dist2 / smooth_width**2 / 2.0 - inten_diff2 / inten_kernel**2 / 2.0) * tau**2 + np.eye(n_V) * tau**2 * 0.001 L = np.linalg.cholesky(K) snr = np.exp(np.dot(L, np.random.randn(n_V))) * snr_level sqrt_v = noise_level * snr betas_simulated = np.dot(L_full, np.random.randn(n_C, n_V)) * sqrt_v signal = np.dot(design.design_used, betas_simulated) # Adding noise to signal as data Y = signal + noise scan_onsets = np.linspace(0, design.n_TR, num=5) # Test fitting with GP prior. brsa = BRSA(GP_space=True, GP_inten=True, verbose=False, n_iter=200) # test if the gradients are correct XTY, XTDY, XTFY, YTY_diag, YTDY_diag, YTFY_diag, XTX, XTDX, XTFX = brsa._prepare_data( design.design_used, Y, n_T, n_V, scan_onsets) l_idx = np.tril_indices(n_C) n_l = np.size(l_idx[0]) idx_param_sing, idx_param_fitU, idx_param_fitV = brsa._build_index_param( n_l, n_V, 2) # Initial parameters are correct parameters with some perturbation param0_fitU = np.random.randn(n_l + n_V) * 0.1 param0_fitV = np.random.randn(n_V + 1) * 0.1 param0_fitV[:n_V - 1] += np.log(snr[:n_V - 1]) * 2 param0_fitV[n_V - 1] += np.log(smooth_width) * 2 param0_fitV[n_V] += np.log(inten_kernel) * 2 # log likelihood and derivative at the initial parameters ll0, deriv0 = brsa._loglike_AR1_diagV_fitU(param0_fitU, XTX, XTDX, XTFX, YTY_diag, YTDY_diag, YTFY_diag, \ XTY, XTDY, XTFY, np.log(snr)*2, l_idx,n_C,n_T,n_V,idx_param_fitU,n_C) # We test if the numerical and analytical gradient wrt to the first element of Cholesky factor is correct vec = np.zeros(np.size(param0_fitU)) vec[idx_param_fitU['Cholesky'][0]] = 1 dd = nd.directionaldiff(lambda x: brsa._loglike_AR1_diagV_fitU(x, XTX, XTDX, XTFX, YTY_diag, YTDY_diag,\ YTFY_diag, XTY, XTDY, XTFY, np.log(snr)*2,\ l_idx,n_C,n_T,n_V,idx_param_fitU,n_C)[0], param0_fitU, vec) assert np.isclose( dd, np.dot(deriv0, vec), rtol=0.01), 'gradient of fitU wrt Cholesky factor incorrect' # We test the gradient wrt the reparametrization of AR(1) coefficient of noise. vec = np.zeros(np.size(param0_fitU)) vec[idx_param_fitU['a1'][0]] = 1 dd = nd.directionaldiff(lambda x: brsa._loglike_AR1_diagV_fitU(x, XTX, XTDX, XTFX, YTY_diag, YTDY_diag,\ YTFY_diag, XTY, XTDY, XTFY, np.log(snr)*2,\ l_idx,n_C,n_T,n_V,idx_param_fitU,n_C)[0], param0_fitU, vec) assert np.isclose( dd, np.dot(deriv0, vec), rtol=0.01), 'gradient of fitU wrt to AR(1) coefficient incorrect' # Test on a random direction vec = np.random.randn(np.size(param0_fitU)) vec = vec / np.linalg.norm(vec) dd = nd.directionaldiff(lambda x: brsa._loglike_AR1_diagV_fitU(x, XTX, XTDX, XTFX, YTY_diag, YTDY_diag,\ YTFY_diag, XTY, XTDY, XTFY, np.log(snr)*2,\ l_idx,n_C,n_T,n_V,idx_param_fitU,n_C)[0], param0_fitU, vec) assert np.isclose(dd, np.dot(deriv0, vec), rtol=0.01), 'gradient of fitU incorrect' # We test the gradient of _fitV wrt to log(SNR^2) assuming no GP prior. ll0, deriv0 = brsa._loglike_AR1_diagV_fitV( param0_fitV[idx_param_fitV['log_SNR2']], XTX, XTDX, XTFX, YTY_diag, YTDY_diag, YTFY_diag, XTY, XTDY, XTFY, L_full[l_idx], np.tan(rho1 * np.pi / 2), l_idx, n_C, n_T, n_V, idx_param_fitV, n_C, False, False) vec = np.zeros(np.size(param0_fitV[idx_param_fitV['log_SNR2']])) vec[idx_param_fitV['log_SNR2'][0]] = 1 dd = nd.directionaldiff( lambda x: brsa._loglike_AR1_diagV_fitV( x, XTX, XTDX, XTFX, YTY_diag, YTDY_diag, YTFY_diag, XTY, XTDY, XTFY, L_full[l_idx], np.tan(rho1 * np.pi / 2), l_idx, n_C, n_T, n_V, idx_param_fitV, n_C, False, False)[0], param0_fitV[idx_param_fitV['log_SNR2']], vec) assert np.isclose( dd, np.dot(deriv0, vec), rtol=0.01 ), 'gradient of fitV wrt log(SNR2) incorrect for model without GP' # We test the gradient of _fitV wrt to log(SNR^2) assuming GP prior. ll0, deriv0 = brsa._loglike_AR1_diagV_fitV(param0_fitV, XTX, XTDX, XTFX, YTY_diag, YTDY_diag, YTFY_diag, XTY, XTDY, XTFY, L_full[l_idx], np.tan(rho1 * np.pi / 2), l_idx, n_C, n_T, n_V, idx_param_fitV, n_C, True, True, dist2, inten_diff2, 100, 100) vec = np.zeros(np.size(param0_fitV)) vec[idx_param_fitV['log_SNR2'][0]] = 1 dd = nd.directionaldiff( lambda x: brsa._loglike_AR1_diagV_fitV( x, XTX, XTDX, XTFX, YTY_diag, YTDY_diag, YTFY_diag, XTY, XTDY, XTFY, L_full[l_idx], np.tan(rho1 * np.pi / 2), l_idx, n_C, n_T, n_V, idx_param_fitV, n_C, True, True, dist2, inten_diff2, 100, 100)[0], param0_fitV, vec) assert np.isclose( dd, np.dot(deriv0, vec), rtol=0.01 ), 'gradient of fitV srt log(SNR2) incorrect for model with GP' # We test the graident wrt spatial length scale parameter of GP prior vec = np.zeros(np.size(param0_fitV)) vec[idx_param_fitV['c_space']] = 1 dd = nd.directionaldiff( lambda x: brsa._loglike_AR1_diagV_fitV( x, XTX, XTDX, XTFX, YTY_diag, YTDY_diag, YTFY_diag, XTY, XTDY, XTFY, L_full[l_idx], np.tan(rho1 * np.pi / 2), l_idx, n_C, n_T, n_V, idx_param_fitV, n_C, True, True, dist2, inten_diff2, 100, 100)[0], param0_fitV, vec) assert np.isclose( dd, np.dot(deriv0, vec), rtol=0.01), 'gradient of fitV wrt spatial length scale of GP incorrect' # We test the graident wrt intensity length scale parameter of GP prior vec = np.zeros(np.size(param0_fitV)) vec[idx_param_fitV['c_inten']] = 1 dd = nd.directionaldiff( lambda x: brsa._loglike_AR1_diagV_fitV( x, XTX, XTDX, XTFX, YTY_diag, YTDY_diag, YTFY_diag, XTY, XTDY, XTFY, L_full[l_idx], np.tan(rho1 * np.pi / 2), l_idx, n_C, n_T, n_V, idx_param_fitV, n_C, True, True, dist2, inten_diff2, 100, 100)[0], param0_fitV, vec) assert np.isclose( dd, np.dot(deriv0, vec), rtol=0.01 ), 'gradient of fitV wrt intensity length scale of GP incorrect' # We test the graident on a random direction vec = np.random.randn(np.size(param0_fitV)) vec = vec / np.linalg.norm(vec) dd = nd.directionaldiff( lambda x: brsa._loglike_AR1_diagV_fitV( x, XTX, XTDX, XTFX, YTY_diag, YTDY_diag, YTFY_diag, XTY, XTDY, XTFY, L_full[l_idx], np.tan(rho1 * np.pi / 2), l_idx, n_C, n_T, n_V, idx_param_fitV, n_C, True, True, dist2, inten_diff2, 100, 100)[0], param0_fitV, vec) assert np.isclose(dd, np.dot(deriv0, vec), rtol=0.01), 'gradient of fitV incorrect'
def _make_eris_outcore(mycc, mo_coeff=None): cput0 = (time.clock(), time.time()) log = logger.Logger(mycc.stdout, mycc.verbose) eris = _PhysicistsERIs() eris._common_init_(mycc, mo_coeff) nocc = eris.nocc nao, nmo = eris.mo_coeff.shape nvir = nmo - nocc assert (eris.mo_coeff.dtype == np.double) mo_a = eris.mo_coeff[:nao // 2] mo_b = eris.mo_coeff[nao // 2:] orbspin = eris.orbspin feri = eris.feri = lib.H5TmpFile() dtype = np.result_type(eris.mo_coeff).char eris.oooo = feri.create_dataset('oooo', (nocc, nocc, nocc, nocc), dtype) eris.ooov = feri.create_dataset('ooov', (nocc, nocc, nocc, nvir), dtype) eris.oovv = feri.create_dataset('oovv', (nocc, nocc, nvir, nvir), dtype) eris.ovov = feri.create_dataset('ovov', (nocc, nvir, nocc, nvir), dtype) eris.ovvo = feri.create_dataset('ovvo', (nocc, nvir, nvir, nocc), dtype) eris.ovvv = feri.create_dataset('ovvv', (nocc, nvir, nvir, nvir), dtype) if orbspin is None: orbo_a = mo_a[:, :nocc] orbv_a = mo_a[:, nocc:] orbo_b = mo_b[:, :nocc] orbv_b = mo_b[:, nocc:] max_memory = mycc.max_memory - lib.current_memory()[0] blksize = min(nocc, max(2, int(max_memory * 1e6 / 8 / (nmo**3 * 2)))) max_memory = max(MEMORYMIN, max_memory) fswap = lib.H5TmpFile() ao2mo.kernel(mycc.mol, (orbo_a, mo_a, mo_a, mo_a), fswap, 'aaaa', max_memory=max_memory, verbose=log) ao2mo.kernel(mycc.mol, (orbo_a, mo_a, mo_b, mo_b), fswap, 'aabb', max_memory=max_memory, verbose=log) ao2mo.kernel(mycc.mol, (orbo_b, mo_b, mo_a, mo_a), fswap, 'bbaa', max_memory=max_memory, verbose=log) ao2mo.kernel(mycc.mol, (orbo_b, mo_b, mo_b, mo_b), fswap, 'bbbb', max_memory=max_memory, verbose=log) for p0, p1 in lib.prange(0, nocc, blksize): tmp = np.asarray(fswap['aaaa'][p0 * nmo:p1 * nmo]) tmp += np.asarray(fswap['aabb'][p0 * nmo:p1 * nmo]) tmp += np.asarray(fswap['bbaa'][p0 * nmo:p1 * nmo]) tmp += np.asarray(fswap['bbbb'][p0 * nmo:p1 * nmo]) tmp = lib.unpack_tril(tmp).reshape(p1 - p0, nmo, nmo, nmo) eris.oooo[p0:p1] = ( tmp[:, :nocc, :nocc, :nocc].transpose(0, 2, 1, 3) - tmp[:, :nocc, :nocc, :nocc].transpose(0, 2, 3, 1)) eris.ooov[p0:p1] = ( tmp[:, :nocc, :nocc, nocc:].transpose(0, 2, 1, 3) - tmp[:, nocc:, :nocc, :nocc].transpose(0, 2, 3, 1)) eris.ovvv[p0:p1] = ( tmp[:, nocc:, nocc:, nocc:].transpose(0, 2, 1, 3) - tmp[:, nocc:, nocc:, nocc:].transpose(0, 2, 3, 1)) eris.oovv[p0:p1] = ( tmp[:, nocc:, :nocc, nocc:].transpose(0, 2, 1, 3) - tmp[:, nocc:, :nocc, nocc:].transpose(0, 2, 3, 1)) eris.ovov[p0:p1] = ( tmp[:, :nocc, nocc:, nocc:].transpose(0, 2, 1, 3) - tmp[:, nocc:, nocc:, :nocc].transpose(0, 2, 3, 1)) eris.ovvo[p0:p1] = ( tmp[:, nocc:, nocc:, :nocc].transpose(0, 2, 1, 3) - tmp[:, :nocc, nocc:, nocc:].transpose(0, 2, 3, 1)) tmp = None cput0 = log.timer_debug1('transforming ovvv', *cput0) eris.vvvv = feri.create_dataset('vvvv', (nvir, nvir, nvir, nvir), dtype) tril2sq = lib.square_mat_in_trilu_indices(nvir) fswap = lib.H5TmpFile() ao2mo.kernel(mycc.mol, (orbv_a, orbv_a, orbv_a, orbv_a), fswap, 'aaaa', max_memory=max_memory, verbose=log) ao2mo.kernel(mycc.mol, (orbv_a, orbv_a, orbv_b, orbv_b), fswap, 'aabb', max_memory=max_memory, verbose=log) ao2mo.kernel(mycc.mol, (orbv_b, orbv_b, orbv_a, orbv_a), fswap, 'bbaa', max_memory=max_memory, verbose=log) ao2mo.kernel(mycc.mol, (orbv_b, orbv_b, orbv_b, orbv_b), fswap, 'bbbb', max_memory=max_memory, verbose=log) for p0, p1 in lib.prange(0, nvir, blksize): off0 = p0 * (p0 + 1) // 2 off1 = p1 * (p1 + 1) // 2 tmp = np.asarray(fswap['aaaa'][off0:off1]) tmp += np.asarray(fswap['aabb'][off0:off1]) tmp += np.asarray(fswap['bbaa'][off0:off1]) tmp += np.asarray(fswap['bbbb'][off0:off1]) if p0 > 0: c = tmp[tril2sq[p0:p1, :p0] - off0] c = lib.unpack_tril(c.reshape((p1 - p0) * p0, -1)) eris.vvvv[p0:p1, :p0] = c.reshape(p1 - p0, p0, nvir, nvir) c = tmp[tril2sq[:p1, p0:p1] - off0] c = lib.unpack_tril(c.reshape((p1 - p0) * p1, -1)) eris.vvvv[:p1, p0:p1] = c.reshape(p1, p1 - p0, nvir, nvir) tmp = None for p0, p1 in lib.prange(0, nvir, blksize): tmp = np.asarray(eris.vvvv[p0:p1]) eris.vvvv[p0:p1] = tmp.transpose(0, 2, 1, 3) - tmp.transpose( 0, 2, 3, 1) cput0 = log.timer_debug1('transforming vvvv', *cput0) else: # with orbspin mo = mo_a + mo_b orbo = mo[:, :nocc] orbv = mo[:, nocc:] max_memory = mycc.max_memory - lib.current_memory()[0] blksize = min(nocc, max(2, int(max_memory * 1e6 / 8 / (nmo**3 * 2)))) max_memory = max(MEMORYMIN, max_memory) fswap = lib.H5TmpFile() ao2mo.kernel(mycc.mol, (orbo, mo, mo, mo), fswap, max_memory=max_memory, verbose=log) sym_forbid = orbspin[:, None] != orbspin for p0, p1 in lib.prange(0, nocc, blksize): tmp = np.asarray(fswap['eri_mo'][p0 * nmo:p1 * nmo]) tmp = lib.unpack_tril(tmp).reshape(p1 - p0, nmo, nmo, nmo) tmp[sym_forbid[p0:p1]] = 0 tmp[:, :, sym_forbid] = 0 eris.oooo[p0:p1] = ( tmp[:, :nocc, :nocc, :nocc].transpose(0, 2, 1, 3) - tmp[:, :nocc, :nocc, :nocc].transpose(0, 2, 3, 1)) eris.ooov[p0:p1] = ( tmp[:, :nocc, :nocc, nocc:].transpose(0, 2, 1, 3) - tmp[:, nocc:, :nocc, :nocc].transpose(0, 2, 3, 1)) eris.ovvv[p0:p1] = ( tmp[:, nocc:, nocc:, nocc:].transpose(0, 2, 1, 3) - tmp[:, nocc:, nocc:, nocc:].transpose(0, 2, 3, 1)) eris.oovv[p0:p1] = ( tmp[:, nocc:, :nocc, nocc:].transpose(0, 2, 1, 3) - tmp[:, nocc:, :nocc, nocc:].transpose(0, 2, 3, 1)) eris.ovov[p0:p1] = ( tmp[:, :nocc, nocc:, nocc:].transpose(0, 2, 1, 3) - tmp[:, nocc:, nocc:, :nocc].transpose(0, 2, 3, 1)) eris.ovvo[p0:p1] = ( tmp[:, nocc:, nocc:, :nocc].transpose(0, 2, 1, 3) - tmp[:, :nocc, nocc:, nocc:].transpose(0, 2, 3, 1)) tmp = None cput0 = log.timer_debug1('transforming ovvv', *cput0) eris.vvvv = feri.create_dataset('vvvv', (nvir, nvir, nvir, nvir), dtype) sym_forbid = (orbspin[nocc:, None] != orbspin[nocc:])[np.tril_indices(nvir)] tril2sq = lib.square_mat_in_trilu_indices(nvir) fswap = lib.H5TmpFile() ao2mo.kernel(mycc.mol, orbv, fswap, max_memory=max_memory, verbose=log) for p0, p1 in lib.prange(0, nvir, blksize): off0 = p0 * (p0 + 1) // 2 off1 = p1 * (p1 + 1) // 2 tmp = np.asarray(fswap['eri_mo'][off0:off1]) tmp[sym_forbid[off0:off1]] = 0 tmp[:, sym_forbid] = 0 if p0 > 0: c = tmp[tril2sq[p0:p1, :p0] - off0] c = lib.unpack_tril(c.reshape((p1 - p0) * p0, -1)) eris.vvvv[p0:p1, :p0] = c.reshape(p1 - p0, p0, nvir, nvir) c = tmp[tril2sq[:p1, p0:p1] - off0] c = lib.unpack_tril(c.reshape((p1 - p0) * p1, -1)) eris.vvvv[:p1, p0:p1] = c.reshape(p1, p1 - p0, nvir, nvir) tmp = None for p0, p1 in lib.prange(0, nvir, blksize): tmp = np.asarray(eris.vvvv[p0:p1]) eris.vvvv[p0:p1] = tmp.transpose(0, 2, 1, 3) - tmp.transpose( 0, 2, 3, 1) cput0 = log.timer_debug1('transforming vvvv', *cput0) return eris
def tril_indices(self): return np.tril_indices(self.ddim)
def call(self, x, mask=None): # TODO: validate input shape # The input of this layer is [L, mu, a] in concatenated form. We first split # those up. idx = 0 if self.mode == 'full': L_flat = x[:, idx:idx + (self.nb_actions * self.nb_actions + self.nb_actions) // 2] idx += (self.nb_actions * self.nb_actions + self.nb_actions) // 2 elif self.mode == 'diag': L_flat = x[:, idx:idx + self.nb_actions] idx += self.nb_actions else: L_flat = None assert L_flat is not None mu = x[:, idx:idx + self.nb_actions] idx += self.nb_actions a = x[:, idx:idx + self.nb_actions] idx += self.nb_actions if self.mode == 'full': # Create L and L^T matrix, which we use to construct the positive-definite matrix P. L = None LT = None if K._BACKEND == 'theano': import theano.tensor as T import theano def fn(x, L_acc, LT_acc): x_ = K.zeros((self.nb_actions, self.nb_actions)) x_ = T.set_subtensor(x_[np.tril_indices(self.nb_actions)], x) diag = K.exp(T.diag(x_) + K.epsilon()) x_ = T.set_subtensor(x_[np.diag_indices(self.nb_actions)], diag) return x_, x_.T outputs_info = [ K.zeros((self.nb_actions, self.nb_actions)), K.zeros((self.nb_actions, self.nb_actions)), ] results, _ = theano.scan(fn=fn, sequences=L_flat, outputs_info=outputs_info) L, LT = results elif K._BACKEND == 'tensorflow': import tensorflow as tf # Number of elements in a triangular matrix. nb_elems = (self.nb_actions * self.nb_actions + self.nb_actions) // 2 # Create mask for the diagonal elements in L_flat. This is used to exponentiate # only the diagonal elements, which is done before gathering. diag_indeces = [0] for row in range(1, self.nb_actions): diag_indeces.append(diag_indeces[-1] + (row + 1)) diag_mask = np.zeros(1 + nb_elems) # +1 for the leading zero diag_mask[np.array(diag_indeces) + 1] = 1 diag_mask = K.variable(diag_mask) # Add leading zero element to each element in the L_flat. We use this zero # element when gathering L_flat into a lower triangular matrix L. nb_rows = tf.shape(L_flat)[0] zeros = tf.expand_dims(tf.tile(K.zeros((1, )), [nb_rows]), 1) L_flat = tf.concat(1, [zeros, L_flat]) # Create mask that can be used to gather elements from L_flat and put them # into a lower triangular matrix. tril_mask = np.zeros((self.nb_actions, self.nb_actions), dtype='int32') tril_mask[np.tril_indices(self.nb_actions)] = range( 1, nb_elems + 1) # Finally, process each element of the batch. init = [ K.zeros((self.nb_actions, self.nb_actions)), K.zeros((self.nb_actions, self.nb_actions)), ] def fn(a, x): # Exponentiate everything. This is much easier than only exponentiating # the diagonal elements, and, usually, the action space is relatively low. x_ = K.exp(x + K.epsilon()) # Only keep the diagonal elements. x_ *= diag_mask # Add the original, non-diagonal elements. x_ += x * (1. - diag_mask) # Finally, gather everything into a lower triangular matrix. L_ = tf.gather(x_, tril_mask) return [L_, tf.transpose(L_)] tmp = tf.scan(fn, L_flat, initializer=init) if isinstance(tmp, (list, tuple)): # TensorFlow 0.10 now returns a tuple of tensors. L, LT = tmp else: # Old TensorFlow < 0.10 returns a shared tensor. L = tmp[:, 0, :, :] LT = tmp[:, 1, :, :] else: raise RuntimeError('Unknown Keras backend "{}".'.format( K._BACKEND)) assert L is not None assert LT is not None P = K.batch_dot(L, LT) elif self.mode == 'diag': if K._BACKEND == 'theano': import theano.tensor as T import theano def fn(x, P_acc): x_ = K.zeros((self.nb_actions, self.nb_actions)) x_ = T.set_subtensor(x_[np.diag_indices(self.nb_actions)], x) return x_ outputs_info = [ K.zeros((self.nb_actions, self.nb_actions)), ] P, _ = theano.scan(fn=fn, sequences=L_flat, outputs_info=outputs_info) elif K._BACKEND == 'tensorflow': import tensorflow as tf # Create mask that can be used to gather elements from L_flat and put them # into a diagonal matrix. diag_mask = np.zeros((self.nb_actions, self.nb_actions), dtype='int32') diag_mask[np.diag_indices(self.nb_actions)] = range( 1, self.nb_actions + 1) # Add leading zero element to each element in the L_flat. We use this zero # element when gathering L_flat into a lower triangular matrix L. nb_rows = tf.shape(L_flat)[0] zeros = tf.expand_dims(tf.tile(K.zeros((1, )), [nb_rows]), 1) L_flat = tf.concat(1, [zeros, L_flat]) # Finally, process each element of the batch. def fn(a, x): x_ = tf.gather(x, diag_mask) return x_ P = tf.scan(fn, L_flat, initializer=K.zeros( (self.nb_actions, self.nb_actions))) else: raise RuntimeError('Unknown Keras backend "{}".'.format( K._BACKEND)) assert P is not None assert K.ndim(P) == 3 # Combine a, mu and P into a scalar (over the batches). What we compute here is # -.5 * (a - mu)^T * P * (a - mu), where * denotes the dot-product. Unfortunately # TensorFlow handles vector * P slightly suboptimal, hence we convert the vectors to # 1xd/dx1 matrices and finally flatten the resulting 1x1 matrix into a scalar. All # operations happen over the batch size, which is dimension 0. prod = K.batch_dot(K.expand_dims(a - mu, dim=1), P) prod = K.batch_dot(prod, K.expand_dims(a - mu, dim=-1)) A = -.5 * K.batch_flatten(prod) assert K.ndim(A) == 2 return A
def gradient_marginal_loglikelihood(self, observations, parameters, forward_message=None, backward_message=None, weights=None, tqdm=None): # Forward Pass forward_messages = self.forward_pass(observations, parameters, forward_message, include_init_message=True) # Backward Pass backward_messages = self.backward_pass(observations, parameters, backward_message, include_init_message=True) # Gradients grad = {var: np.zeros_like(value) for var, value in parameters.as_dict().items()} Pi, expanded_pi = parameters.pi, parameters.expanded_pi D = parameters.D LRinv, Rinv, R = parameters.LRinv, parameters.Rinv, parameters.R pbar = enumerate(zip(forward_messages[:-1], backward_messages[1:])) if tqdm is not None: pbar = tqdm(pbar) pbar.set_description("gradient loglike") for t, (forward_t, backward_t) in pbar: # r_t is Pr(z_{t-1} | y_{< t}) # s_t is Pr(z_t | y_{< t}) # q_t is Pr(y_{> t} | z_t) r_t = forward_t['prob_vector'] s_t = np.dot(r_t, Pi) q_t = backward_t['likelihood_vector'] weight_t = 1.0 if weights is None else weights[t] # Calculate P_t = Pr(y_t | z_t) y_cur = observations[t] P_t, _ = self._likelihoods( y_cur=y_cur, parameters=parameters ) # Marginal + Pairwise Marginal joint_post = np.diag(r_t).dot(Pi).dot(np.diag(P_t*q_t)) joint_post = joint_post/np.sum(joint_post) marg_post = np.sum(joint_post, axis=0) # Grad for pi if parameters.pi_type == "logit": # Gradient of logit_pi grad['logit_pi'] += weight_t * (joint_post - \ np.diag(np.sum(joint_post, axis=1)).dot(Pi)) elif parameters.pi_type == "expanded": grad['expanded_pi'] += weight_t * np.array([ (expanded_pi[k]**-1)*( joint_post[k] - np.sum(joint_post[k])*Pi[k]) for k in range(self.num_states) ]) else: raise RuntimeError() # grad for mu and LRinv y_prev = y_cur[1:].flatten() for k, D_k, LRinv_k, Rinv_k, R_k in zip( range(self.num_states), D, LRinv, Rinv, R): diff_k = y_cur[0] - np.dot(D_k, y_prev) grad['D'][k] += weight_t * ( np.outer(Rinv_k.dot(diff_k), y_prev) * marg_post[k]) grad_LRinv_k = weight_t * ( (R_k - np.outer(diff_k, diff_k)).dot(LRinv_k) ) * marg_post[k] grad['LRinv_vec'][k] += grad_LRinv_k[np.tril_indices(self.m)] return grad
def test_ft_aoao(self): #coords = pdft.gen_grid.gen_uniform_grids(cell) #aoR = pdft.numint.eval_ao(cell, coords) #ngrids, nao = aoR.shape #ref = numpy.asarray([tools.fft(aoR[:,i].conj()*aoR[:,j], cell.mesh) # for i in range(nao) for j in range(nao)]) #ref = ref.reshape(nao,nao,-1).transpose(2,0,1) * (cell.vol/ngrids) #dat = ft_ao.ft_aopair(cell, cell.Gv, aosym='s1hermi') #self.assertAlmostEqual(numpy.linalg.norm(ref[:,0,0]-dat[:,0,0]) , 0, 5) #self.assertAlmostEqual(numpy.linalg.norm(ref[:,1,1]-dat[:,1,1]) , 0.02315483195832373, 4) #self.assertAlmostEqual(numpy.linalg.norm(ref[:,2:,2:]-dat[:,2:,2:]), 0, 9) #self.assertAlmostEqual(numpy.linalg.norm(ref[:,0,2:]-dat[:,0,2:]) , 0, 9) #self.assertAlmostEqual(numpy.linalg.norm(ref[:,2:,0]-dat[:,2:,0]) , 0, 9) #idx = numpy.tril_indices(nao) #ref = dat[:,idx[0],idx[1]] #dat = ft_ao.ft_aopair(cell, cell.Gv, aosym='s2') #self.assertAlmostEqual(abs(dat-ref).sum(), 0, 9) coords = pdft.gen_grid.gen_uniform_grids(cell1) Gv, Gvbase, kws = cell1.get_Gv_weights(cell1.mesh) b = cell1.reciprocal_vectors() gxyz = lib.cartesian_prod([numpy.arange(len(x)) for x in Gvbase]) dat = ft_ao.ft_aopair(cell1, cell1.Gv, aosym='s1', b=b, gxyz=gxyz, Gvbase=Gvbase) self.assertAlmostEqual(lib.fp(dat), 1.5666516306798806 + 1.953555017583245j, 9) dat = ft_ao.ft_aopair(cell1, cell1.Gv, aosym='s2', b=b, gxyz=gxyz, Gvbase=Gvbase) self.assertAlmostEqual(lib.fp(dat), -0.85276967757297917 + 1.0378751267506394j, 9) dat = ft_ao.ft_aopair(cell1, cell1.Gv, aosym='s1hermi', b=b, gxyz=gxyz, Gvbase=Gvbase) self.assertAlmostEqual(lib.fp(dat), 1.5666516306798806 + 1.953555017583245j, 9) aoR = pdft.numint.eval_ao(cell1, coords) ngrids, nao = aoR.shape aoaoR = numpy.einsum('pi,pj->ijp', aoR, aoR) ref = tools.fft(aoaoR.reshape(nao * nao, -1), cell1.mesh) ref = ref.reshape(nao, nao, -1).transpose(2, 0, 1) * (cell1.vol / ngrids) self.assertAlmostEqual(numpy.linalg.norm(ref[:, 0, 0] - dat[:, 0, 0]), 0, 7) self.assertAlmostEqual(numpy.linalg.norm(ref[:, 1, 1] - dat[:, 1, 1]), 0, 7) self.assertAlmostEqual( numpy.linalg.norm(ref[:, 2:, 2:] - dat[:, 2:, 2:]), 0, 7) self.assertAlmostEqual( numpy.linalg.norm(ref[:, 0, 2:] - dat[:, 0, 2:]), 0, 7) self.assertAlmostEqual( numpy.linalg.norm(ref[:, 2:, 0] - dat[:, 2:, 0]), 0, 7) idx = numpy.tril_indices(nao) ref = dat[:, idx[0], idx[1]] dat = ft_ao.ft_aopair(cell1, cell1.Gv, aosym='s2') self.assertAlmostEqual(abs(dat - ref).sum(), 0, 9)
def __init__(self, m=None, P=None, U=None, S=None, Pm=None, L=None): """Initialize a gaussian pdf given a valid combination of its parameters. Valid combinations are: m-P, m-U, m-S, Pm-P, Pm-U, Pm-S. :param m: mean :param P: precision :param U: upper triangular precision factor such that U'U = P :param S: covariance :param C : upper or lower triangular covariance factor, in any case S = C'C :param Pm: precision times mean such that P*m = Pm :param L: lower triangular covariance factor given as 1D array such that LL' = S """ if m is not None: m = np.asarray(m) self.m = m self.ndim = m.size if P is not None: P = np.asarray(P) L = np.linalg.cholesky(P) self.P = P self.C = np.linalg.inv(L) self.S = np.dot(self.C.T, self.C) self.Pm = np.dot(P, m) self.logdetP = 2.0 * np.sum(np.log(np.diagonal(L))) elif U is not None: U = np.asarray(U) self.P = np.dot(U.T, U) self.C = np.linalg.inv(U.T) self.S = np.dot(self.C.T, self.C) self.Pm = np.dot(self.P, m) self.logdetP = 2.0 * np.sum(np.log(np.diagonal(U))) elif L is not None: L = np.asarray(L) Lm = np.zeros((self.ndim, self.ndim)) idx_l = np.tril_indices(self.ndim, -1) idx_d = np.diag_indices(self.ndim) Lm[idx_l] = L[self.ndim:] Lm[idx_d] = L[0:self.ndim] self.C = Lm.T self.S = np.dot(self.C.T, self.C) self.P = np.linalg.inv(self.S) self.Pm = np.dot(self.P, m) self.logdetP = -2.0 * np.sum(np.log(np.diagonal(self.C))) elif S is not None: S = np.asarray(S) self.P = np.linalg.inv(S) self.C = np.linalg.cholesky(S).T self.S = S self.Pm = np.dot(self.P, m) self.logdetP = -2.0 * np.sum(np.log(np.diagonal(self.C))) else: raise ValueError("Precision information missing.") elif Pm is not None: Pm = np.asarray(Pm) self.Pm = Pm self.ndim = Pm.size if P is not None: P = np.asarray(P) L = np.linalg.cholesky(P) self.P = P self.C = np.linalg.inv(L) self.S = np.dot(self.C.T, self.C) self.m = np.linalg.solve(P, Pm) self.logdetP = 2.0 * np.sum(np.log(np.diagonal(L))) elif U is not None: U = np.asarray(U) self.P = np.dot(U.T, U) self.C = np.linalg.inv(U.T) self.S = np.dot(self.C.T, self.C) self.m = np.linalg.solve(self.P, Pm) self.logdetP = 2.0 * np.sum(np.log(np.diagonal(U))) elif S is not None: S = np.asarray(S) self.P = np.linalg.inv(S) self.C = np.linalg.cholesky(S).T self.S = S self.m = np.dot(S, Pm) self.logdetP = -2.0 * np.sum(np.log(np.diagonal(self.C))) else: raise ValueError("Precision information missing.") else: raise ValueError("Mean information missing.")
def s2ij_s1(symmetry, eri, norb): idx = numpy.tril_indices(norb) eri1 = numpy.empty((norb, norb, norb, norb)) eri1[idx] = eri.reshape(-1, norb, norb) eri1[idx[1], idx[0]] = eri.reshape(-1, norb, norb) return eri1
def get_topologies(self, symbols, saturate=False): ''' Return the possible topologies of a given chemical species. Parameters ---------- symbols : str Atomic symbols to construct the topologies from. saturate : bool Saturate the molecule with hydrogen based on the default.radicals set. Returns ------- molecules : list (N,) Gratoms objects with unique connectivity matrix attached. No 3D positions will be provided for these structures. ''' num, cnt = Molecule.get_atomic_numbers(symbols, True) # print(num, cnt) mcnt = cnt[num != 1] mnum = num[num != 1] if cnt[num == 1]: hcnt = cnt[num == 1][0] else: hcnt = 0 elements = np.repeat(mnum, mcnt) max_degree = defaults.get('radicals')[elements] n = mcnt.sum() hmax = int(max_degree.sum() - (n - 1) * 2) if hcnt > hmax: hcnt = hmax if saturate: hcnt = hmax if n == 1: atoms = Gratoms(elements, cell=[1, 1, 1]) hatoms = Molecule.hydrogenate(atoms, np.array([hcnt])) return [hatoms] elif n == 0: hatoms = Gratoms('H{}'.format(hcnt)) if hcnt == 2: hatoms.graph.add_edge(0, 1, bonds=1) return [hatoms] ln = np.arange(n).sum() il = np.tril_indices(n, -1) backbones, molecules = [], [] combos = combinations(np.arange(ln), n - 1) for c in combos: # Construct the connectivity matrix ltm = np.zeros(ln) ltm[np.atleast_2d(c)] = 1 connectivity = np.zeros((n, n)) connectivity[il] = ltm connectivity = np.maximum(connectivity, connectivity.T) degree = connectivity.sum(axis=0) # Not fully connected (subgraph) if np.any(degree == 0) or not \ is_connected(from_numpy_matrix(connectivity)): continue # Overbonded atoms. remaining_bonds = (max_degree - degree).astype(int) if np.any(remaining_bonds < 0): continue atoms = Gratoms(numbers=elements, edges=connectivity, cell=[1, 1, 1]) isomorph = False for G0 in backbones: if atoms.is_isomorph(G0): isomorph = True break if not isomorph: backbones += [atoms] # The backbone is saturated, do not enumerate if hcnt == hmax: hatoms = Molecule.hydrogenate(atoms, remaining_bonds) molecules += [hatoms] continue # Enumerate hydrogens across backbone for bins in self.bin_hydrogen(hcnt, n): if not np.all(bins <= remaining_bonds): continue hatoms = Molecule.hydrogenate(atoms, bins) isomorph = False for G0 in molecules: if hatoms.is_isomorph(G0): isomorph = True break if not isomorph: molecules += [hatoms] return molecules
return np.linalg.norm(A_lamb, 'fro') ###parameters muTot=200 mu_arr=np.logspace(-10,5.0,muTot) norm_arr_nonint=np.zeros(muTot) norm_arr_int=np.zeros(muTot) hz=5.00#for int Ham ### t_start_nonint=time.time() ###nonint H=Ham_nonint(L) wij, num_lamb_mat=output_gauge_potent(H,L) ###finding minimum and maximum wij index_lower = np.tril_indices(2**L,-1) wij_arr=wij[index_lower] wij_min_nonint= min(wij_arr) wij_max_nonint=max(wij_arr) print wij_min_nonint,wij_max_nonint ###running the loop for i in range(muTot): mu=mu_arr[i] A_lamb=gauge_potent_mu(wij, num_lamb_mat,mu) norm_arr_nonint[i]=norm(A_lamb) t_end_nonint=time.time() t_nonint_code=(t_end_nonint-t_start_nonint)/60 #######
def test_outcore(self): ftmp = tempfile.NamedTemporaryFile(dir=lib.param.TMPDIR) cderi0 = df.incore.cholesky_eri(mol) df.outcore.cholesky_eri(mol, ftmp.name) with h5py.File(ftmp.name, 'r') as feri: self.assertTrue(numpy.allclose(feri['j3c'], cderi0)) df.outcore.cholesky_eri(mol, ftmp.name, ioblk_size=.05) with h5py.File(ftmp.name, 'r') as feri: self.assertTrue(numpy.allclose(feri['j3c'], cderi0)) nao = mol.nao_nr() naux = cderi0.shape[0] df.outcore.general(mol, (numpy.eye(nao), ) * 2, ftmp.name, max_memory=.05, ioblk_size=.02) with h5py.File(ftmp.name, 'r') as feri: self.assertTrue(numpy.allclose(feri['eri_mo'], cderi0)) #### buf = numpy.zeros((naux, nao, nao)) idx = numpy.tril_indices(nao) buf[:, idx[0], idx[1]] = cderi0 buf[:, idx[1], idx[0]] = cderi0 cderi0 = buf df.outcore.cholesky_eri(mol, ftmp.name, aosym='s1', ioblk_size=.05) with h5py.File(ftmp.name, 'r') as feri: self.assertTrue( numpy.allclose(feri['j3c'], cderi0.reshape(naux, -1))) ftmp = tempfile.NamedTemporaryFile(dir=lib.param.TMPDIR) numpy.random.seed(1) co = numpy.random.random((nao, 4)) cv = numpy.random.random((nao, 25)) cderi0 = numpy.einsum('kpq,pi,qj->kij', cderi0, co, cv) df.outcore.general(mol, (co, cv), ftmp.name, ioblk_size=.05) with h5py.File(ftmp.name, 'r') as feri: self.assertTrue( numpy.allclose(feri['eri_mo'], cderi0.reshape(naux, -1))) cderi0 = df.incore.aux_e2(mol, auxmol, intor='int3c2e_ip1_sph', aosym='s1', comp=3).reshape(3, nao**2, -1) j2c = df.incore.fill_2c2e(mol, auxmol) low = scipy.linalg.cholesky(j2c, lower=True) cderi0 = [ scipy.linalg.solve_triangular(low, j3c.T, lower=True) for j3c in cderi0 ] nao = mol.nao_nr() df.outcore.general(mol, (numpy.eye(nao), ) * 2, ftmp.name, int3c='int3c2e_ip1_sph', aosym='s1', int2c='int2c2e_sph', comp=3, max_memory=.05, ioblk_size=.02) with h5py.File(ftmp.name, 'r') as feri: self.assertTrue(numpy.allclose(feri['eri_mo'], cderi0))
def test_silhouette_paper_example(): # Explicitly check per-sample results against Rousseeuw (1987) # Data from Table 1 lower = [ 5.58, 7.00, 6.50, 7.08, 7.00, 3.83, 4.83, 5.08, 8.17, 5.83, 2.17, 5.75, 6.67, 6.92, 4.92, 6.42, 5.00, 5.58, 6.00, 4.67, 6.42, 3.42, 5.50, 6.42, 6.42, 5.00, 3.92, 6.17, 2.50, 4.92, 6.25, 7.33, 4.50, 2.25, 6.33, 2.75, 6.08, 6.67, 4.25, 2.67, 6.00, 6.17, 6.17, 6.92, 6.17, 5.25, 6.83, 4.50, 3.75, 5.75, 5.42, 6.08, 5.83, 6.67, 3.67, 4.75, 3.00, 6.08, 6.67, 5.00, 5.58, 4.83, 6.17, 5.67, 6.50, 6.92, ] D = np.zeros((12, 12)) D[np.tril_indices(12, -1)] = lower D += D.T names = [ "BEL", "BRA", "CHI", "CUB", "EGY", "FRA", "IND", "ISR", "USA", "USS", "YUG", "ZAI", ] # Data from Figure 2 labels1 = [1, 1, 2, 2, 1, 1, 2, 1, 1, 2, 2, 1] expected1 = { "USA": 0.43, "BEL": 0.39, "FRA": 0.35, "ISR": 0.30, "BRA": 0.22, "EGY": 0.20, "ZAI": 0.19, "CUB": 0.40, "USS": 0.34, "CHI": 0.33, "YUG": 0.26, "IND": -0.04, } score1 = 0.28 # Data from Figure 3 labels2 = [1, 2, 3, 3, 1, 1, 2, 1, 1, 3, 3, 2] expected2 = { "USA": 0.47, "FRA": 0.44, "BEL": 0.42, "ISR": 0.37, "EGY": 0.02, "ZAI": 0.28, "BRA": 0.25, "IND": 0.17, "CUB": 0.48, "USS": 0.44, "YUG": 0.31, "CHI": 0.31, } score2 = 0.33 for labels, expected, score in [ (labels1, expected1, score1), (labels2, expected2, score2), ]: expected = [expected[name] for name in names] # we check to 2dp because that's what's in the paper pytest.approx( expected, silhouette_samples(D, np.array(labels), metric="precomputed"), abs=1e-2, ) pytest.approx( score, silhouette_score(D, np.array(labels), metric="precomputed"), abs=1e-2 )
def fcidump(wfn, fname='INTDUMP', oe_ints=None): """Save integrals to file in FCIDUMP format as defined in Comp. Phys. Commun. 54 75 (1989) Additional one-electron integrals, including orbital energies, can also be saved. This latter format can be used with the HANDE QMC code but is not standard. :returns: None :raises: ValidationError when SCF wavefunction is not RHF :type wfn: :py:class:`~psi4.core.Wavefunction` :param wfn: set of molecule, basis, orbitals from which to generate cube files :param fname: name of the integrals file, defaults to INTDUMP :param oe_ints: list of additional one-electron integrals to save to file. So far only EIGENVALUES is a valid option. :examples: >>> # [1] Save one- and two-electron integrals to standard FCIDUMP format >>> E, wfn = energy('scf', return_wfn=True) >>> fcidump(wfn) >>> # [2] Save orbital energies, one- and two-electron integrals. >>> E, wfn = energy('scf', return_wfn=True) >>> fcidump(wfn, oe_ints=['EIGENVALUES']) """ # Get some options reference = core.get_option('SCF', 'REFERENCE') ints_tolerance = core.get_global_option('INTS_TOLERANCE') # Some sanity checks if reference not in ['RHF', 'UHF']: raise ValidationError( 'FCIDUMP not implemented for {} references\n'.format(reference)) if oe_ints is None: oe_ints = [] molecule = wfn.molecule() docc = wfn.doccpi() frzcpi = wfn.frzcpi() frzvpi = wfn.frzvpi() active_docc = docc - frzcpi active_socc = wfn.soccpi() active_mopi = wfn.nmopi() - frzcpi - frzvpi nbf = active_mopi.sum() if wfn.same_a_b_orbs() else 2 * active_mopi.sum() nirrep = wfn.nirrep() nelectron = 2 * active_docc.sum() + active_socc.sum() irrep_map = _irrep_map(wfn) wfn_irrep = 0 for h, n_socc in enumerate(active_socc): if n_socc % 2 == 1: wfn_irrep ^= h core.print_out('Writing integrals in FCIDUMP format to ' + fname + '\n') # Generate FCIDUMP header header = '&FCI\n' header += 'NORB={:d},\n'.format(nbf) header += 'NELEC={:d},\n'.format(nelectron) header += 'MS2={:d},\n'.format(wfn.nalpha() - wfn.nbeta()) header += 'UHF=.{}.,\n'.format(not wfn.same_a_b_orbs()).upper() orbsym = '' for h in range(active_mopi.n()): for n in range(frzcpi[h], frzcpi[h] + active_mopi[h]): orbsym += '{:d},'.format(irrep_map[h]) if not wfn.same_a_b_orbs(): orbsym += '{:d},'.format(irrep_map[h]) header += 'ORBSYM={}\n'.format(orbsym) header += 'ISYM={:d},\n'.format(irrep_map[wfn_irrep]) header += '&END\n' with open(fname, 'w') as intdump: intdump.write(header) # Get an IntegralTransform object check_iwl_file_from_scf_type(core.get_global_option('SCF_TYPE'), wfn) spaces = [core.MOSpace.all()] trans_type = core.IntegralTransform.TransformationType.Restricted if not wfn.same_a_b_orbs(): trans_type = core.IntegralTransform.TransformationType.Unrestricted ints = core.IntegralTransform(wfn, spaces, trans_type) ints.transform_tei(core.MOSpace.all(), core.MOSpace.all(), core.MOSpace.all(), core.MOSpace.all()) core.print_out('Integral transformation complete!\n') DPD_info = { 'instance_id': ints.get_dpd_id(), 'alpha_MO': ints.DPD_ID('[A>=A]+'), 'beta_MO': 0 } if not wfn.same_a_b_orbs(): DPD_info['beta_MO'] = ints.DPD_ID("[a>=a]+") # Write TEI to fname in FCIDUMP format core.fcidump_tei_helper(nirrep, wfn.same_a_b_orbs(), DPD_info, ints_tolerance, fname) # Read-in OEI and write them to fname in FCIDUMP format # Indexing functions to translate from zero-based (C and Python) to # one-based (Fortran) mo_idx = lambda x: x + 1 alpha_mo_idx = lambda x: 2 * x + 1 beta_mo_idx = lambda x: 2 * (x + 1) with open(fname, 'a') as intdump: core.print_out('Writing frozen core operator in FCIDUMP format to ' + fname + '\n') if reference == 'RHF': PSIF_MO_FZC = 'MO-basis Frozen-Core Operator' moH = core.Matrix(PSIF_MO_FZC, wfn.nmopi(), wfn.nmopi()) moH.load(core.IO.shared_object(), psif.PSIF_OEI) mo_slice = core.Slice(frzcpi, active_mopi) MO_FZC = moH.get_block(mo_slice, mo_slice) offset = 0 for h, block in enumerate(MO_FZC.nph): il = np.tril_indices(block.shape[0]) for index, x in np.ndenumerate(block[il]): row = mo_idx(il[0][index] + offset) col = mo_idx(il[1][index] + offset) if (abs(x) > ints_tolerance): intdump.write( '{:29.20E} {:4d} {:4d} {:4d} {:4d}\n'.format( x, row, col, 0, 0)) offset += block.shape[0] # Additional one-electron integrals as requested in oe_ints # Orbital energies core.print_out('Writing orbital energies in FCIDUMP format to ' + fname + '\n') if 'EIGENVALUES' in oe_ints: eigs_dump = write_eigenvalues( wfn.epsilon_a().get_block(mo_slice).to_array(), mo_idx) intdump.write(eigs_dump) else: PSIF_MO_A_FZC = 'MO-basis Alpha Frozen-Core Oper' moH_A = core.Matrix(PSIF_MO_A_FZC, wfn.nmopi(), wfn.nmopi()) moH_A.load(core.IO.shared_object(), psif.PSIF_OEI) mo_slice = core.Slice(frzcpi, active_mopi) MO_FZC_A = moH_A.get_block(mo_slice, mo_slice) offset = 0 for h, block in enumerate(MO_FZC_A.nph): il = np.tril_indices(block.shape[0]) for index, x in np.ndenumerate(block[il]): row = alpha_mo_idx(il[0][index] + offset) col = alpha_mo_idx(il[1][index] + offset) if (abs(x) > ints_tolerance): intdump.write( '{:29.20E} {:4d} {:4d} {:4d} {:4d}\n'.format( x, row, col, 0, 0)) offset += block.shape[0] PSIF_MO_B_FZC = 'MO-basis Beta Frozen-Core Oper' moH_B = core.Matrix(PSIF_MO_B_FZC, wfn.nmopi(), wfn.nmopi()) moH_B.load(core.IO.shared_object(), psif.PSIF_OEI) mo_slice = core.Slice(frzcpi, active_mopi) MO_FZC_B = moH_B.get_block(mo_slice, mo_slice) offset = 0 for h, block in enumerate(MO_FZC_B.nph): il = np.tril_indices(block.shape[0]) for index, x in np.ndenumerate(block[il]): row = beta_mo_idx(il[0][index] + offset) col = beta_mo_idx(il[1][index] + offset) if (abs(x) > ints_tolerance): intdump.write( '{:29.20E} {:4d} {:4d} {:4d} {:4d}\n'.format( x, row, col, 0, 0)) offset += block.shape[0] # Additional one-electron integrals as requested in oe_ints # Orbital energies core.print_out('Writing orbital energies in FCIDUMP format to ' + fname + '\n') if 'EIGENVALUES' in oe_ints: alpha_eigs_dump = write_eigenvalues( wfn.epsilon_a().get_block(mo_slice).to_array(), alpha_mo_idx) beta_eigs_dump = write_eigenvalues( wfn.epsilon_b().get_block(mo_slice).to_array(), beta_mo_idx) intdump.write(alpha_eigs_dump + beta_eigs_dump) # Dipole integrals #core.print_out('Writing dipole moment OEI in FCIDUMP format to ' + fname + '\n') # Traceless quadrupole integrals #core.print_out('Writing traceless quadrupole moment OEI in FCIDUMP format to ' + fname + '\n') # Frozen core + nuclear repulsion energy core.print_out( 'Writing frozen core + nuclear repulsion energy in FCIDUMP format to ' + fname + '\n') e_fzc = ints.get_frozen_core_energy() e_nuc = molecule.nuclear_repulsion_energy( wfn.get_dipole_field_strength()) intdump.write('{: 29.20E} {:4d} {:4d} {:4d} {:4d}\n'.format( e_fzc + e_nuc, 0, 0, 0, 0)) core.print_out( 'Done generating {} with integrals in FCIDUMP format.\n'.format(fname))
def psthcorr(rec, nids=None, ssnids=None, ssseps=None, natexps=False, strange=None, plot=True): if nids == None: nids = sorted(rec.n) # use active neurons if ssnids == None: ssnids = nids # use nids as the superset nn = len(nids) nnss = len(ssnids) # note that using norm=True or norm='ntrials' doesn't seem to make a difference to the # results, probably doesn't matter for calculating corrs: midbins, psths, spikets = rec.psth(nids=nids, natexps=natexps, strange=strange, plot=False, binw=0.02, tres=0.005, norm=True) rho = np.corrcoef(psths) # defaults to bias=1 rho[np.diag_indices( nn)] = np.nan # nan the diagonal, which imshow plots as white ssrho = np.zeros((nnss, nnss)) # superset rho matrix ssrho.fill(np.nan) # init with nans # load up values into appropriate spots in superset rho matrix: for i in range(nn): for j in range(nn): ssi, ssj = ssnids.searchsorted([nids[i], nids[j]]) ssrho[ssi, ssj] = rho[i, j] if plot == False: return ssrho # plot superset rho matrix: figure(figsize=FIGSIZE) imshow(ssrho, vmin=-1, vmax=1, cmap='jet') # cmap='gray' is too bland ssnidticks = np.arange(0, nnss, 10) xticks(ssnidticks) yticks(ssnidticks) if SHOWCOLORBAR: colorbar() basetitle = rec.absname if strange != None: strange_sec = tuple(np.array(strange) / 1e6) # convert to sec for display basetitle += '_strange=(%.f, %.f)' % strange_sec gcfm().window.setWindowTitle(basetitle + '_rho_mat') tight_layout(pad=0.3) # plot rho histogram: lti = np.tril_indices( nnss, -1) # lower triangle (below diagonal) indices of ssrho ssrhol = ssrho[lti] notnanis = np.logical_not(np.isnan(ssrhol)) # indices of non-nan values fssrhol = ssrhol[notnanis] # ssrhol filtered out for nans fssrholmean = fssrhol.mean() t, p = ttest_1samp(fssrhol, 0) # 2-sided ttest relative to 0 print('mean=%g, t=%g, p=%g' % (fssrholmean, t, p)) if p < ALPHA0: pstring = '$p<%g$' % ceilsigfig(p) else: pstring = '$p>%g$' % floorsigfig(p) figure(figsize=FIGSIZE) rhobins = np.arange(RHOMIN, RHOMAX + 0.0333, 0.0333) # left edges + rightmost edge n = hist(fssrhol, bins=rhobins, color='k')[0] axvline(x=fssrholmean, c='r', ls='--') # draw vertical red line at mean fssrhol axvline(x=0, c='e', ls='--') # draw vertical grey line at x=0 xlim(xmin=RHOMIN, xmax=RHOMAX) ylim(ymax=n.max()) # effectively normalizes the histogram rhoticks = np.arange(-0.2, 1 + 0.2, 0.2) # excluding the final 1 xticks(rhoticks) yticks([n.max()]) # turn off y ticks to save space #yticks([0, n.max()]) text(0.98, 0.98, '$\mu$=%.2g\n%s' % (fssrholmean, pstring), color='k', transform=gca().transAxes, horizontalalignment='right', verticalalignment='top') gcfm().window.setWindowTitle(basetitle + '_rho_hist') tight_layout(pad=0.3) # plot rho vs separation: fssseps = ssseps[notnanis] # ssseps filtered out for nans figure(figsize=FIGSIZE) # scatter plot: pl.plot(fssseps, fssrhol, 'k.') # bin seps and plot mean rho in each bin: sepbins = np.arange(0, fssseps.max() + SEPBINW, SEPBINW) # left edges sepmeans, rhomeans, rhostds = scatterbin(fssseps, fssrhol, sepbins) #pl.plot(sepmeans, rhomeans, 'r.-', ms=10, lw=2) errorbar(sepmeans, rhomeans, yerr=rhostds, fmt='r.-', ms=10, lw=2, zorder=9999) xlim(xmin=0, xmax=SEPMAX) ylim(ymin=RHOMIN, ymax=RHOMAX) septicks = np.arange(0, fssseps.max() + 100, 500) xticks(septicks) yticks(rhoticks) gcfm().window.setWindowTitle(basetitle + '_rho_sep') tight_layout(pad=0.3) return ssrho
def as_array(self): """Return standard numpy array equivalent""" a = np.zeros((self.size, self.size)) a[np.tril_indices(self.size)] = self._elements a[np.triu_indices(self.size)] = a.T[np.triu_indices(self.size)] return a
def __init__(self): alleles = _AMINO_ACIDS num_alleles = len(alleles) root_distribution = [ 0.074, 0.052, 0.045, 0.054, 0.025, 0.034, 0.054, 0.074, 0.026, 0.068, 0.099, 0.058, 0.025, 0.047, 0.039, 0.057, 0.051, 0.013, 0.032, 0.073, ] relative_rates = [ 0.735790389698, 0.485391055466, 1.297446705134, 0.543161820899, 0.500964408555, 3.180100048216, 1.459995310470, 0.227826574209, 0.397358949897, 0.240836614802, 1.199705704602, 3.020833610064, 1.839216146992, 1.190945703396, 0.329801504630, 1.170949042800, 1.360574190420, 1.240488508640, 3.761625208368, 0.140748891814, 5.528919177928, 1.955883574960, 0.418763308518, 1.355872344485, 0.798473248968, 0.418203192284, 0.609846305383, 0.423579992176, 0.716241444998, 1.456141166336, 2.414501434208, 0.778142664022, 0.354058109831, 2.435341131140, 1.626891056982, 0.539859124954, 0.605899003687, 0.232036445142, 0.283017326278, 0.418555732462, 0.774894022794, 0.236202451204, 0.186848046932, 0.189296292376, 0.252718447885, 0.800016530518, 0.622711669692, 0.211888159615, 0.218131577594, 0.831842640142, 0.580737093181, 0.372625175087, 0.217721159236, 0.348072209797, 3.890963773304, 1.295201266783, 5.411115141489, 1.593137043457, 1.032447924952, 0.285078800906, 3.945277674515, 2.802427151679, 0.752042440303, 1.022507035889, 0.406193586642, 0.445570274261, 1.253758266664, 0.983692987457, 0.648441278787, 0.222621897958, 0.767688823480, 2.494896077113, 0.555415397470, 0.459436173579, 0.984311525359, 3.364797763104, 6.030559379572, 1.073061184332, 0.492964679748, 0.371644693209, 0.354861249223, 0.281730694207, 0.441337471187, 0.144356959750, 0.291409084165, 0.368166464453, 0.714533703928, 1.517359325954, 2.064839703237, 0.266924750511, 1.773855168830, 1.173275900924, 0.448133661718, 0.494887043702, 0.730628272998, 0.356008498769, 0.858570575674, 0.926563934846, 0.504086599527, 0.527007339151, 0.388355409206, 0.374555687471, 1.047383450722, 0.454123625103, 0.233597909629, 4.325092687057, 1.122783104210, 2.904101656456, 1.582754142065, 1.197188415094, 1.934870924596, 1.769893238937, 1.509326253224, 1.117029762910, 0.357544412460, 0.352969184527, 1.752165917819, 0.918723415746, 0.540027644824, 1.169129577716, 1.729178019485, 0.914665954563, 1.898173634533, 0.934187509431, 1.119831358516, 1.277480294596, 1.071097236007, 0.641436011405, 0.585407090225, 1.179091197260, 0.915259857694, 1.303875200799, 1.488548053722, 0.488206118793, 1.005451683149, 5.151556292270, 0.465839367725, 0.426382310122, 0.191482046247, 0.145345046279, 0.527664418872, 0.758653808642, 0.407635648938, 0.508358924638, 0.301248600780, 0.341985787540, 0.691474634600, 0.332243040634, 0.888101098152, 2.074324893497, 0.252214830027, 0.387925622098, 0.513128126891, 0.718206697586, 0.720517441216, 0.538222519037, 0.261422208965, 0.470237733696, 0.958989742850, 0.596719300346, 0.308055737035, 4.218953969389, 0.674617093228, 0.811245856323, 0.717993486900, 0.951682162246, 6.747260430801, 0.369405319355, 0.796751520761, 0.801010243199, 4.054419006558, 2.187774522005, 0.438388343772, 0.312858797993, 0.258129289418, 1.116352478606, 0.530785790125, 0.524253846338, 0.253340790190, 0.201555971750, 8.311839405458, 2.231405688913, 0.498138475304, 2.575850755315, 0.838119610178, 0.496908410676, 0.561925457442, 2.253074051176, 0.266508731426, 1.000000000000, ] transition_matrix = np.zeros((num_alleles, num_alleles)) tril = np.tril_indices(num_alleles, k=-1) transition_matrix[tril] = relative_rates transition_matrix += np.tril(transition_matrix).T transition_matrix *= root_distribution row_sums = transition_matrix.sum(axis=1) transition_matrix = transition_matrix / max(row_sums) row_sums = transition_matrix.sum(axis=1, dtype="float64") np.fill_diagonal(transition_matrix, 1.0 - row_sums) super().__init__(alleles, root_distribution, transition_matrix)
def psthcorrdiff(rhos, seps, basetitle): """Plot difference of a pair of rho matrices (rhos[0] - rhos[1]). seps is the corresponding distance matrix""" assert len(rhos) == 2 # calc rho difference matrix: rhod = rhos[0] - rhos[1] assert rhod.shape[0] == rhod.shape[1] # square nn = rhod.shape[0] # plot rho diff matrix: figure(figsize=FIGSIZE) imshow(rhod, vmin=-1, vmax=1, cmap='jet') # cmap='gray' is too bland ssnidticks = np.arange(0, nn, 10) xticks(ssnidticks) yticks(ssnidticks) if SHOWCOLORBAR: colorbar() gcfm().window.setWindowTitle(basetitle + '_rhod_mat') tight_layout(pad=0.3) # plot rho difference histogram: lti = np.tril_indices(nn, -1) # lower triangle (below diagonal) indices rhol = rhod[lti] notnanis = np.logical_not(np.isnan(rhol)) # indices of non-nan values frhol = rhol[notnanis] # rhol filtered out for nans frholmean = frhol.mean() t, p = ttest_1samp(frhol, 0) # 2-sided ttest relative to 0 print('mean=%g, t=%g, p=%g' % (frholmean, t, p)) if p < ALPHA0: pstring = '$p<%g$' % ceilsigfig(p) else: pstring = '$p>%g$' % floorsigfig(p) figure(figsize=FIGSIZE) rhobins = np.arange(RHODIFFMIN, RHODIFFMAX + 0.0333, 0.0333) # left edges + rightmost edge n = hist(frhol, bins=rhobins, color='k')[0] axvline(x=frholmean, c='r', ls='--') # draw vertical red line at mean frhol axvline(x=0, c='e', ls='--') # draw vertical grey line at x=0 xlim(xmin=RHODIFFMIN, xmax=RHODIFFMAX) ylim(ymax=n.max()) # effectively normalizes the histogram rhoticks = np.arange(-0.6, 0.6 + 0.2, 0.2) xticks(rhoticks) yticks([n.max()]) # turn off y ticks to save space #yticks([0, n.max()]) text(0.98, 0.98, '$\mu$=%.2g\n%s' % (frholmean, pstring), color='k', transform=gca().transAxes, horizontalalignment='right', verticalalignment='top') gcfm().window.setWindowTitle(basetitle + '_rhod_hist') tight_layout(pad=0.3) # plot rho difference vs separation: fseps = seps[notnanis] # seps filtered out for nans figure(figsize=FIGSIZE) # scatter plot: pl.plot(fseps, frhol, 'k.') # bin seps and plot mean rho in each bin: sortis = np.argsort(fseps) seps = fseps[sortis] rhos = frhol[sortis] sepbins = np.arange(0, seps.max() + SEPBINW, SEPBINW) # left edges sepis = seps.searchsorted(sepbins) sepmeans, rhomeans, rhostds = [], [], [] for sepi0, sepi1 in zip(sepis[:-1], sepis[1:]): # iterate over sepbins sepmeans.append( seps[sepi0:sepi1].mean()) # mean sep of all points in this sepbin rhoslice = rhos[sepi0:sepi1] # rhos in this sepbin rhomeans.append( rhoslice.mean()) # mean rho of all points in this sepbin rhostds.append(rhoslice.std()) # std of rho in this sepbin #pl.plot(sepmeans, rhomeans, 'r.-', ms=10, lw=2) errorbar(sepmeans, rhomeans, yerr=rhostds, fmt='r.-', ms=10, lw=2, zorder=9999) xlim(xmin=0, xmax=SEPMAX) ylim(ymin=RHODIFFMIN, ymax=RHODIFFMAX) septicks = np.arange(0, seps.max() + 100, 500) xticks(septicks) yticks(rhoticks) gcfm().window.setWindowTitle(basetitle + '_rhod_sep') tight_layout(pad=0.3)
def tril_indices(*args, **kwargs): return tuple(map(torch.from_numpy, _np.tril_indices(*args, **kwargs)))
def to_fcivec(cisdvec, norb, nelec, frozen=None): '''Convert CISD coefficients to FCI coefficients''' if isinstance(nelec, (int, numpy.number)): nelecb = nelec // 2 neleca = nelec - nelecb else: neleca, nelecb = nelec assert (neleca == nelecb) frozen_mask = numpy.zeros(norb, dtype=bool) if frozen is None: nfroz = 0 elif isinstance(frozen, (int, numpy.integer)): nfroz = frozen frozen_mask[:frozen] = True else: nfroz = len(frozen) frozen_mask[frozen] = True nocc = numpy.count_nonzero(~frozen_mask[:neleca]) nmo = norb - nfroz nvir = nmo - nocc c0, c1, c2 = cisdvec_to_amplitudes(cisdvec, nmo, nocc) t1addr, t1sign = tn_addrs_signs(nmo, nocc, 1) na = cistring.num_strings(nmo, nocc) fcivec = numpy.zeros((na, na)) fcivec[0, 0] = c0 fcivec[0, t1addr] = fcivec[t1addr, 0] = c1.ravel() * t1sign c2ab = c2.transpose(0, 2, 1, 3).reshape(nocc * nvir, -1) c2ab = numpy.einsum('i,j,ij->ij', t1sign, t1sign, c2ab) fcivec[t1addr[:, None], t1addr] = c2ab if nocc > 1 and nvir > 1: c2aa = c2 - c2.transpose(1, 0, 2, 3) ooidx = numpy.tril_indices(nocc, -1) vvidx = numpy.tril_indices(nvir, -1) c2aa = c2aa[ooidx][:, vvidx[0], vvidx[1]] t2addr, t2sign = tn_addrs_signs(nmo, nocc, 2) fcivec[0, t2addr] = fcivec[t2addr, 0] = c2aa.ravel() * t2sign if nfroz == 0: return fcivec assert (norb < 63) strs = cistring.gen_strings4orblist(range(norb), neleca) na = len(strs) count = numpy.zeros(na, dtype=int) parity = numpy.zeros(na, dtype=bool) core_mask = numpy.ones(na, dtype=bool) # During the loop, count saves the number of occupied orbitals that # lower (with small orbital ID) than the present orbital i. # Moving all the frozen orbitals to the beginning of the orbital list # (before the occupied orbitals) leads to parity odd (= True, with # negative sign) or even (= False, with positive sign). for i in range(norb): if frozen_mask[i]: if i < neleca: # frozen occupied orbital should be occupied core_mask &= (strs & (1 << i)) != 0 parity ^= (count & 1) == 1 else: # frozen virtual orbital should not be occupied. # parity is not needed since it's unoccupied core_mask &= (strs & (1 << i)) == 0 else: count += (strs & (1 << i)) != 0 sub_strs = strs[core_mask & (count == nocc)] addrs = cistring.strs2addr(norb, neleca, sub_strs) fcivec1 = numpy.zeros((na, na)) fcivec1[addrs[:, None], addrs] = fcivec fcivec1[parity, :] *= -1 fcivec1[:, parity] *= -1 return fcivec1
def trans_e1_outcore(mol, mo, ncore, ncas, erifile, max_memory=None, level=1, verbose=logger.WARN): time0 = (time.clock(), time.time()) if isinstance(verbose, logger.Logger): log = verbose else: log = logger.Logger(mol.stdout, verbose) log.debug1('trans_e1_outcore level %d max_memory %d', level, max_memory) nao, nmo = mo.shape nao_pair = nao * (nao + 1) // 2 nocc = ncore + ncas _tmpfile1 = tempfile.NamedTemporaryFile(dir=lib.param.TMPDIR) faapp_buf = h5py.File(_tmpfile1.name) feri = h5py.File(erifile, 'w') mo_c = numpy.asarray(mo, order='C') mo = numpy.asarray(mo, order='F') pashape = (0, nmo, ncore, nocc) papa_buf = numpy.zeros((nao, ncas, nmo * ncas)) j_pc = numpy.zeros((nmo, ncore)) k_pc = numpy.zeros((nmo, ncore)) mem_words = int(max(2000, max_memory - papa_buf.nbytes / 1e6) * 1e6 / 8) aobuflen = mem_words // (nao_pair + nocc * nmo) + 1 ao_loc = numpy.array(mol.ao_loc_nr(), dtype=numpy.int32) shranges = outcore.guess_shell_ranges(mol, True, aobuflen, None, ao_loc) intor = mol._add_suffix('int2e') ao2mopt = _ao2mo.AO2MOpt(mol, intor, 'CVHFnr_schwarz_cond', 'CVHFsetnr_direct_scf') nstep = len(shranges) paapp = 0 maxbuflen = max([x[2] for x in shranges]) log.debug('mem_words %.8g MB, maxbuflen = %d', mem_words * 8 / 1e6, maxbuflen) bufs1 = numpy.empty((maxbuflen, nao_pair)) bufs2 = numpy.empty((maxbuflen, nmo * ncas)) if level == 1: bufs3 = numpy.empty((maxbuflen, nao * ncore)) log.debug('mem cache %.8g MB', (bufs1.nbytes + bufs2.nbytes + bufs3.nbytes) / 1e6) else: log.debug('mem cache %.8g MB', (bufs1.nbytes + bufs2.nbytes) / 1e6) ti0 = log.timer('Initializing trans_e1_outcore', *time0) # fmmm, ftrans, fdrv for level 1 fmmm = libmcscf.AO2MOmmm_ket_nr_s2 ftrans = libmcscf.AO2MOtranse1_nr_s4 fdrv = libmcscf.AO2MOnr_e2_drv for istep, sh_range in enumerate(shranges): log.debug('[%d/%d], AO [%d:%d], len(buf) = %d', istep + 1, nstep, *sh_range) buf = bufs1[:sh_range[2]] _ao2mo.nr_e1fill(intor, sh_range, mol._atm, mol._bas, mol._env, 's4', 1, ao2mopt, buf) if log.verbose >= logger.DEBUG1: ti1 = log.timer('AO integrals buffer', *ti0) bufpa = bufs2[:sh_range[2]] _ao2mo.nr_e1(buf, mo, pashape, 's4', 's1', out=bufpa) # jc_pp, kc_pp if level == 1: # ppaa, papa and vhf, jcp, kcp if log.verbose >= logger.DEBUG1: ti1 = log.timer('buffer-pa', *ti1) buf1 = bufs3[:sh_range[2]] fdrv(ftrans, fmmm, buf1.ctypes.data_as(ctypes.c_void_p), buf.ctypes.data_as(ctypes.c_void_p), mo.ctypes.data_as(ctypes.c_void_p), ctypes.c_int(sh_range[2]), ctypes.c_int(nao), (ctypes.c_int * 4)(0, nao, 0, ncore), ctypes.POINTER(ctypes.c_void_p)(), ctypes.c_int(0)) p0 = 0 for ij in range(sh_range[0], sh_range[1]): i, j = lib.index_tril_to_pair(ij) i0 = ao_loc[i] j0 = ao_loc[j] i1 = ao_loc[i + 1] j1 = ao_loc[j + 1] di = i1 - i0 dj = j1 - j0 if i == j: dij = di * (di + 1) // 2 buf = numpy.empty((di, di, nao * ncore)) idx = numpy.tril_indices(di) buf[idx] = buf1[p0:p0 + dij] buf[idx[1], idx[0]] = buf1[p0:p0 + dij] buf = buf.reshape(di, di, nao, ncore) mo1 = mo_c[i0:i1] tmp = numpy.einsum('uvpc,pc->uvc', buf, mo[:, :ncore]) tmp = lib.dot(mo1.T, tmp.reshape(di, -1)) j_pc += numpy.einsum('vp,pvc->pc', mo1, tmp.reshape(nmo, di, ncore)) tmp = numpy.einsum('uvpc,uc->vcp', buf, mo1[:, :ncore]) tmp = lib.dot(tmp.reshape(-1, nmo), mo).reshape(di, ncore, nmo) k_pc += numpy.einsum('vp,vcp->pc', mo1, tmp) else: dij = di * dj buf = buf1[p0:p0 + dij].reshape(di, dj, nao, ncore) mo1 = mo_c[i0:i1] mo2 = mo_c[j0:j1] tmp = numpy.einsum('uvpc,pc->uvc', buf, mo[:, :ncore]) tmp = lib.dot(mo1.T, tmp.reshape(di, -1)) j_pc += numpy.einsum('vp,pvc->pc', mo2, tmp.reshape(nmo, dj, ncore)) * 2 tmp = numpy.einsum('uvpc,uc->vcp', buf, mo1[:, :ncore]) tmp = lib.dot(tmp.reshape(-1, nmo), mo).reshape(dj, ncore, nmo) k_pc += numpy.einsum('vp,vcp->pc', mo2, tmp) tmp = numpy.einsum('uvpc,vc->ucp', buf, mo2[:, :ncore]) tmp = lib.dot(tmp.reshape(-1, nmo), mo).reshape(di, ncore, nmo) k_pc += numpy.einsum('up,ucp->pc', mo1, tmp) p0 += dij if log.verbose >= logger.DEBUG1: ti1 = log.timer('j_cp and k_cp', *ti1) if log.verbose >= logger.DEBUG1: ti1 = log.timer('half transformation of the buffer', *ti1) # ppaa, papa faapp_buf[str(istep)] = \ bufpa.reshape(sh_range[2],nmo,ncas)[:,ncore:nocc].reshape(-1,ncas**2).T p0 = 0 for ij in range(sh_range[0], sh_range[1]): i, j = lib.index_tril_to_pair(ij) i0 = ao_loc[i] j0 = ao_loc[j] i1 = ao_loc[i + 1] j1 = ao_loc[j + 1] di = i1 - i0 dj = j1 - j0 if i == j: dij = di * (di + 1) // 2 buf1 = numpy.empty((di, di, nmo * ncas)) idx = numpy.tril_indices(di) buf1[idx] = bufpa[p0:p0 + dij] buf1[idx[1], idx[0]] = bufpa[p0:p0 + dij] else: dij = di * dj buf1 = bufpa[p0:p0 + dij].reshape(di, dj, -1) mo1 = mo[j0:j1, ncore:nocc].copy() for i in range(di): lib.dot(mo1.T, buf1[i], 1, papa_buf[i0 + i], 1) mo1 = mo[i0:i1, ncore:nocc].copy() buf1 = lib.dot(mo1.T, buf1.reshape(di, -1)) papa_buf[j0:j1] += buf1.reshape(ncas, dj, -1).transpose(1, 0, 2) p0 += dij if log.verbose >= logger.DEBUG1: ti1 = log.timer('ppaa and papa buffer', *ti1) ti0 = log.timer('gen AO/transform MO [%d/%d]' % (istep + 1, nstep), *ti0) buf = buf1 = bufpa = None bufs1 = bufs2 = bufs3 = None time1 = log.timer('mc_ao2mo pass 1', *time0) log.debug1('Half transformation done. Current memory %d', lib.current_memory()[0]) nblk = int( max(8, min(nmo, (max_memory * 1e6 / 8 - papa_buf.size) / (ncas**2 * nmo)))) log.debug1('nblk for papa = %d', nblk) dset = feri.create_dataset('papa', (nmo, ncas, nmo, ncas), 'f8') for i0, i1 in prange(0, nmo, nblk): tmp = lib.dot(mo[:, i0:i1].T, papa_buf.reshape(nao, -1)) dset[i0:i1] = tmp.reshape(i1 - i0, ncas, nmo, ncas) papa_buf = tmp = None time1 = log.timer('papa pass 2', *time1) tmp = numpy.empty((ncas**2, nao_pair)) p0 = 0 for istep, sh_range in enumerate(shranges): tmp[:, p0:p0 + sh_range[2]] = faapp_buf[str(istep)] p0 += sh_range[2] nblk = int( max(8, min(nmo, (max_memory * 1e6 / 8 - tmp.size) / (ncas**2 * nmo) - 1))) log.debug1('nblk for ppaa = %d', nblk) dset = feri.create_dataset('ppaa', (nmo, nmo, ncas, ncas), 'f8') for i0, i1 in prange(0, nmo, nblk): tmp1 = _ao2mo.nr_e2(tmp, mo, (i0, i1, 0, nmo), 's4', 's1', ao_loc=ao_loc) tmp1 = tmp1.reshape(ncas, ncas, i1 - i0, nmo) for j in range(i1 - i0): dset[i0 + j] = tmp1[:, :, j].transpose(2, 0, 1) tmp = tmp1 = None time1 = log.timer('ppaa pass 2', *time1) faapp_buf.close() feri.close() _tmpfile1 = None time0 = log.timer('mc_ao2mo', *time0) return j_pc, k_pc
cbar=False, square=True) plt.title('{} Canonical Angle'.format(['First', 'Second', 'Third'][ii])) plt.xticks(unique_idx, whisker_unique, rotation=45) plt.yticks(unique_idx, whisker_unique, rotation=0) plt.tight_layout() plt.savefig(os.path.join(p_load, 'canonical_angles_between_neuron_weights.pdf')) # =========================================== # plot a histogram of the canonical angles between the weights # =========================================== wd = figsize[0] / 1.5 ht = wd f = plt.figure(figsize=(wd, ht)) x, y = np.tril_indices(canonical_angles.shape[0], -1) first = canonical_angles[x, y, 0] second = canonical_angles[x, y, 1] third = canonical_angles[x, y, 2] sns.distplot(first, kde=False) sns.distplot(second, kde=False) sns.distplot(third, kde=False) sns.despine() plt.title('Values of angles between\nsubspaces covered by all neurons') plt.ylabel('# of pairwise comparisons') plt.xlabel('$cos(\\theta)$') plt.legend( ['{} Canonical Angle'.format(x) for x in ['First', 'Second', 'Third']]) plt.tight_layout() plt.savefig(
def initialize(self, model): """ Called on the first call to update `ilabels` is a list of n_i x n_i matrices containing integer labels that correspond to specific correlation parameters. Two elements of ilabels[i] with the same label share identical variance components. `designx` is a matrix, with each row containing dummy variables indicating which variance components are associated with the corresponding element of QY. """ super(Nested, self).initialize(model) if self.model.weights is not None: warnings.warn( "weights not implemented for nested cov_struct, " "using unweighted covariance estimate", NotImplementedWarning) # A bit of processing of the nest data id_matrix = np.asarray(self.model.dep_data) if id_matrix.ndim == 1: id_matrix = id_matrix[:, None] self.id_matrix = id_matrix endog = self.model.endog_li designx, ilabels = [], [] # The number of layers of nesting n_nest = self.id_matrix.shape[1] for i in range(self.model.num_group): ngrp = len(endog[i]) glab = self.model.group_labels[i] rix = self.model.group_indices[glab] # Determine the number of common variance components # shared by each pair of observations. ix1, ix2 = np.tril_indices(ngrp, -1) ncm = (self.id_matrix[rix[ix1], :] == self.id_matrix[rix[ix2], :] ).sum(1) # This is used to construct the working correlation # matrix. ilabel = np.zeros((ngrp, ngrp), dtype=np.int32) ilabel[(ix1, ix2)] = ncm + 1 ilabel[(ix2, ix1)] = ncm + 1 ilabels.append(ilabel) # This is used to estimate the variance components. dsx = np.zeros((len(ix1), n_nest + 1), dtype=np.float64) dsx[:, 0] = 1 for k in np.unique(ncm): ii = np.flatnonzero(ncm == k) dsx[ii, 1:k + 1] = 1 designx.append(dsx) self.designx = np.concatenate(designx, axis=0) self.ilabels = ilabels svd = np.linalg.svd(self.designx, 0) self.designx_u = svd[0] self.designx_s = svd[1] self.designx_v = svd[2].T
def quadratic_terms(x): n = len(x) x = x[np.newaxis] # turn x from 1D to a 2D array temp = x * x.T - (1 - 1 / np.sqrt(2)) * np.diag(pow(x, 2)[0]) return temp[np.tril_indices(n)]
def _get_transformed_inputs(self, features: np.ndarray, edges: np.ndarray, invalid_prob: float = 1e-9) -> GraphData: assert edges.shape[0] == edges.shape[1], '{} != {}'.format( edges.shape[0], edges.shape[1]) assert edges.shape[0] == self._nodes, '{} != {}'.format( edges.shape[0], self._nodes) assert features.shape[0] == self._nodes, '{} != {}'.format( features.shape[0], self._nodes) assert edges.max() <= self._edge_types, '{} > {}'.format( edges.max(), self._edge_types) assert edges.min() == 0, '{} != 0'.format(edges.min()) nn = self._nodes # lower triangle idx_lower = np.tril_indices(nn) row, col = idx_lower assert row.shape[0] == self._n_adj_flat # degree[i]: degree of nodes before adding i-th edge degree = [] for i, (x, y) in enumerate(zip(row, col)): weight = edges[x, y] if weight == 0: continue if len(degree) == 0: nc = np.zeros(nn) else: nc = degree[-1].copy() nc[x] += weight nc[y] += weight degree.append(nc) # weight_indicators[i]: binary mask of allowed edge types before # adding i-th edge vv = self._valence[features.argmax(axis=-1)] indicator = np.ones((nn, self._edge_types), dtype=np.float32) for i, left in enumerate(vv.astype(int)): indicator[i, left:] = 0 weight_indicators = [indicator] for deg in degree[: -1]: # we don't need valence _after_ adding the last edge indicator = weight_indicators[-1].copy() d = (vv - deg).astype(int) for i, left in enumerate(d): indicator[i, left:] = 0 weight_indicators.append(indicator) num_edges = len(weight_indicators) assert np.count_nonzero(edges[idx_lower]) == num_edges # shape = (num_edges, num_nodes, num_edge_types) weight_indicators = np.asarray(weight_indicators, dtype=np.float32) # zero-padding if num_edges < max_edges null_edges = self._max_edges - num_edges if null_edges > 0: null_ind = np.zeros((null_edges, edges.shape[0], self._edge_types), dtype=np.float32) weight_indicators = np.row_stack((weight_indicators, null_ind)) # mask zero values weight_indicators = np.log((1.0 - weight_indicators) * invalid_prob + weight_indicators) adj_one_hot = np.zeros((self._edge_types, nn, nn), dtype=np.int32) # lower triangle of one-hot encoded edges matrix weight_bin = np.zeros([self._n_adj_flat, self._edge_types], dtype=np.int32) # indices of true edges (wrt flat lower triangular matrix) coord = np.zeros(self._max_edges, dtype=np.int32) # edge list of true edges (with row, col index) edge_list = np.zeros((self._max_edges, 2), dtype=np.int32) cur_edge = 0 for k, (i, j) in enumerate(zip(row, col)): if edges[i, j] > 0: ti = edges[i, j] - 1 adj_one_hot[ti, i, j] = 1 adj_one_hot[ti, j, i] = 1 weight_bin[k, ti] = 1 coord[cur_edge] = k edge_list[cur_edge, :] = i, j cur_edge += 1 assert cur_edge == num_edges adj = np.zeros(edges.shape, dtype=np.int32) adj[edges.nonzero()] = 1 adj_lower = adj[idx_lower][:, np.newaxis] return GraphData(adj_one_hot, adj_lower, weight_bin, features, weight_indicators, coord, edge_list, num_edges)
def __init__(self): alleles = _AMINO_ACIDS num_alleles = len(alleles) root_distribution = [ 0.087127, 0.040904, 0.040432, 0.046872, 0.033474, 0.038255, 0.049530, 0.088612, 0.033619, 0.036886, 0.085357, 0.080481, 0.014753, 0.039772, 0.050680, 0.069577, 0.058542, 0.010494, 0.029916, 0.064717, ] relative_rates = [ 0.267828, 0.984474, 0.327059, 1.199805, 0.000000, 8.931515, 0.360016, 0.232374, 0.000000, 0.000000, 0.887753, 2.439939, 1.028509, 1.348551, 0.000000, 1.961167, 0.000000, 1.493409, 11.388659, 0.000000, 7.086022, 2.386111, 0.087791, 1.385352, 1.240981, 0.107278, 0.281581, 0.811907, 0.228116, 2.383148, 5.290024, 0.868241, 0.282729, 6.011613, 0.439469, 0.106802, 0.653416, 0.632629, 0.768024, 0.239248, 0.438074, 0.180393, 0.609526, 0.000000, 0.076981, 0.406431, 0.154924, 0.341113, 0.000000, 0.000000, 0.730772, 0.112880, 0.071514, 0.443504, 2.556685, 0.258635, 4.610124, 3.148371, 0.716913, 0.000000, 1.519078, 0.830078, 0.267683, 0.270475, 0.460857, 0.180629, 0.717840, 0.896321, 0.000000, 0.000000, 0.000000, 1.127499, 0.304803, 0.170372, 0.000000, 3.332732, 5.230115, 2.411739, 0.183641, 0.136906, 0.138503, 0.000000, 0.000000, 0.000000, 0.000000, 0.153478, 0.475927, 1.951951, 1.565160, 0.000000, 0.921860, 2.485920, 1.028313, 0.419244, 0.133940, 0.187550, 1.526188, 0.507003, 0.347153, 0.933709, 0.119152, 0.316258, 0.335419, 0.170205, 0.110506, 4.051870, 1.531590, 4.885892, 0.956097, 1.598356, 0.561828, 0.793999, 2.322243, 0.353643, 0.247955, 0.171432, 0.954557, 0.619951, 0.459901, 2.427202, 3.680365, 0.265745, 2.271697, 0.660930, 0.162366, 0.525651, 0.340156, 0.306662, 0.226333, 1.900739, 0.331090, 1.350599, 1.031534, 0.136655, 0.782857, 5.436674, 0.000000, 2.001375, 0.224968, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.270564, 0.000000, 0.461776, 0.000000, 0.000000, 0.762354, 0.000000, 0.740819, 0.000000, 0.244139, 0.078012, 0.946940, 0.000000, 0.953164, 0.000000, 0.214717, 0.000000, 1.265400, 0.374834, 0.286572, 0.132142, 0.000000, 6.952629, 0.000000, 0.336289, 0.417839, 0.608070, 2.059564, 0.240368, 0.158067, 0.178316, 0.484678, 0.346983, 0.367250, 0.538165, 0.438715, 8.810038, 1.745156, 0.103850, 2.565955, 0.123606, 0.485026, 0.303836, 1.561997, 0.000000, 0.279379, ] transition_matrix = np.zeros((num_alleles, num_alleles)) tril = np.tril_indices(num_alleles, k=-1) transition_matrix[tril] = relative_rates transition_matrix += np.tril(transition_matrix).T transition_matrix *= root_distribution row_sums = transition_matrix.sum(axis=1) transition_matrix = transition_matrix / max(row_sums) row_sums = transition_matrix.sum(axis=1, dtype="float64") np.fill_diagonal(transition_matrix, 1.0 - row_sums) super().__init__(alleles, root_distribution, transition_matrix)