def get_means_and_cov(num_vars, iid='clustered', cov_param=None): means = np.zeros(num_vars) inv_sum = num_vars if iid == 'clustered': eigs = [] while len(eigs) < num_vars - 1: if inv_sum <= 1e-2: eig = 0 else: eig = np.random.uniform(0, inv_sum) eigs.append(eig) inv_sum -= eig eigs.append(num_vars - np.sum(eigs)) covs = random_correlation.rvs(eigs) elif iid == 'spike': covs = random_correlation.rvs( np.ones(num_vars)) # basically identity with some noise covs = covs + 0.5 * np.ones(covs.shape) elif iid == 'decay': eigs = np.array([1 / ((i + 1)**cov_param) for i in range(num_vars)]) eigs = eigs * num_vars / eigs.sum() covs = random_correlation.rvs(eigs) elif iid == 'mult_decay': eigs = np.array([cov_param**(i) for i in range(num_vars)]) eigs = eigs * num_vars / eigs.sum() covs = random_correlation.rvs(eigs) elif iid == 'lin_decay': eigs = np.array([1 / (cov_param**i) for i in range(num_vars)]) eigs = eigs * num_vars / eigs.sum() covs = random_correlation.rvs(eigs) return means, covs
def test_reproducibility(self): np.random.seed(514) eigs = (.5, .8, 1.2, 1.5) x = random_correlation.rvs((.5, .8, 1.2, 1.5)) x2 = random_correlation.rvs((.5, .8, 1.2, 1.5), random_state=514) expected = np.array([[1., -0.20387311, 0.18366501, -0.04953711], [-0.20387311, 1., -0.24351129, 0.06703474], [0.18366501, -0.24351129, 1., 0.38530195], [-0.04953711, 0.06703474, 0.38530195, 1.]]) assert_array_almost_equal(x, expected) assert_array_almost_equal(x2, expected)
def test_definition(self): # Test the defintion of a correlation matrix in several dimensions: # # 1. Det is product of eigenvalues (and positive by construction # in examples) # 2. 1's on diagonal # 3. Matrix is symmetric def norm(i, e): return i*e/sum(e) np.random.seed(123) eigs = [norm(i, np.random.uniform(size=i)) for i in range(2, 6)] eigs.append([4,0,0,0]) ones = [[1.]*len(e) for e in eigs] xs = [random_correlation.rvs(e) for e in eigs] # Test that determinants are products of eigenvalues # These are positive by construction # Could also test that the eigenvalues themselves are correct, # but this seems sufficient. dets = [np.fabs(np.linalg.det(x)) for x in xs] dets_known = [np.prod(e) for e in eigs] assert_allclose(dets, dets_known, rtol=1e-13, atol=1e-13) # Test for 1's on the diagonal diags = [np.diag(x) for x in xs] for a, b in zip(diags, ones): assert_allclose(a, b, rtol=1e-13) # Correlation matrices are symmetric for x in xs: assert_allclose(x, x.T, rtol=1e-13)
def test_definition(self): # Test the defintion of a correlation matrix in several dimensions: # # 1. Det is product of eigenvalues (and positive by construction # in examples) # 2. 1's on diagonal # 3. Matrix is symmetric def norm(i, e): return i * e / sum(e) np.random.seed(123) eigs = [norm(i, np.random.uniform(size=i)) for i in range(2, 6)] eigs.append([4, 0, 0, 0]) ones = [[1.] * len(e) for e in eigs] xs = [random_correlation.rvs(e) for e in eigs] # Test that determinants are products of eigenvalues # These are positive by construction # Could also test that the eigenvalues themselves are correct, # but this seems sufficient. dets = [np.fabs(np.linalg.det(x)) for x in xs] dets_known = [np.prod(e) for e in eigs] assert_allclose(dets, dets_known, rtol=1e-13, atol=1e-13) # Test for 1's on the diagonal diags = [np.diag(x) for x in xs] for a, b in zip(diags, ones): assert_allclose(a, b, rtol=1e-13) # Correlation matrices are symmetric for x in xs: assert_allclose(x, x.T, rtol=1e-13)
def random_cov(ranges, O_std_min=1e-2, O_std_max=1, n_modes=1, mpi_warn=True): """ Returns a random covariance matrix, with standard deviations sampled log-uniformly from the length of the parameter ranges times ``O_std_min`` and ``O_std_max``, and uniformly sampled correlation coefficients between ``rho_min`` and ``rho_max``. The output of this function can be used directly as the value of the option ``cov`` of the :class:`likelihoods.gaussian`. If ``n_modes>1``, returns a list of such matrices. """ if not is_main_process() and mpi_warn: print("WARNING! " "Using with MPI: different process will produce different random results.") dim = len(ranges) scales = np.array([r[1] - r[0] for r in ranges]) cov = [] for i in range(n_modes): stds = scales * 10 ** (uniform.rvs(size=dim, loc=np.log10(O_std_min), scale=np.log10(O_std_max / O_std_min))) this_cov = np.diag(stds).dot( (random_correlation.rvs(dim * stds / sum(stds)) if dim > 1 else np.eye(1)) .dot(np.diag(stds))) # Symmetrize (numerical noise is usually introduced in the last step) cov += [(this_cov + this_cov.T) / 2] if n_modes == 1: cov = cov[0] return cov
def get_normal(n_dims=10, seed=111): np.random.seed(seed) eig = np.random.rand(n_dims) eig *= n_dims / eig.sum() precision_matrix = 0.1 * random_correlation.rvs(eig) mu = np.zeros(n_dims) fn_grad = lambda theta: - (theta - mu) @ precision_matrix return fn_grad, mu, precision_matrix
def gen_random_cov(n_dim): try: eigs = np.random.rand(n_dim) eigs = eigs / np.sum(eigs) * eigs.shape[0] cov = random_correlation.rvs(eigs) cov = np.multiply(cov, np.sqrt(np.outer(eigs, eigs))) return cov except Exception as e: cov = np.random.randn(n_dim, n_dim) return np.dot(cov, cov.T) / n_dim
def __init__(self, seed=None, usePort=True, expYield=None): """ :param seed: optional if omitted, the seed is randomly generated internally init constructs a DUT with a random (equal distributed) number of 10 to 50 measurements "n" A random (equal distributed) number of 4,8,12,16,20 or 24 so called "ports" "p" is generated. Each measurement is randomly associated to two not necessarily different ports. These ports contribute to the measurement error with a random drift. All measurements might be correlated through a nxn covariance matrix, generated through its n randomly generated eigenvalues. """ if seed is not None: np.random.seed(seed) self.usePort = usePort self.exp_yield = expYield if expYield is not None else np.random.randint( 3, 4.4) self.numMeas = np.random.randint(10, 50) self.numPorts = np.random.randint(1, 6) * 4 self.lastCal = 0 self.DutMeasTime = 0 self.dist_max = 0 self.ports = [port(count) for count in range(self.numPorts)] self.meas = [ meas(count, np.random.randint(1, self.numPorts), np.random.randint(1, self.numPorts), np.random.randint(50, 5000000) * 1e-06) for count in range(self.numMeas) ] for idx in range(self.numMeas): p_a = self.meas[idx].port_a p_b = self.meas[idx].port_b meas_time = self.meas[idx].meas_time self.ports[p_a].set_longest_meas_time(meas_time) self.ports[p_b].set_longest_meas_time(meas_time) self.DutMeasTime += meas_time self.portcountused = 0 for idx in range(self.numPorts): if self.ports[idx].longest_measurement_time > 0: self.portcountused += 1 self.meas_eigenvalue = np.random.random(self.numMeas) self.meas_eigenvalue *= self.numMeas / np.sum(self.meas_eigenvalue) self.meas_cov = rndcorr.rvs(self.meas_eigenvalue) self.meas_noise = np.zeros((self.numMeas, self.numMeas)) self.port_noise = np.zeros(self.numPorts) self.measurement_time = 0 self.dT_newDut = 0.1 self.dut_result = True self.meas_result = True self.errordutcount = 0 self.errormeascount = 0
def test_sphericity_from_inertia_tensors(): """ Use `scipy.stats.random_correlation` to generate matrices with known eigenvalues. Call the `sphericity_from_inertia_tensors` function to operate on these matrices, and verify that the returned sphericity agrees with an independent calculation of the sphericity of the input eigenvalues used to define the matrices. """ spherical_evals = (1., 1., 1.) non_spherical_evals = (0.1, 0.9, 2.) tensors = [] tensors.append(random_correlation.rvs(spherical_evals)) tensors.append(random_correlation.rvs(non_spherical_evals)) tensors.append(random_correlation.rvs(non_spherical_evals)) tensors.append(random_correlation.rvs(spherical_evals)) matrices = np.array(tensors) sphericity = sphericity_from_inertia_tensors(matrices) correct_non_sphericity = non_spherical_evals[0]/non_spherical_evals[2] assert np.allclose(sphericity, (1, correct_non_sphericity, correct_non_sphericity, 1))
def get_means_and_cov(num_vars, fix_eigs=False): means = np.zeros(num_vars) inv_sum = num_vars if fix_eigs == 'iid': eigs = [1] * num_vars elif fix_eigs == True: if num_vars == 5: eigs = [2, 2, 1, 0, 0] elif num_vars == 10: eigs = [4, 3, 2, 1, 0, 0, 0, 0, 0, 0] print(np.sum(eigs)) else: eigs = [] while len(eigs) < num_vars - 1: eig = np.random.uniform(0, inv_sum) eigs.append(eig) inv_sum -= eig eigs.append(inv_sum) covs = random_correlation.rvs(eigs) covs = random_correlation.rvs(eigs) return means, covs
def gen_semi_random_cov(model, eps = 0): a = np.insert(np.cumsum([model.states[u] * model.states[v] for u, v in model.graph.edgelist]), 0, 0) marginals, A = model.infer() eigs = np.random.rand(model.weights.shape[0]) eigs = eigs / np.sum(eigs) * eigs.shape[0] cov = random_correlation.rvs(eigs) cov -= -np.outer(marginals[:model.weights.shape[0]], marginals[:model.weights.shape[0]]) rhs = np.outer(marginals[:model.weights.shape[0]], marginals[:model.weights.shape[0]]) diag = np.diag(marginals[:model.weights.shape[0]] - marginals[:model.weights.shape[0]]**2) for x in range(a.shape[0] - 1): cov[a[x]:a[x + 1], a[x]:a[x + 1]] = - rhs[a[x]:a[x + 1], a[x]:a[x + 1]] cov -= np.diag(np.diag(cov)) cov += diag + np.diag(np.full(model.weights.shape[0], eps)) return cov
def test_triaxiity_from_inertia_tensors(): """ Use `scipy.stats.random_correlation` to generate matrices with known eigenvalues. Call the `triaxility_from_inertia_tensors` function to operate on these matrices, and verify that the returned triaxility agrees with an independent calculation of the triaxility of the input eigenvalues used to define the matrices. """ triaxial_evals = (2., 0.5, 0.5) non_triaxial_evals = (2., 0.9, 0.1) tensors = [] tensors.append(random_correlation.rvs(triaxial_evals)) tensors.append(random_correlation.rvs(non_triaxial_evals)) tensors.append(random_correlation.rvs(non_triaxial_evals)) tensors.append(random_correlation.rvs(triaxial_evals)) matrices = np.array(tensors) triaxility = triaxility_from_inertia_tensors(matrices) correct_non_triaxility = ( (non_triaxial_evals[0]**2-non_triaxial_evals[1]**2) / (non_triaxial_evals[0]**2-non_triaxial_evals[2]**2) ) assert np.allclose(triaxility, (1, correct_non_triaxility, correct_non_triaxility, 1))
def add_random_correlation(data): """ Step 5 """ dims = data.shape[-1] evs = np.random.uniform(0.01, 1, size=dims) evs = evs / np.sum(evs) * dims random_corr_matrix = random_correlation.rvs(evs) cholesky_transform = np.linalg.cholesky(random_corr_matrix) for i in range(data.shape[0]): normal_eq_mean = cholesky_transform.dot( data[i].T) # Generating random MVN (0, cov_matrix) normal_eq_mean = normal_eq_mean.transpose() normal_eq_mean = normal_eq_mean.transpose() # Transposing back data[i] = normal_eq_mean.T return data
def generate_synthetic_data(data_dim, data_segments, n_obs_seg, n_layer, simulationMethod='TCL', seed=1, one_hot_labels=False, varyMean=False): np.random.seed(seed) if simulationMethod.lower() == 'tcl': dat_all = gen_TCL_data_ortho(Ncomp=data_dim, Nsegment=data_segments, Nlayer=n_layer, NsegmentObs=n_obs_seg, source='Gaussian', NonLin='leaky', negSlope=.2, seed=seed, varyMean=varyMean) elif simulationMethod.lower() == 'imca': baseEvals = np.random.rand(data_dim) baseEvals /= ((1. / data_dim) * baseEvals.sum()) baseCov = random_correlation.rvs(baseEvals) dat_all = gen_IMCA_data(Ncomp=data_dim, Nsegment=data_segments, Nlayer=n_layer, NsegmentObs=n_obs_seg, NonLin='leaky', negSlope=.2, BaseCovariance=baseCov, seed=seed) else: raise ValueError( 'invalid simulation method: {}'.format(simulationMethod)) x = dat_all['obs'] if one_hot_labels: y = to_one_hot(dat_all['labels'])[0] else: y = dat_all['labels'] s = dat_all['source'] return x, y, s
def test_NeighbourCovariance(): from scipy.stats import random_correlation as randc from numpy.random import multivariate_normal import numpy as np import matplotlib.pyplot as plt nvert=200 np.random.seed(514) x = randc.rvs((.2, 1.8)) print(x) coords = multivariate_normal([1.,2.], x, size=nvert) print('coords in',coords.shape) #plt.scatter(coords[:,0],coords[:,1]) #plt.show() import tensorflow as tf from neighbour_covariance_op import NeighbourCovariance coordinates = tf.constant(coords,dtype='float32') n_idxs = tf.tile(tf.expand_dims(tf.range(nvert),axis=0),[nvert,1]) print(n_idxs) distsq = tf.cast(n_idxs, dtype='float32')/100. features = tf.constant(np.random.rand(nvert,2),dtype='float32')+1e-2 cov, mean_C = NeighbourCovariance(coordinates, distsq, features, n_idxs) #expected shapes: V x F x 2 * 2, V x F x 2 print(cov.shape, mean_C.shape) print('op neighbour covariance and mean') print(cov[0],'\n',mean_C[0]) #all the same print('numpy neighbour covariance') print(np.cov(coords, aweights=features[:,0]*np.exp(-distsq[0]), rowvar=False, ddof=0)) print(np.cov(coords, aweights=features[:,1]*np.exp(-distsq[0]), rowvar=False, ddof=0))
def test_principal_axes_from_inertia_tensors1(): """ Starting from 500 random positive definite symmetric matrices, enforce that the axes returned by the _principal_axes_from_inertia_tensors function are actually eigenvectors with the correct eigenvalue. """ npts = int(500) correct_evals = (0.3, 0.7, 2.0) matrices = np.array([random_correlation.rvs(correct_evals) for __ in range(npts)]) assert matrices.shape == (npts, 3, 3) principal_axes, evals = principal_axes_from_inertia_tensors(matrices) assert np.shape(principal_axes) == (npts, 3) assert np.shape(evals) == (npts, ) assert np.allclose(evals, correct_evals[2]) for i in range(npts): m = matrices[i, :, :] x = principal_axes[i, 0] y = principal_axes[i, 1] z = principal_axes[i, 2] p = np.array((x, y, z)) q = np.matmul(m, p) assert np.allclose(q, correct_evals[2]*p)
def __init__(self, seed=None, usePort=True, expYield=None): """ :param seed: optional if omitted, the seed is randomly generated internally init constructs a DUT with a random (equal distributed) number of 10 to 50 measurements "n" A random (equal distributed) number of 4,8,12,16,20 or 24 so called "ports" "p" is generated. Each measurement is randomly associated to two not necessarily different ports. These ports contribute to the measurement error with a random drift. All measurements might be correlated through a nxn covariance matrix, generated through its n randomly generated eigenvalues. """ if seed is not None: np.random.seed(seed) self.usePort = usePort self.exp_yield = expYield if expYield is not None else np.random.randint( 3, 4.4) self.numMeas = np.random.randint(10, 50) self.numPorts = np.random.randint(1, 6) * 4 self.lastCal = 0 self.DutMeasTime = 0 self.dist_max = 0 # list of ports with uncorrelated random walk generators self.ports = [port(count) for count in range(self.numPorts)] self.meas = [ meas(count, np.random.randint(1, self.numPorts), np.random.randint(1, self.numPorts), np.random.randint(50, 5000000) * 1e-6) for count in range(self.numMeas) ] for idx in range(self.numMeas): p_a = self.meas[idx].port_a p_b = self.meas[idx].port_b meas_time = self.meas[idx].meas_time self.ports[p_a].set_longest_meas_time(meas_time) self.ports[p_b].set_longest_meas_time(meas_time) self.DutMeasTime += meas_time # count used ports self.portcountused = 0 for idx in range(self.numPorts): if self.ports[idx].longest_measurement_time > 0: self.portcountused += 1 # generate random eigenvalues for covariance matrix ... self.meas_eigenvalue = np.random.random(self.numMeas) # ... and normalise it self.meas_eigenvalue *= self.numMeas / np.sum(self.meas_eigenvalue) # generate covarance matrix based on random eigenvector self.meas_cov = rndcorr.rvs(self.meas_eigenvalue) self.meas_noise = np.zeros((self.numMeas, self.numMeas)) self.port_noise = np.zeros(self.numPorts) # instantiate p biased ports errors #self.ports = [RandomWalk(count) for count in range(self.numPorts)] # measurement time self.measurement_time = 0 self.dT_newDut = 0.1 self.dut_result = True self.meas_result = True self.errordutcount = 0 self.errormeascount = 0 self.defected_meas = np.random.choice(range(self.numMeas), np.random.randint( 0, self.numMeas // 8), replace=False) self.neg_corr = [(i, j) for i, j in zip( self.defected_meas, np.random.choice( list(set(range(self.numMeas)) - set(self.defected_meas)), len(self.defected_meas)))] self.pos_corr = [(i, j) for i, j in zip( self.defected_meas, np.random.choice( list( set(range(self.numMeas)) - set(self.defected_meas) - set([k[1] for k in self.neg_corr])), len(self.defected_meas)))]
def simulate(self): self.simulated = True # Constants ascale = np.arange(0.1, 0.51, 0.01) dt = 1 / self.srate nsamples = len(self.time) min_state = np.argmin( np.abs(self.time - 0.15)) + 1 # Minimum duration in frames # - strurctural links (binary mask) I = np.eye(self.n) if self.SC is None: SClinks = 0.8 # Markov et al. 2012 UT = np.triu( np.random.choice([0, 1], self.n**2, replace=True, p=[1 - SClinks, SClinks]).reshape(self.n, self.n)) MK = UT + UT.T - np.diag(np.diag(UT + UT.T)) self.SC = MK + I else: MK = np.copy(self.SC) MK[np.diag_indices(self.n)] = 0 # - directed interactions (binary mask) FC = np.zeros(self.SC.shape) # HERE USE SC.shape instead of MK.shape MKconnections = np.array(np.where(MK)) num_connections = len(MKconnections[0]) ids = np.random.choice(num_connections, int( np.fix( (1 - self.sparsity) * num_connections)), replace=False) FC[np.array(MKconnections)[0, ids], np.array(MKconnections)[1, ids]] = 1 self.FC = FC + I # - AR process (univariate) self.AR = np.zeros((self.n, self.n, self.popt, nsamples)) for i in range(self.n): c1 = np.random.choice(ascale, 1) c2 = (np.max(ascale) - c1) * .95 self.AR[i, i, 0:2, :] = np.tile( np.concatenate((c1, c2)).reshape(1, -1).T, (1, 1, nsamples)) # Low Pass #self.AR[i,i,0:2,:] = np.tile(np.concatenate((c1,c2)).reshape(1,-1,order='F').T,(1,1,nsamples)) # Low Pass # - AR process (univariate) cON = np.where(MK * self.FC) m = nsamples // min_state bf = np.arange(m * min_state).reshape(min_state, m, order='F') start_at = np.sort( np.random.choice(np.arange(2, m), self.nstates, replace=False)) state_ons = bf[0, start_at] state_end = np.concatenate([bf[0, start_at[1:]], [nsamples]]) state_dur = state_end - state_ons # Determine states self.regimes = dict() # Starting (no scaling) scalef = 1 for k in range(self.nstates): self.regimes[k] = np.arange(state_ons[k], state_ons[k] + state_dur[k]) summary = np.ones((len(cON[0]), 9)) * np.nan while True: # generate off-diag AR and check stability tmpAR = self.AR[:, :, :, self.regimes[k][0]] for ij in range(len(cON[0])): i, j = cON[0][ij], cON[1][ij] ij_p = np.random.choice(self.delay, 1)[0] ampl1 = np.random.choice(ascale * .5, 1) ampl2 = (np.max(ascale * 0.5) - ampl1) * 0.95 osc = np.sign(np.random.randn(2)) * np.concatenate( (ampl1, ampl2)) * scalef summary[ij, :] = np.concatenate(([k, i, j], ampl1, [ self.time[self.regimes[k][0]], self.time[self.regimes[k][-1]] ], osc, [ij_p])) tmpAR[i, j, ij_p:ij_p + 2] = osc # Stability check blockA = np.zeros((self.n * self.popt, self.n * self.popt)) blockA[:self.n, :] = tmpAR.reshape(tmpAR.shape[0], -1, order='F') blockA[self.n:, :(self.popt - 1) * self.n] = np.eye( (self.popt - 1) * self.n) blockA[self.n:, (self.popt - 1) * self.n:] = np.zeros( ((self.popt - 1) * self.n, self.n)) w, v = np.linalg.eig(blockA) if any(abs(w) > .95): scalef = scalef * .95 else: break if k == 0: self.summary = np.copy(summary) else: self.summary = np.concatenate((self.summary, summary), 0) # Add stable matrices to the dynamical system self.AR[:, :, :, self.regimes[k]] = np.tile( tmpAR.reshape(self.n, self.n, self.popt, 1), [1, 1, 1, len(self.regimes[k])]) # - Data (add nuisance segment at the beginning) nuisance = np.min([state_ons[0] * 2, int(.5 / dt)]) self.X = np.zeros((self.ntrials, self.n, nsamples + nuisance)) ARplus = np.concatenate((self.AR[:, :, :, :nuisance], self.AR), 3) # simulate between-trials correlation (correlated generative noise) while True: try: CTeigvals = np.random.rand(self.ntrials) CTeigvals = CTeigvals / np.sum(CTeigvals) * self.ntrials CT = abs(random_correlation.rvs(CTeigvals)) * 3 break except: pass self.CT = CT.clip(max=1) dgI = np.random.choice( np.where(np.eye(self.ntrials).reshape(-1) == 0)[0], int((self.ntrials**2 - self.ntrials) * .1), replace=False) NegCT = np.ones(self.ntrials * self.ntrials) NegCT[dgI] = -1 self.CT *= NegCT.reshape(self.ntrials, self.ntrials) # - Generate time-series for k_p in range(self.AR.shape[2]): # the actual or p_opt self.X[:, :, k_p] = self.CT @ np.random.randn(self.ntrials, self.n) self.E = np.copy(self.X) for k in range(self.AR.shape[2] + 1, nsamples + nuisance): innovation = self.CT @ np.random.randn( self.ntrials, self.n) # across trials correlation self.E[:, :, k] = innovation self.X[:, :, k] += innovation for l in range(self.AR.shape[2]): self.X[:, :, k] += (ARplus[:, :, l, k] @ self.X[:, :, k - l].T).T self.X = self.X[:, :, nuisance:] # remove nuisance data self.Y = np.copy(self.X) # obseved signal self.E = self.E[:, :, nuisance:] # innovation self.AR = self.AR[:, :, :, :nsamples] # ensure size, AR coeffs tmp = self.E.transpose(1, 2, 0).reshape(self.n, -1, order='F') self.R = tmp @ tmp.T # - SNR if self.snr_db is not None: self.addnoise('w') else: self.noise = 0 self.Y_pre_lm = np.copy(self.Y) # - Linear mixing if self.lmix > 0: if self.DM is None: x = np.random.choice(np.arange(1, 151), self.n, replace=False) # 2d lattice 15x15cm y = np.random.choice(np.arange(1, 151), self.n, replace=False) mixf = norm.pdf(np.arange(0, 151), 0, self.lmix) xy = np.concatenate(([x], [y]), axis=0) self.DM = squareform(pdist(xy.T)) else: mixf = norm.pdf(np.arange(0, np.max(self.DM)), 0, self.lmix) self.LMx = norm.pdf(self.DM, 0, self.lmix) / np.max(mixf) self.Y = np.matmul(np.tile(self.LMx, [self.ntrials, 1, 1]), self.Y) else: self.DM = np.ones((self.n, self.n)) * np.nan self.LMx = np.zeros((self.n, self.n)) self.scaling = scalef
def random_correlation(d, a, seed): np.random.seed(seed) eigs = np.diff(sorted(np.r_[0, np.random.rand(d - 1), 1])) * d assert np.allclose(sum(eigs), d) and len(eigs) == d return _random_correlation.rvs(eigs), np.array([a] * d)
import math import random import matplotlib.pyplot as plt import numpy as np from scipy.stats import random_correlation as rndcorr exp_yield = 2 numMeas = np.random.randint(10, 50) numPorts = np.random.randint(1, 6) * 4 meas_eigenvalue = np.random.random(numMeas) meas_eigenvalue *= numMeas / np.sum(meas_eigenvalue) meas_cov = rndcorr.rvs(meas_eigenvalue) meas_noise = np.zeros((numMeas, numMeas)) port_noise = np.zeros(numPorts) meas_noise = np.random.multivariate_normal(np.zeros(numMeas), meas_cov) / exp_yield meas_dist = [] for i in range(0, numMeas): meas_dist.append(meas_noise[i] ** 2) plt.plot(meas_dist) plt.show()
def r_corr(size): r_arr = np.random.uniform(0, 5, size=size) r_arr = size * r_arr / sum(r_arr) return random_correlation.rvs(r_arr)
import seaborn as sns import yfinance as yf # %% # get fake data # specify eigenvalues num_fake_stocks = 10 # eig_v = np.random.rand(num_fake_stocks) eig_v = np.random.uniform(0.5, 1, num_fake_stocks) eig_v[-1] = eig_v.shape[0] - np.sum(eig_v[:-1]) np.random.seed(666) mean_true = np.random.uniform(0, 0.15, num_fake_stocks) std_true = np.random.uniform(0.15, 0.5, num_fake_stocks).reshape(-1, 1) corr_true = random_correlation.rvs(eig_v) cov_true = np.outer(std_true, std_true) * corr_true benchmark_weight_fake = np.ones(num_fake_stocks) / num_fake_stocks maximum_deviation_fake = 1 # %% # fake data # MiniVar MiniVar_weight_fake = optmodels.MiniVar(cov_true, num_fake_stocks, benchmark_weight_fake, maximum_deviation_fake) # RiskParity RiskParity_weight_fake = optmodels.RiskParity(cov_true, num_fake_stocks,
def random_covariance(num_trjs): cov = random_correlation.rvs((.5, .8, 1.2, 1.5, 1.0, .5, .8, 1.2, 1.5, 1.0, .5, .8, 1.2, 1.5, 1.0)) positions = np.random.multivariate_normal(np.zeros((15,)), cov, num_trjs) return positions
""" evalues, Q = np.linalg.eig(cov) evalues[evalues < evalues[n_components]] = 0 # Since cov is symmetric, by spectral theorem Q_inv = Q_T Q_T = np.matrix.transpose(Q) L = np.diag(evalues) # Due to spectral theorem all eigenvalues must be real. I encountered # complex eigenvalues regardless. Hence, extract real part. denoised_cov = pd.DataFrame(Q.dot(L).dot(Q_T)).apply(np.real) return denoised_cov if __name__ == '__main__': from scipy.stats import random_correlation expected_returns = pd.Series([1, 1, -1, -1], index=['a', 'b', 'c', 'd']) covar = random_correlation.rvs((.5, .8, 1.2, 1.5)) weights = minimize_objective( expected_returns.index, negative_sharpe, True, (-1, 1), expected_returns, covar, 0.0, 0.0, ) print(weights)
def data_generation(num_control, num_treated, num_covs): mean = [0] * num_covs eigenvalues = num_covs * np.random.dirichlet(np.ones(num_covs), size=1)[0] cov = random_correlation.rvs(eigenvalues) X = [] for i in range(num_treated + num_control): z = list(np.random.multivariate_normal(mean, cov)) x = [1 if elem > 0 else 0 for elem in z] X.append(x) xc = np.array(X[:num_control]) xt = np.array(X[num_control:]) #print(X[15000:]) #print(xt) errors1 = np.random.normal(0, 0.1, size=num_control) # some noise errors2 = np.random.normal(0, 0.1, size=num_treated) # some noise dense_bs_sign = np.random.choice([-1, 1], num_covs) #dense_bs = [ np.random.normal(dense_bs_sign[i]* (i+2), 1) for i in range(len(dense_bs_sign)) ] dense_bs = [np.random.normal(s * 10, 10) for s in dense_bs_sign] yc = np.dot( xc, np.array(dense_bs)) #+ errors1 # y for conum_treatedrol group catt_c = [0] * num_control treatment_eff_coef = np.random.normal(0.5, 0.15, size=num_covs) treatment_effect = np.dot(xt, treatment_eff_coef) second = construct_sec_order(xt[:, :5]) treatment_eff_sec = np.sum(second, axis=1) yt = np.dot( xt, np.array(dense_bs) ) + treatment_effect + treatment_eff_sec #+ errors2 # y for treated group catt_t = treatment_effect + treatment_eff_sec df1 = pd.DataFrame(xc, columns=range(num_covs)) df1['outcome'] = yc df1['treated'] = 0 df2 = pd.DataFrame(xt, columns=range(num_covs)) df2['outcome'] = yt df2['treated'] = 1 df = pd.concat([df1, df2]) df['matched'] = 0 #df['outcome'] += 2 * np.random.normal(0,5) catt = catt_c + list(catt_t) df['true_effect'] = catt miss = [] for i in range(num_control + num_treated): miss.append(1 if random.randint(1, 101) <= 20 else 0) miss = pd.DataFrame(np.zeros((num_control + num_treated, num_covs))) select = set() k = 0 total_miss_num = (num_control + num_treated) * num_covs * 0.05 while k < total_miss_num: row = random.randint(0, num_control + num_treated - 1) col = random.randint(0, num_covs - 1) if (row, col) in select: continue miss.iloc[row, col] = 1 select.add((row, col)) k += 1 return df, df[list(range(num_covs))], df['treated'], df['outcome'], miss
def args_maker(): x, mean, cov = map(rng, (shapex, shapex, (dim, dim)), dtypes) cov = random_correlation.rvs( onp.arange(1, 1 + dim) * 2 / (dim + 1)) return [x, mean, cov]
import multiprocessing as mp from nearest_correlation import nearcorr from scipy.stats import random_correlation from knowledge_gradient import KG_Alg from knowledge_gradient import KG_multi from knowledge_gradient import update_mu_S if __name__ == "__main__": processes = mp.cpu_count() pool = mp.Pool(processes) np.random.seed(126) g = 7/sum([.5, .8, 1.2, 2.5, 1.7, 2.1, 2.2]) G = np.round(random_correlation.rvs((g*.5, g*.8, g*1.2, g*2.5, g*1.7, g*2.1, g*2.2)), 3) S = nearcorr(G, tol=[], flag=0, max_iterations=1000, n_pos_eig=0, weights=None, verbose=False, except_on_too_many_iterations=True) M = S.shape[0] lambda_ = np.array([0.2, 1.1, 1.3, 0.12, 0.4, 0.3, 0.12]) mu = np.array([0.2, 0.21, 0.92, 0.11, 0.7, 0.2, -0.1]) print(KG_Alg(mu, S, lambda_)) print(KG_multi(mu, S, lambda_, pool)) y=0.22