def qEI(sigma, eps, mu, plugin, q): # hardcoded lower limit for multinormal pdf integration # todo # note that here we explicitly assume data is 2-d low = np.array([-20.0 for _ in range(q)]) pk = np.zeros((q, )) first_term = np.zeros((q, )) second_term = np.zeros((q, )) for k in range(q): Sigma_k = covZk(sigma, k) mu_k = mu - mu[k] mu_k[k] = -mu[k] b_k = np.zeros((q, )) b_k[k] = -plugin # suspect that it is equal to pmnorm function of original R package zeros = np.array([0.0 for i in range(q)]) p, _ = mvn.mvnun(low, b_k - mu_k, zeros, Sigma_k) pk[k] = p first_term[k] = (mu[k] - plugin) * pk[k] upper_temp = b_k + eps * Sigma_k[:, k] - mu_k p1, _ = mvn.mvnun(low, upper_temp, zeros, Sigma_k) second_term[k] = 1 / eps * (p1 - pk[k]) expectedImprov = np.sum(first_term) + np.sum(second_term) return expectedImprov
def verify(self): """ Verify our results against Yunsi Fei's results where she publishes her signal to noise ration and success rate curves """ x = [] yDPA = [] yCPA = [] # Amended Formula which works for i in range(1, 200, 20): cov, mean = self.conf.YCovMeanDPA(0.0016, 0.0046, i, correctKey=60) yDPA.append( mvn.mvnun(DESSuccess.lower, DESSuccess.upper, mean.reshape(63), cov)[0]) x.append(i) cov, mean = self.conf.YCovMeanCPA2(0.0016, 0.0048, i, correctKey=60) yCPA.append( mvn.mvnun(self.lower, self.upper, mean.reshape(63), cov)[0]) p1 = plot(xest, ySR1est, '--') p2 = plot(x, yCPA) legend(loc='lower right') ylabel('Success Rate') xlabel('Number of Measurements') show()
def cal_c(): from scipy.stats import mvn import numpy as np low = np.array([0, 0]) upp = np.array([1, 1]) mu = np.array([.5, .5]) S = np.array([[1, 0], [0, 1]]) p, i = mvn.mvnun(low, upp, mu, S) print(p, i) low = np.array([0, 0]) upp = np.array([1, 1]) mu = np.array([.5, .5]) S = np.array([[.1, 0], [0, 0.1]]) p, i = mvn.mvnun(low, upp, mu, S) print(p, i) low = np.array([-.5, -.5]) upp = np.array([.5, .5]) mu = np.array([0, 0]) S = np.array([[1, 0], [0, 1]]) p, i = mvn.mvnun(low, upp, mu, S) print(p, i) return p
def EIBV_2D(Threshold_T, Threshold_S, mu, Sig, F, R): ''' :param Threshold_T: :param Threshold_S: :param mu: :param Sig: :param F: sampling matrix :param R: noise matrix :return: EIBV evaluated at every point ''' # Update the field variance a = np.dot(Sig, F.T) b = np.dot(np.dot(F, Sig), F.T) + R c = np.dot(F, Sig) Sigxi = np.dot(a, np.linalg.solve(b, c)) # new covariance matrix V = Sig - Sigxi # Uncertainty reduction # updated covariance IntA = 0.0 N = mu.shape[0] # integrate out all elements in the bernoulli variance term for i in np.arange(0, N, 2): # extract the corresponding variance reduction term SigMxi = Sigxi[np.ix_([i, i + 1], [i, i + 1])] # extract the corresponding mean terms Mxi = [mu[i], mu[i + 1]] # temp and salinity sn2 = V[np.ix_([i, i + 1], [i, i + 1])] # vv2 = np.add(sn2, SigMxi) # was originally used to make it obscure vv2 = Sig[np.ix_([i, i + 1], [i, i + 1])] # compute the first part of the integration Thres = np.vstack((Threshold_T, Threshold_S)) mur = np.subtract(Thres, Mxi) IntB_a = mvn.mvnun(np.array([[-np.inf], [-np.inf]]), np.zeros([2, 1]), mur, vv2)[0] # compute the second part of the integration, which is squared mm = np.vstack((Mxi, Mxi)) # SS = np.array([[vv2, SigMxi], [SigMxi, vv2]]) # thought of it as a simplier version SS = np.add( np.vstack((np.hstack((sn2, np.zeros( (2, 2)))), np.hstack((np.zeros((2, 2)), sn2)))), np.vstack((np.hstack((SigMxi, SigMxi)), np.hstack( (SigMxi, SigMxi))))) Thres = np.vstack((Threshold_T, Threshold_S, Threshold_T, Threshold_S)) mur = np.subtract(Thres, mm) IntB_b = mvn.mvnun( np.array([[-np.inf], [-np.inf], [-np.inf], [-np.inf]]), np.zeros([4, 1]), mur, SS)[0] # compute the total integration IntA = IntA + np.nansum([IntB_a, -IntB_b]) return IntA
def EIBV_1D(self, threshold, mu, Sig, F, R): Sigxi = Sig @ F.T @ np.linalg.solve(F @ Sig @ F.T + R, F @ Sig) V = Sig - Sigxi sa2 = np.diag(V).reshape( -1, 1) # the corresponding variance term for each location IntA = 0.0 for i in range(len(mu)): sn2 = sa2[i] m = mu[i] IntA = IntA + mvn.mvnun(-np.inf, threshold, m, sn2)[0] - mvn.mvnun( -np.inf, threshold, m, sn2)[0]**2 return IntA
def marginal_desviaciones2(x1, x2, p, rrho, factor, qq1, qq2, gg1, gg2, maxpts1, abseps1): pedazo1=np.zeros((1,1)); pedazo2=np.zeros((1,1)); pedazo3=np.zeros((1,1)); pedazo4=np.zeros((1,1)); pedazo5=np.zeros((1,1)); pedazo6=np.zeros((1,1)); upp=np.zeros((1,2)); mu=np.zeros((2,1)); #test = np.inf; test = -30; low=np.array([test,test]); rhoi=(qq1*(qq2*rrho)); Sigma1=np.array([[1,rhoi],[rhoi,1]]); #MX=np.asmatrix(np.dot(np.transpose(x),x)); q1XB1=(qq1*x1); q2XB2=(qq2*x2); upp=np.array([q1XB1,q2XB2]); mu=np.array([0,0]); phi2,inform = mvn.mvnun(low,upp,mu,Sigma1); #phi2=mvstdnormcdf(low,upp,Sigma1,maxpts1,abseps1,abseps1); if phi2<=abseps1: phi2=abseps1; mphi2=normpdf(upp, mu, Sigma1, 2); pedazo1=((q1XB1*gg1)/phi2)-((rhoi*mphi2)/phi2)-((gg1**2)/(phi2**2)); pedazo2=((q2XB2*gg2)/phi2)-((rhoi*mphi2)/phi2)-((gg2**2)/(phi2**2)); pedazo3=(mphi2/phi2)-((gg1*gg2)/(phi2**2)); pedazo4=qq2*((mphi2/phi2)*(rhoi*(factor*(factor*(q2XB2-rhoi*q1XB1)))-q1XB1-(gg1/phi2))); pedazo5=qq1*((mphi2/phi2)*(rhoi*(factor*(factor*(q1XB1-rhoi*q2XB2)))-q2XB2-(gg2/phi2))); pedazo6=(mphi2/phi2)*(((factor**2)*rhoi)*(1-(factor**2)*(((x1**2)+(x2**2))-2*rhoi*(q1XB1*q2XB2)))+((factor*(q1XB1*q2XB2))-(mphi2/phi2))); return pedazo1, pedazo2, pedazo3, pedazo4, pedazo5, pedazo6;
def _gaussian(u, rho): """ Generates values of the Gaussian copula Inputs: u -- u is an N-by-P matrix of values in [0,1], representing N points in the P-dimensional unit hypercube. rho -- a P-by-P correlation matrix. Outputs: y -- the value of the Gaussian Copula """ n = u.shape[0] p = u.shape[1] lo = np.full((1, p), -10) hi = norm.ppf(u) mu = np.zeros(p) # need to use ppf(q, loc=0, scale=1) as replacement for norminv # need to use mvn.mvnun as replacement for mvncdf # the upper bound needs to be the output of the ppf call, right now it is set to random above y = np.zeros(n) # I don't know if mvnun is vectorized, I couldn't get that to work for ii in np.arange(n): # do some error checking. if we have any -inf or inf values, # p, i = mvn.mvnun(lo, hi[ii, :], mu, rho) y[ii] = p return y
def estimatePartitionFunction(K, mean, cov): # https://github.com/scipy/scipy/blob/v1.2.1/scipy/stats/_multivariate.py # line 525, def _cdf assert cov.shape[1:] == (K, K) M = len(cov) assert mean.shape[1:] == (M, K) N = len(mean) arr = np.empty([N, M], dtype=np.float) for a, m, c in zip(arr.T, mean.transpose(1, 0, 2), cov): a[:] = [ mvn.mvnun( lower=-x, upper=np.full(K, np.inf), means=np.zeros(K), covar=c, maxpts=1000000 * K, abseps=1e-6, releps=1e-6, )[0] for x in m ] # [mvn.mvnun(lower=-m, upper=np.full(K, np.inf), means=np.zeros(K), covar=cov)[0] for m in mean] # [mvn.mvnun(lower=np.zeros(K), upper=np.full(K, np.inf), means=m, covar=cov)[0] for m in mean] # [mvn.mvnun(lower=np.full(K, -np.inf), upper=m, means=np.zeros(K), covar=cov)[0] for m in mean] # multivariate_normal(mean=None, cov=cov).cdf(mean) # multivariate_normal(mean=np.zeros(K), cov=cov).cdf(mean) # [multivariate_normal(mean=-m, cov=cov).cdf(np.zeros(K)) for m in mean] arr = torch.tensor(arr, dtype=dtype, device=device) return arr
def integrate_box(self, low_bounds, high_bounds, maxpts=None): """Computes the integral of a pdf over a rectangular interval. Parameters ---------- low_bounds : vector lower bounds of integration high_bounds : vector upper bounds of integration maxpts=None : int maximum number of points to use for integration Returns ------- value : scalar the result of the integral """ if maxpts is not None: extra_kwds = {'maxpts': maxpts} else: extra_kwds = {} value, inform = mvn.mvnun(low_bounds, high_bounds, self.dataset, self.covariance, **extra_kwds) if inform: msg = ('an integral in mvn.mvnun requires more points than %s' % (self.d*1000)) warnings.warn(msg) return value
def PIBV(loc_pilot, mu_upd, Q_upd, Mvar): IntA = 0.0 L = cholesky(Q_upd) for ind in loc_pilot: # loop through 53 pilot points mvar = Mvar[ind] mu = mu_upd[ind] e = np.zeros([N, 1]) # unit vector to find sigma e[ind] = True Sigma = L.solve_A(e) Cov = np.array([mvar, mvar - Sigma[ind], mvar - Sigma[ind], mvar]).reshape(2, 2) IntA = IntA + mvn.mvnun(-np.inf, Thres, mu, mvar)[0] - \ mvn.mvnun(np.array([[-np.inf], [-np.inf]]), np.ones([2, 1]) * Thres, np.ones([2, 1]) * mu, Cov)[0] return IntA
def integrate_box(self, low_bounds, high_bounds, maxpts=None): """Computes the integral of a pdf over a rectangular interval. Parameters ---------- low_bounds : vector lower bounds of integration high_bounds : vector upper bounds of integration maxpts=None : int maximum number of points to use for integration Returns ------- value : scalar the result of the integral """ if maxpts is not None: extra_kwds = {'maxpts': maxpts} else: extra_kwds = {} value, inform = mvn.mvnun(low_bounds, high_bounds, self.dataset, self.covariance, **extra_kwds) if inform: msg = ('an integral in mvn.mvnun requires more points than %s' % (self.d * 1000)) warnings.warn(msg) return value
def integrate_box(self, low_bounds, high_bounds, maxpts=None): """Computes the integral of a pdf over a rectangular interval. Parameters ---------- low_bounds : array_like A 1-D array containing the lower bounds of integration. high_bounds : array_like A 1-D array containing the upper bounds of integration. maxpts : int, optional The maximum number of points to use for integration. Returns ------- value : scalar The result of the integral. """ if maxpts is not None: extra_kwds = {'maxpts': maxpts} else: extra_kwds = {} value, inform = mvn.mvnun(low_bounds, high_bounds, self.dataset, self.covariance, **extra_kwds) if inform: msg = ('An integral in mvn.mvnun requires more points than %s' % (self.d * 1000)) warnings.warn(msg) return value
def GaussianUniformIntegral(x_range, y_range, x_range_shift, y_range_shift, cov, num_x_grid=3, num_y_grid=3): ''' Exact equation: 1 / (size of x_range) * 1 / (size of y_range) * integral_{box of x_range y range} dxdy integral_{box of x_range_shift y_range_shift} dxshift dyshift N(xshift - x, yshift - y | cov) Approximation: the uniform distribution using grids in x_range and y_range 1 / (number x grids) * 1 / (number y grids) * sum_{x grids} sum_{y grids} integral dxshift dyshift N(xgrid - xshift, ygrid - yshift | cov) ''' value = 0 size_x_grid = (x_range[1] - x_range[0]) / (num_x_grid + 1) size_y_grid = (y_range[1] - y_range[0]) / (num_y_grid + 1) for xgrid in np.arange(x_range[0] + size_x_grid, x_range[1], size_x_grid): for ygrid in np.arange(y_range[0] + size_y_grid, y_range[1], size_y_grid): prob, _ = mvn.mvnun( np.array([x_range_shift[0] - xgrid, y_range_shift[0] - ygrid]), np.array([x_range_shift[1] - xgrid, y_range_shift[1] - ygrid]), np.zeros(2), cov) value += prob value /= (num_x_grid * num_y_grid) return value
def _cdf(self, x, mean, cov, maxpts, abseps, releps): """ Parameters ---------- x : ndarray Points at which to evaluate the cumulative distribution function. mean : ndarray Mean of the distribution cov : array_like Covariance matrix of the distribution maxpts: integer The maximum number of points to use for integration abseps: float Absolute error tolerance releps: float Relative error tolerance Notes ----- As this function does no argument checking, it should not be called directly; use 'cdf' instead. .. versionadded:: 1.0.0 """ lower = np.full(mean.shape, -np.inf) # mvnun expects 1-d arguments, so process points sequentially func1d = lambda x_slice: mvn.mvnun(lower, x_slice, mean, cov, maxpts, abseps, releps)[0] out = np.apply_along_axis(func1d, -1, x) return _squeeze_output(out)
def p(self): # TODO : hard-coded error margin cov = self.ukf.P[:3, :3] # ignore velocity components ( for now ) # error margin, +- 0.5m, 30 deg. d = np.asarray([0.5, 0.5, np.deg2rad(30)]) _p, _ = mvn.mvnun(-d, d, np.zeros_like(d), cov) return _p
def _gaussian(u, rho): """ Generates values of the Gaussian copula Inputs: u -- u is an N-by-P matrix of values in [0,1], representing N points in the P-dimensional unit hypercube. rho -- a P-by-P correlation matrix. Outputs: y -- the value of the Gaussian Copula """ n = u.shape[0] p = u.shape[1] lo = np.full((1,p), -10) hi = norm.ppf(u) mu = np.zeros(p) # need to use ppf(q, loc=0, scale=1) as replacement for norminv # need to use mvn.mvnun as replacement for mvncdf # the upper bound needs to be the output of the ppf call, right now it is set to random above y = np.zeros(n) # I don't know if mvnun is vectorized, I couldn't get that to work for ii in np.arange(n): # do some error checking. if we have any -inf or inf values, # p,i = mvn.mvnun(lo, hi[ii,:], mu, rho) y[ii] = p return y
def computeProbability(self, x, G, g): # compute probability of violating the constraints self.m_in = G.shape[0] self.Sigma_eG = np.dot(np.dot(G, self.Sigma), G.transpose()) low = np.array(-100 * np.sqrt(np.diag(self.Sigma_eG))) ineq_margins = np.dot(G, x) + g p, i = mvn.mvnun(low, ineq_margins, np.zeros(self.m_in), self.Sigma_eG) return p
def M_MVN(a,b,c): from scipy.stats import mvn mean = np.array([0,0]) Sigma = np.array([[1,c],[c,1]]) lower=np.array([-10000,-10000]) upper=np.array([a,b]) p,i=mvn.mvnun(lower,upper,mean,Sigma) return p
def getDefaultCorAB(A, B, muA, muB, sigmaA, sigmaB, rhoAB, dt, KA, KB): dA = getDelta(muA, sigmaA, dt, A, KA) dB = getDelta(muB, sigmaB, dt, B, KB) pA = norm.cdf(dA) pB = norm.cdf(dB) pAB, err = mvn.mvnun(np.array([-100, -100]), np.array([dA, dB]), np.array([0, 0]), np.array([[1, rhoAB], [rhoAB, 1]])) return np.divide(pAB - pA * pB, np.sqrt(pA * pB * (1 - pA) * (1 - pB)))
def subproc(self,args): sub_mu = args[0] sub_cov = args[1] sub_var = np.diag(sub_cov) fstar = args[2] grid_num = 100 J = np.shape(sub_var)[0]-1 ##### compute 2nd term from here ##### upper = np.min(np.array([sub_mu + 5*np.sqrt(sub_var), fstar*np.ones(J+1)]),axis=0) lower = np.min(np.array([sub_mu - 5*np.sqrt(sub_var), fstar*np.ones(J+1)]),axis=0) Z = mvn.mvnun(lower,upper,sub_mu,sub_cov)[0] if Z > 0: if J < 3: ##### compute conditional Entropy by Division quadrature ##### grid = [] for j in range(J+1): jth_space = np.unique(np.linspace(lower[j],upper[j],grid_num))[:,np.newaxis] grid.append(jth_space) grid = yz.gen_mesh(grid) H = np.array(multivariate_normal.pdf(grid,mean=sub_mu, cov=sub_cov)) # compute pdf H = H[H >= 1e-9] hyper_volume = np.prod(upper-lower) ce = ((1/Z)*(hyper_volume/(grid_num**(J+1)))*np.sum(-H*np.log(H)) + np.log(Z)) # integrate else: ##### compute conditional Entropy by Monte calro simulation ##### sampleN = 0 samples = np.atleast_2d(sub_mu) while 1: s = np.random.multivariate_normal(sub_mu,sub_cov,1000) sampleN += 1000 ##### check if samples is in the truncated area ##### for j in range(J+1): in_area = (lower[j] <= s[:,j]) & (s[:,j] <= upper[j]) s = s[in_area,:] samples = np.r_[samples,s] if np.shape(samples)[0] < 50000: break H = multivariate_normal.pdf(samples,mean=sub_mu, cov=sub_cov) H = H[H >= 1e-9] ce = (1/Z)*((1/sampleN)*np.sum(-np.log(H))) + np.log(Z) # integrate else: ce pass return ce
def TheoreticalCPA(self, numTraces): """ Returns the DES, CPA Success Rate (calculated using Yunsi Fei's Confusion Analysis formula) for the indicated number of traces """ cov, mean = self.conf.YCovMeanCPA2(self.signal, self.noise, numTraces, correctKey=self.correctSubKey) return mvn.mvnun(self.lower, self.upper, mean.reshape(63), cov)[0]
def bboxes2gt_normal(classes, input_shape, bboxes, grid_division, std_ratio=(6, 6)): """This method :param list classes: list of strings, the names of the classes :param tuple input_shape: shape of the image :param np.ndarray bboxes: list of bboxes in the format class_id, xmin, ymin, xmax, ymax :param grid_division: division in cells of the image (x_axis, y_axis) :param std_ratio: tuple to compute std along the 2 axes, it is calculated as side_length/std_ratio :return: array with the count per class per cell :rtype: np.ndarray """ grids = GTGen.get_grids(input_shape, grid_division) counter = np.zeros((grids.shape[0], len(classes))) for b in range(len(bboxes)): bbox = bboxes[b] # Define mu and covariance matrix mu = np.array([(bbox[3] - bbox[1]) / 2 + bbox[1], (bbox[4] - bbox[2]) / 2 + bbox[2]]) S = np.array([[((bbox[3] - bbox[1]) / std_ratio[0])**2, 0], [0, ((bbox[4] - bbox[2]) / std_ratio[1])**2]]) norm_coeff = mvn.mvnun(np.array(bbox[1:3]), np.array(bbox[3:]), mu, S)[0] if bbox[1] > input_shape[1] or bbox[3] > input_shape[1] or bbox[2] > input_shape[0] or bbox[4] > \ input_shape[0]: raise ValueError( "Bounding Boxes coordinates exceeds the image boundaries") if any(bbox) < 0: raise ValueError("Found negative Bounding Boxes coordinates") for i in range(grids.shape[0]): low, upp = GTUtils.coordinates_intersection(grids[i], bbox[1:]) if low is None: continue counter[i, int(bbox[0])] += mvn.mvnun(low, upp, mu, S)[0] / norm_coeff return counter
def gaussian_testRectangle(): """ Test integral of bvn over polygon Test case: zero mean and unit covariance 2D Gaussian area with [xlim, ylim] is rectangle boundary from 0 to 1 Method used: scipy.stats.mvnmvnun: closed form solution for rectangle bound scipy.integrate.dblquad: double integral """ from scipy.stats import mvn import scipy.integrate as integrate import time # set values mean = np.array([0, 0]) cov = np.eye(2) # mvnun start_mvnun = time.time() lower = np.array([0, 0]) upper = np.array([1, 1]) area_mvnun, _ = mvn.mvnun(lower, upper, mean, cov) time_mvnun = time.time() - start_mvnun # dblquad def pdfDBL(x, y, mean, cov): x1 = x - mean[0] x2 = y - mean[1] detC = cov[0, 0] * cov[1, 1] - cov[0, 1] * cov[1, 0] e = (cov[1, 1] * (x1**2) + (-cov[0, 1] - cov[1, 0]) * x1 * x2 + cov[0, 0] * (x2**2)) / detC return 1 / (2 * np.pi * np.sqrt(abs(detC))) * np.exp(-0.5 * e) start_dblquad = time.time() area_dblquad = integrate.dblquad(lambda x, y: pdfDBL(x, y, mean, cov), lower[0], upper[0], lambda x: lower[1], lambda x: upper[1])[0] time_dblquad = time.time() - start_dblquad # polygon triangulation method start_poly = time.time() poly = np.array([[lower[0], lower[1]], [upper[0], lower[1]], [upper[0], upper[1]], [lower[0], upper[1]]]) area_poly = polyIntegratePdf(poly, mean, cov, eps=1.0e-5) time_poly = time.time() - start_poly # print output print("Test integral over rectangle bound :") print("Scipy mvnun: {:5.6f}".format(area_mvnun), " | time: {:6.9f} s".format(time_mvnun)) print("Scipy dblquad: {:5.6f}".format(area_dblquad), " | time: {:6.9f} s".format(time_dblquad)) print("Polygon integral: {:5.6f}".format(area_poly), " | time: {:6.9f} s".format(time_poly), " \nerror: {:5.8f}".format(abs(area_mvnun - area_poly)))
def match(self, p2): # TODO : incorporate velocities # TODO : consider color[r/g] and type[t/o], when provided p1 = self.as_vec() p2 = p2.as_vec() d = np.abs(ukf_residual(p1, p2)) cov = np.diag(self.SIGMAS[:3]) # assume independent x-y-t p, _ = mvn.mvnun(d, [20, 20, np.pi], np.zeros_like(d), cov) return p * (2**3) # account for quartiles
def normalize(self,mean=None,cov=None,low=None,high=None): """ Calculate normalization term for the truncated MVN. Involves calculation of four CDF terms: P(4) - P(3) - P(2) + P(1) """ #calculate CDF of full region cdf_4,i = mvn.mvnun(self.low_int_limit,high,mean,cov) #calculate CDFs of outside regions cdf_3,i = mvn.mvnun(self.low_int_limit,np.array([high[0],low[1]]),mean,cov) cdf_2,i = mvn.mvnun(self.low_int_limit,np.array([low[0],high[1]]),mean,cov) #calculate CDF of lower-left corner region cdf_1,i = mvn.mvnun(self.low_int_limit,low,mean,cov) reg_prob = (cdf_4 - cdf_3 - cdf_2 + cdf_1) return 1./reg_prob
def compute_univariate_marginal(x, k, sigma, a, b): """Compute the univariate marginal of a normally distributed dataset. Args: x (1D numpy array): The k-th dimension of the dataset. sigma (2D numpy array): The covariance matrix of the density function. k (int): The index of the target dimension. a (1D numpy array): The truncation lower bound. b (1D numpy array): The truncation upper bound. Returns: Fk (1D numpy array): The calculated first moment. """ # Basic initialization N = x.shape[0] D = sigma.shape[0] Fk = np.zeros([N]) # CASE 1: If x = inf or -inf, then Fk = 0 # Find infinity elements idx = ~np.isinf(x) if np.sum(~idx): Fk[~idx] = 0 if (np.sum(idx) == 0): return Fk # CASE 2: Check if the dataset is univariate (D = 1) if D == 1: Fk[idx] = norm.pdf(x[idx], 0, np.sqrt(float(sigma))) return Fk # CASE 3: Consider the case of multivariate dataset (D > 1) o = np.zeros([D], dtype=np.bool) o[k] = True m = ~o # Get the mean and covarance excepted the k-th dimension cmu = (sigma[m, :][:, o] / sigma[o, :][:, o] * x[idx]).T csig = sigma[m, :][:, m] - sigma[m, :][:, o] / sigma[o, :][:, o] *\ sigma[o, :][:, m] # Estimate the value of cumulative distribution function for each dimension cdf = np.zeros([a[idx, :].shape[0], 1]) for i in range(a[idx, :].shape[0]): cdf[i] = mvn.mvnun((a[idx, :][:, m] - cmu)[i], (b[idx, :][:, m] - cmu)[i], np.zeros([D-1]), np.squeeze(csig))[0] # Calculate the the first moment for each dimension Fk[idx] = (norm.pdf(x[idx], 0, np.squeeze(np.sqrt(sigma[o, :][:, o]))).\ reshape([-1, 1]) * cdf).reshape([-1]) return Fk
def computeIndividualProbabilities2(self, x, G, g): self.m_in = G.shape[0] R = np.zeros(self.m_in) for i in range(self.m_in): Sigma_eG = np.dot(np.dot(G[i, :], self.Sigma), G[i, :].transpose()) low = np.array([-100 * np.sqrt(Sigma_eG)]) ineq_margins = np.dot(G[i, :], x) + g[i] p, dummy = mvn.mvnun(low, ineq_margins, 0.0, Sigma_eG) R[i] = p # print "Ineq %d, p(%f<x<%f) = %.2f,\t sigma %f" % (i, low[0], ineq_margins, p, Sigma_eG); return R
def EIBV_1D(threshold, mu, Sig, F, R): ''' :param threshold: :param mu: :param Sig: :param F: sampling matrix :param R: noise matrix :return: EIBV evaluated at every point ''' Sigxi = Sig @ F.T @ np.linalg.solve(F @ Sig @ F.T + R, F @ Sig) V = Sig - Sigxi sa2 = np.diag(V).reshape( -1, 1) # the corresponding variance term for each location IntA = 0.0 for i in range(len(mu)): sn2 = sa2[i] m = mu[i] IntA = IntA + mvn.mvnun(-np.inf, threshold, m, sn2)[0] - mvn.mvnun( -np.inf, threshold, m, sn2)[0]**2 return IntA
def cal_c(): from scipy.stats import mvn import numpy as np low = np.array([0, 0]) upp = np.array([1, 1]) mu = np.array([.5, .5]) sigx = 1 S = np.array([[sigx, 0], [0, sigx]]) scale = 1 p, i = mvn.mvnun(low, upp, mu, S) # print(p, i) return p*scale
def _mvn_un(rho, lower, upper): """ Perform integral of bivariate normal gauss with correlation Integral is performed using scipy's mvn library. :returns float: integral value """ mu = np.array([0., 0.]) S = np.array([[1., rho], [rho, 1.0]]) p, i = mvn.mvnun(lower, upper, mu, S) return p
def ExpectedVarianceUsr(threshold, mu, Sig, F, R): ''' :param threshold: :param mu: :param Sig: :param F: sampling matrix :param R: noise matrix :return: ''' Sigxi = np.dot(Sig, np.dot(F.T, np.linalg.solve(np.dot(F, np.dot(Sig, F.T)) + R, np.dot(F, Sig)))) V = Sig - Sigxi sa2 = np.diag(V).reshape(-1, 1) # the corresponding variance term for each location IntA = 0.0 for i in range(len(mu)): sn2 = sa2[i] sn = np.sqrt(sn2) # the corresponding standard deviation term m = mu[i] # mur = (threshold - m) / sn IntA = IntA + mvn.mvnun(-np.inf, threshold, m, sn2)[0] - mvn.mvnun(-np.inf, threshold, m, sn2)[0] ** 2 return IntA
def verify(self): """ Verify our results against Yunsi Fei's results where she publishes her signal to noise ration and success rate curves """ x = [] yDPA = [] yCPA = [] # Amended Formula which works for i in range(1,200,20): cov, mean = self.conf.YCovMeanDPA(0.0016, 0.0046, i, correctKey=60) yDPA.append(mvn.mvnun(DESSuccess.lower, DESSuccess.upper, mean.reshape(63), cov)[0]) x.append(i) cov, mean = self.conf.YCovMeanCPA2(0.0016, 0.0048, i, correctKey=60) yCPA.append(mvn.mvnun(self.lower, self.upper, mean.reshape(63), cov)[0]) p1 = plot(xest,ySR1est,'--') p2 = plot(x,yCPA) legend(loc='lower right') ylabel('Success Rate') xlabel('Number of Measurements') show()
def delete_features(features, pose, observations, noise, min_visibility, max_visibility, threshold): new_features = [] for f in features: mu = f[0].transpose()[0] xf, yf = mu d = sqrt((xf - pose[0])**2 + (yf - pose[1])**2) if d > min_visibility and d < max_visibility: highest_likelihood = 0 for o in observations: phi = atan2(yf - pose[1], xf - pose[0]) - pose[2] d_noise, phi_noise = noise d_sigma = d_noise[0] * d + d_noise[1] * abs(phi) + d_noise[2] phi_sigma = phi_noise[0]*d + \ phi_noise[1]*abs(phi) + phi_noise[2] sigma = [[d_sigma, 0], [0, phi_sigma]] if abs(o[1] - phi) > abs(o[1] - phi + 2 * pi): phi -= 2 * pi elif abs(o[1] - phi) > abs(o[1] - phi - 2 * pi): phi += 2 * pi d_interval, phi_interval = d_sigma / 10, phi_sigma / 10 if d_sigma == 0 or phi_sigma == 0: if o[0] == d and o[1] == phi: feature_likelihood = 1 else: feature_likelihood = 0 else: feature_likelihood = mvnun( np.array([ o[0] - d_interval / 2.0, o[1] - phi_interval / 2.0 ]), np.array([ o[0] + d_interval / 2.0, o[1] + phi_interval / 2.0 ]), np.array([d, phi]), np.array(sigma))[0] assert feature_likelihood <= 1, feature_likelihood if feature_likelihood > highest_likelihood: highest_likelihood = feature_likelihood if highest_likelihood > threshold: new_features.append(f) else: new_features.append(f) return new_features
def correspondence(features, pose, observation, min_visibility, max_visibility, noise, threshold): if len(features) == 0: return [], [] corresponding_features = len(features) * [0] x, y, theta = pose.transpose()[0] for i, f in enumerate(features): mu = f[0].transpose()[0] xf, yf = mu d = sqrt((xf - x)**2 + (yf - y)**2) phi = atan2(yf - y, xf - x) - theta if d > min_visibility and d < max_visibility: d_noise, phi_noise = noise d_sigma = d_noise[0] * d + d_noise[1] * abs(phi) + d_noise[2] phi_sigma = phi_noise[0] * d + phi_noise[1] * abs( phi) + phi_noise[2] sigma = [[d_sigma, 0], [0, phi_sigma]] if abs(observation[1] - phi) > abs(observation[1] - phi + 2 * pi): phi -= 2 * pi elif abs(observation[1] - phi) > abs(observation[1] - phi - 2 * pi): phi += 2 * pi d_interval, phi_interval = d_sigma / 10, phi_sigma / 10 feature_likelihood = mvnun( np.array([ observation[0] - d_interval / 2.0, observation[1] - phi_interval / 2.0 ]), np.array([ observation[0] + d_interval / 2.0, observation[1] + phi_interval / 2.0 ]), np.array([d, phi]), np.array(sigma))[0] assert feature_likelihood <= 1, "Probability greater than 1 !" if feature_likelihood > threshold: corresponding_features[i] = feature_likelihood corresponding_likelihoods = list(np.sort(np.array(corresponding_features))) corresponding_features = list(np.argsort(np.array(corresponding_features))) corresponding_likelihoods.reverse() corresponding_features.reverse() for i, e in enumerate(corresponding_likelihoods): if e == 0: corresponding_features[i] = None return corresponding_features, corresponding_likelihoods
def phi_func(rho, K): '''Here we define the phi as a function of rho and K''' # construct an array of covariance matrices for each rho COV = np.array([[[1, r], [r, 1]] for r in rho]) # scipy doesn't offer a survival function (i.e. complementary cdf), so we have to build it threshold = np.array([K, K]) upper = np.array([100, 100]) nom_phi = np.array( [mvn.mvnun(threshold, upper, mean, cov)[0] for cov in COV]) return nom_phi / (1 - norm.cdf(K))
def funcion_margina1_loglike2(x, y, qq1, qq2, rho, maxpts1, abseps1): auxiliar=qq1*qq2*rho; Sigma1=np.array([[1,auxiliar],[auxiliar,1]]); #test = -1*np.inf; test = -30; low=np.array([test,test]); uppaux=np.array([qq1*x,qq2*y]); #phi2=mvstdnormcdf(low,uppaux,Sigma1,maxpts1,abseps1,abseps1); mu=np.array([0,0]); phi2,inform = mvn.mvnun(low,uppaux,mu,Sigma1); if phi2<=abseps1: phi2=abseps1; return np.log(phi2);
def calculo_margina1_v4(x, y, qq1, qq2, rho, factor, maxpts1, abseps1): Sigma1=np.array([[1,rho],[rho,1]]); #test = np.inf; test=30; x0=-1*x; y0=-1*y; lowaux=np.array([-1*test,-1*test]); uppaux=np.array([x0,y0]); mu=np.array([0,0]); phi2,inform = mvn.mvnun(lowaux,uppaux,mu,Sigma1); #phi2=mvstdnormcdf(lowaux,uppaux,Sigma1,maxpts1,abseps1,abseps1); if phi2<=abseps1: phi2=abseps1; v1=x-(1/phi2)*((stats.norm._pdf(x0)*(stats.norm._cdf((y0+rho*x)*factor)))+rho*(stats.norm._pdf(y0)*(stats.norm._cdf((x0+rho*y)*factor)))); v2=y-(1/phi2)*((stats.norm._pdf(y0)*(stats.norm._cdf((x0+rho*y)*factor)))+rho*(stats.norm._pdf(x0)*(stats.norm._cdf((y0+rho*x)*factor)))); return v1, v2;
def marginal_score2(p, x, x1, x2, rho1, qqq1, qqq2, ggg1, ggg2, maxpts1, abseps1): summa_rho=np.zeros((1, 1)); summa1=np.zeros((1, p)); summa2=np.zeros((1, p)); #test = -1*np.inf; test = -30; low=np.array([test,test]); mu=np.zeros((2,1)); upp=np.zeros((2,1)); auxiliar=qqq1*qqq2*rho1; Sigma1=np.array([[1,auxiliar],[auxiliar,1]]); upp=np.array([qqq1*x1,qqq2*x2]); phi2,inform = mvn.mvnun(low,upp,mu,Sigma1); #phi2=mvstdnormcdf(low,upp,Sigma1,maxpts1,abseps1,abseps1); if phi2<=abseps1: phi2=abseps1; summa1=(((qqq1*ggg1)/phi2)*(x)); summa2=(((qqq2*ggg2)/phi2)*(x)); summa_rho=((qqq1*(qqq2*normpdf(upp, mu, Sigma1, 2)))/phi2); return summa1, summa2, summa_rho;
def gauss_bivar(x, y, rho): return mvnun(-999 * np.ones((2)), (x, y), (0, 0), np.array([[1, rho], [rho, 1]]))[0]
if(np.any(N>0)): i = np.argmin(N[N>0]) p_emp_IS = M[N>0][i] erreur_emp_IS = N[N>0][i] var_emp_IS = P[N>0][i] C.append(p_emp_IS) D.append(erreur_emp_IS) F.append(var_emp_IS) low = epsilon * np.ones(npoint) upp = 100 * np.ones(npoint) P = [] for distance in np.linspace(0,8,100): mean = distance * np.ones(npoint) p,i = mvn.mvnun(low,upp,mean,cov) P.append(1-p) latexify() plt.figure() plt.grid(True) plt.semilogy(np.linspace(0, 8, 20), A, 'rx', label = 'MC') plt.semilogy(np.linspace(0, 8, 20), C, 'b.', label ='IS') plt.semilogy(np.linspace(0, 8, 100), P, 'k', label ='num') plt.xlabel("Separation distance") plt.ylabel("Probability") plt.legend() plt.savefig('Outputs/Script_8_ISmc_1.pdf', bbox_inches='tight') plt.figure() plt.grid(True)
def mutual_proximity_gauss(D: np.ndarray, metric: str = "distance", test_set_ind: np.ndarray = None, verbose: int = 0): """Transform a distance matrix with Mutual Proximity (normal distribution). Applies Mutual Proximity (MP) [1]_ on a distance/similarity matrix. Gauss variant assumes dependent normal distributions (VERY SLOW). The resulting second. distance/similarity matrix should show lower hubness. Parameters ---------- D : ndarray - ndarray: The ``n x n`` symmetric distance or similarity matrix. metric : {'distance', 'similarity'}, optional (default: 'distance') Define, whether matrix `D` is a distance or similarity matrix. test_sed_ind : ndarray, optional (default: None) Define data points to be hold out as part of a test set. Can be: - None : Rescale all distances - ndarray : Hold out points indexed in this array as test set. verbose : int, optional (default: 0) Increasing level of output (progress report). Returns ------- D_mp : ndarray Secondary distance MP gauss matrix. References ---------- .. [1] Schnitzer, D., Flexer, A., Schedl, M., & Widmer, G. (2012). Local and global scaling reduce hubs in space. The Journal of Machine Learning Research, 13(1), 2871–2902. """ # Initialization n = D.shape[0] log = Logging.ConsoleLogging() # Checking input IO._check_distance_matrix_shape(D) IO._check_valid_metric_parameter(metric) if metric == "similarity": self_value = 1 else: # metric == 'distance': self_value = 0 if issparse(D): log.error("Sparse matrices not supported by MP Gauss.") raise TypeError("Sparse matrices not supported by MP Gauss.") if test_set_ind is None: train_set_ind = slice(0, n) else: train_set_ind = np.setdiff1d(np.arange(n), test_set_ind) # Start MP D = D.copy() np.fill_diagonal(D, self_value) # np.fill_diagonal(D, np.nan) mu = np.mean(D[train_set_ind], 0) sd = np.std(D[train_set_ind], 0, ddof=0) # =========================================================================== # mu = np.nanmean(D[train_set_ind], 0) # sd = np.nanstd(D[train_set_ind], 0, ddof=0) # =========================================================================== # Code for the BadMatrixSigma error [derived from matlab] # =========================================================================== # eps = np.spacing(1) # epsmat = np.array([[1e5 * eps, 0], [0, 1e5 * eps]]) # =========================================================================== D_mp = np.zeros_like(D) # MP Gauss for i in range(n): if verbose and ((i + 1) % 1000 == 0 or i + 1 == n): log.message("MP_gauss: {} of {}.".format(i + 1, n)) for j in range(i + 1, n): # =================================================================== # mask = np.isnan(D[[i, j], :]) # D_mask = np.ma.array(D[[i, j], :], mask=mask) # c = np.ma.cov(D_mask, ddof=0) # =================================================================== c = np.cov(D[[i, j], :], ddof=0) x = np.array([D[i, j], D[j, i]]) m = np.array([mu[i], mu[j]]) low = np.tile(np.finfo(np.float32).min, 2) p12 = mvn.mvnun(low, x, m, c)[0] # [0]...p, [1]...inform if np.isnan(p12): # =============================================================== # power = 7 # while np.isnan(p12): # c += epsmat * (10**power) # p12 = mvn.mvnun(low, x, m, c)[0] # power += 1 # log.warning("p12 is NaN: i={}, j={}. Increased cov matrix by " # "O({}).".format(i, j, epsmat[0, 0]*(10**power))) # =============================================================== p12 = 0.0 log.warning("p12 is NaN: i={}, j={}. Set to zero.".format(i, j)) if metric == "similarity": D_mp[i, j] = p12 else: # distance p1 = norm.cdf(D[i, j], mu[i], sd[i]) p2 = norm.cdf(D[i, j], mu[j], sd[j]) D_mp[i, j] = p1 + p2 - p12 D_mp += D_mp.T np.fill_diagonal(D_mp, self_value) return D_mp
pmf_dscrlognorm = np.insert(np.diff(cdf_dscrlognorm), 0, cdf_dscrlognorm[0]) h_line_true = plt.plot(yy, pmf_dscrlognorm, '--', c=np.ones(3)*1.0*i/N_interation) # plt.legend() # plt.plot(importance_ratio) plt.plot(xs_rs, xs_weight, '.') # test mvn cdf efficency: from scipy.stats import mvn, multivariate_normal for M in 2**np.arange(9): tic=time.time() temp = np.log2(mvn.mvnun(np.ones(M)*0,np.ones(M)*10,np.zeros(M),np.eye(M))) toc=time.time() h_ind = plt.loglog(M, toc-tic, 'ok') for M in 2**np.arange(9): tic=time.time() temp = np.log2(mvn.mvnun(np.ones(M)*0,np.ones(M)*10,np.zeros(M),np.eye(M)*0.8+0.2)) toc=time.time() h_cor = plt.loglog(M, toc-tic, '+r') for M in 2**np.arange(9): N = 10000 tic = time.time() dist = multivariate_normal(mean=np.zeros(M), cov=np.eye(M)*0.8+0.2) sps = np.random.rand(N, M)*10 logpdf_unnmlz = dist.logpdf(x=sps) temp = np.mean(dist.pdf(x=sps)*10**M) toc= time.time()
def ml_map_by_cdf(cov, number_of_bins, mod_size, number_of_modulos=7, plots=False, debug=False): ''' :param cov: :param number_of_bins: :param mod_size: :param number_of_modulos: number of multiply for each side. at 2 we will get multiply of 5X5 :param plots: :param debug: :return: ''' import numpy as np import pandas as pd import itertools from scipy.stats import multivariate_normal pd.set_option("display.max_columns", 1000) # don’t put … instead of multi columns pd.set_option('expand_frame_repr', False) # for not wrapping columns if you have many pd.set_option("display.max_rows", 30) pd.set_option('display.max_colwidth', 1000) rounding=10 df=bins_edges_with_duplication_of_modulo(rounding=rounding, mod_size=mod_size, number_of_bins=number_of_bins, number_of_modulos=number_of_modulos) '''now df has pixels with center and edges for each''' if plots: print('doing cdf') if 1: rv = multivariate_normal([0, 0], cov) if 1: if plots: print('starting product') cdf=pd.DataFrame(list(itertools.product(*[df[['x_high','x_low']].stack().unique().tolist()] * 2)), columns=list('xy')).round(rounding) # we do all the unique stuff instead of taking bin_edges because we have floating point issue if plots: print('done product') '''now calcuating cdf per pixel. each pixel has shared edges with it's neighbors so we will do the cdf offline and then merge it back''' if 0: # cannot do this because at the right upper the cdf will be 1 and the pdf 0... cdf['cdf']=0 # for saving cdf calculation, that is slower... we will only calculate cdf on ones that their pdf is high cdf['pdf']=multivariate_normal.pdf(cdf[['x', 'y']].values, mean=[0, 0], cov=cov) cdf['pdf']=1 cdf.loc[cdf.pdf>1e-10, 'cdf']=rv.cdf(cdf[cdf.pdf>1e-10][['x', 'y']].values) if plots: print('doing cdf on {cdf:,} from total of {total:,} rows'.format(cdf=cdf[cdf.pdf>1e-10].shape[0], total=cdf.shape[0])) cdf=cdf.drop('pdf', axis=1) cdf['cdf']=rv.cdf(cdf[['x', 'y']].values) if plots: print('starting merging results') if 0: # merging is slow, we better use join df['high_cdf'] = pd.merge(df[['x_high', 'y_high']].round(rounding), cdf.round(rounding), left_on=['x_high', 'y_high'], right_on=list('xy'), how='left').cdf.values df['low_cdf'] = pd.merge(df[['x_low', 'y_low']].round(rounding), cdf.round(rounding), left_on=['x_low', 'y_low'], right_on=list('xy'), how='left').cdf.values df['left_cdf'] = pd.merge(df[['x_low', 'y_high']].round(rounding), cdf.round(rounding), left_on=['x_low', 'y_high'], right_on=list('xy'), how='left').cdf.values df['down_cdf'] = pd.merge(df[['x_high', 'y_low']].round(rounding), cdf.round(rounding), left_on=['x_high', 'y_low'], right_on=list('xy'), how='left').cdf.values else: def merging(left, right): ''' pd.merge is slow, you better use join and dont do sort_index. it takes time more than it helps join right should be bigger than left, becaues left is only upper lower etc. :param left: :param right: :return: ''' col = list('xy') left.columns = col left.set_index(col, inplace=True) if 0: m=left.index.to_frame().reset_index(drop=True).merge(right.index.to_frame().reset_index(drop=True), on=list('xy'), indicator=True, how='outer', suffixes=['','_']) if not m[m._merge=='left'].sort_values(by=['x','y']).empty: print(m[m._merge=='left'].sort_values(by=['x','y'])) print(m._merge.value_counts()) # print(left.join(right, how='outer').loc[left.join(right, how='outer').isna().cdf.values]) # print(pd.concat([left.index.to_frame(),right.index.to_frame()]).drop_duplicates(keep=False).reset_index(drop=True).sort_values(by=['x','y'])) merged=left.join(right, how='left').cdf if merged.isna().sum(): print('we have nan at the merged step') return merged.values cdf=cdf.set_index(list('xy')) df['high_cdf'] = merging(df[['x_high', 'y_high']], cdf) df['low_cdf'] = merging(df[['x_low', 'y_low' ]], cdf) df['left_cdf'] = merging(df[['x_low', 'y_high']], cdf) df['down_cdf'] = merging(df[['x_high', 'y_low' ]], cdf) if df.applymap(np.isnan).sum().sum(): print('WARNING - we have nan values after merging cdf values, and it should be!') print('found {nans:,} nans from total of {total:,} cells'.format(nans=df.applymap(np.isnan).sum().sum(), total=df.size)) if plots: print('done merging results') else: # more time, because we calculate the same dot 4 times df['high_cdf']=rv.cdf(df[['x_high', 'y_high']].values) df['low_cdf']=rv.cdf(df[['x_low', 'y_low']].values) df['left_cdf']=rv.cdf(df[['x_low', 'y_high']].values) df['down_cdf']=rv.cdf(df[['x_high', 'y_low']].values) df['bin_cdf']=df.high_cdf-df.left_cdf-df.down_cdf+df.low_cdf else: # at bins >10 it's slower because it's not vector operation from scipy.stats import mvn df['bin_cdf']=df.apply(lambda row:mvn.mvnun(row[['x_low','y_low']].values,row[['x_high','y_high']].values,[0,0],cov.tolist())[0], axis=1) if plots: print('done cdf') if plots: print('''finding best group at the main modulo''') probability_shifts = df.pivot_table(index=['x_modulo_shifts', 'y_modulo_shifts', 'modulo_group_number'], columns=['x_mod', 'y_mod'], values='bin_cdf').idxmax().unstack() try: probability_map=probability_shifts.applymap(lambda x:x[2]) x_shift=probability_shifts.applymap(lambda x:x[0]) y_shift=probability_shifts.applymap(lambda x:x[1]) except: print(probability_shifts) print(probability_shifts.nunique()) print('unknown error. for some reason instead of dictionary, we have float at the content. maybe its when we have only 1 sample?. return') return dict() pvt=df.pivot_table(index='modulo_group_number', columns=['x_mod', 'y_mod'], values='bin_cdf') probability_map_max = pvt.max().unstack() if pvt.count().unstack().std().std(): print('WARNING - probably map modulo didnt worked correctly') # df.modulo_group_number.value_counts().sort_values() if pvt.applymap(np.isnan).sum().sum(): print('WARNING - you have nan cells after the pivot, it means that each group has its own x_mod value, which cannot be, unless you have float precision issue') if debug: print('group_occurrence') group_occurrence = 100 * probability_map.stack().value_counts() / probability_map.size group_occurrence = group_occurrence.to_frame('percentages').reset_index().rename(columns=dict(index='modulo_group_number')) group_occurrence = pd.merge(modulo_group, group_occurrence, on='modulo_group_number', how='right').sort_values('percentages', ascending=False) print(group_occurrence) if plots: print('starting plotting') import plotly as py import cufflinks if debug: if 0: original_heatmap=df[['x_center','y_center','bin_cdf']][(df.x_center==df.x_mod)&(df.y_center==df.y_mod)].set_index(['x_center','y_center']).unstack() else: original_heatmap=df[['x_center','y_center','bin_cdf']].set_index(['x_center','y_center']).unstack() original_heatmap.columns=original_heatmap.columns.get_level_values(1) fig = original_heatmap.figure(kind='heatmap', colorscale='Reds') # fig = original_heatmap.figure(kind='surface', colorscale='Reds') py.offline.plot(fig, filename='original_heatmap.html') if 1: fig=probability_map.figure(kind='heatmap', colorscale='Reds') py.offline.plot(fig, filename='probability_map.html') fig = probability_map_max.figure(kind='heatmap', colorscale='Reds') py.offline.plot(fig, filename='probability_map_max.html') else: probability_map=df.pivot_table(index='modulo_group_number', columns=['x_mod', 'y_mod'], values='bin_cdf').idxmax().sort_values().to_frame('modulo_group_number').astype(str).reset_index() fig=probability_map.figure(kind='scatter', x='x_mod', y='y_mod', categories='modulo_group_number') py.offline.plot(fig) ml=x_shift.stack().to_frame('x_shift').join(y_shift.stack().to_frame('y_shift')) return ml
def gauss_bivar(x, y, rho): return mvnun((-999, -999), (x, y), (0, 0), ((1, rho), (rho, 1)))[0]
def Cdf(self,x): # cdf = integral of pdf from lower limit to upper limit (x) # Reference http://www.nhsilbert.net/source/2014/04/multivariate-normal-cdf-values-in-python/ return mvn.mvnun(self.lowerLimit63, x, self.meanNorm63, self.covNorm63)
def tripleRectIntegrate2(bounds, mu, var): lower, upper = tuple(map(tuple, zip(*bounds))) value, inform = mvn.mvnun(lower, upper, mu, np.diag(var)) return value
i = np.argmin(A[A>0]) p_emp_IS = B[A>0][i] erreur_emp_IS = A[A>0][i] return (p_emp_IS, erreur_emp_IS, Si[A>0][i]) return (0,0,0) epsilon = 0.1 # Choc distance npoint = 20 for distance in np.linspace(4, 10, 4): for Nsim in [100, 1000, 100000]: text_file = open("OutFiles/Output_IS_linear_%s_%s.csv" % (distance,Nsim), "w") text_file.write("Distance entre avions : %s \n" % distance) text_file.write("Nombre de simulations : %s \n" % Nsim) text_file.write("Distance, Probability, Error, Relative error, mu \n \n") for i in range(23): prob, err, mu = IS(distance, Nsim, npoint) text_file.write("%s, %.3E, %.3E, %.2f\%%, %s, %.3f \n" % (distance, decimal.Decimal(prob), decimal.Decimal(err),100*err/prob, Nsim, mu)) low = epsilon * np.ones(npoint) upp = 100 * np.ones(npoint) mean = moyenne(npoint, distance) cov = covariance(npoint) p,i = mvn.mvnun(low,upp,mean,cov) # p prob toutes > 0.1 real = 1-p # prob qu'il existe une < 0.1 text_file.write("Real value : %s" % real) text_file.close()