def figure_6_14(): """Reproduces figure 6.14 in ESLii displaying a density estimate for sbp levels in chd/no-chd groups using a Gaussian kernel density estimate """ sa = eslii.read_sa_heart_data() sbp = sa["sbp"] sbp_chd = sa[sa["chd"] == 1]["sbp"].copy() sbp_chd.sort() sbp_no_chd = sa[sa["chd"] == 0]["sbp"].copy() sbp_no_chd.sort() kde_chd = KernelDensity(kernel='gaussian', bandwidth=7.5).fit( sbp_chd.reshape(len(sbp_chd), 1)) chd_log_dens = kde_chd.score_samples(sbp_chd.reshape((len(sbp_chd), 1))) plt.subplot(121) plt.plot(sbp_chd, np.exp(chd_log_dens), label="CHD") kde_no_chd = KernelDensity(kernel='gaussian', bandwidth=7.5).fit( sbp_no_chd.reshape(len(sbp_no_chd), 1)) no_chd_log_dens = kde_no_chd.score_samples( sbp_no_chd.reshape((len(sbp_no_chd), 1))) plt.plot(sbp_no_chd, np.exp(no_chd_log_dens), label="no CHD") plt.legend(loc='best') sbp_range = np.linspace(min(sbp), max(sbp), 100).reshape((100, 1)) chd_dens = np.exp(kde_chd.score_samples(sbp_range)) no_chd_dens = np.exp(kde_no_chd.score_samples(sbp_range)) p_chd = float(len(sbp_chd))/(len(sbp_chd) + len(sbp_no_chd)) posterior_est = [p_chd * chd_dens[i] / (p_chd * chd_dens[i] + (1 - p_chd) * no_chd_dens[i]) for i in range(len(sbp_range))] plt.subplot(122) plt.plot(sbp_range, posterior_est) plt.show()
def cistrans(args): cob = co.COB(args.cob) if args.out == None: args.out = '{}_cistrans'.format(cob.name) # np.newaxis adds an empty axis in that position of the slice # the sklearn module requires the values to be in the rows: # http://scikit-learn.org/stable/auto_examples/neighbors/plot_kde_1d.html cis = cob.coex \ .score[cob.coex.distance <= args.cis_distance]\ .values[:,np.newaxis] trans = cob.coex\ .score[np.isinf(cob.coex.distance)]\ .values[:,np.newaxis] X_plot = np.linspace(-10,10,1000)[:,np.newaxis] print( 'Found {:,} cis interactions and {:,} trans interactions'.format( cis.shape[0], trans.shape[0] )) # Fit the kernel kd=KernelDensity(bandwidth=0.2) kd.fit(cis) cis_kde = np.exp(kd.score_samples(X_plot)) plt.fill(X_plot,cis_kde,alpha=0.5,label='Cis Interactions') # Fit the trans kd.fit(trans[0:50000]) trans_kde = np.exp(kd.score_samples(X_plot)) plt.fill(X_plot,trans_kde,alpha=0.5,label='Trans Interactions') plt.legend() plt.title('Cis vs Trans Density: {}'.format(cob.name)) # Calculate the mann whitney U test u,pval = sp.stats.mannwhitneyu(cis[:,0],trans[:,0]) print('P-val: {}'.format(pval)) plt.savefig(args.out+'.png')
def plot_samples(self, folder = '', title = ''): print("Saving plots...") numbins = int(8 * math.log(len(self.samples[self.store_list[0]]))) for p in self.store_list: if p == 'D': continue samples = np.array([self.samples[p]]).T a = np.min(samples) b = np.max(samples) band = 0.1 * (b-a + 0.001) kde = KD(kernel='gaussian', bandwidth=band).fit(samples) n, bins, patches = plt.hist(self.samples[p], numbins, normed=1) log_dens = kde.score_samples(np.array([bins]).T) plt.plot(bins, np.exp(log_dens), 'r-') MAP = self.get_MAP(kde, a, b) self.params['MAP'][p] = MAP plt.plot([MAP], np.exp(kde.score_samples([MAP])), 'go') plt.title(title + " MAP estimate: " + str(MAP)) plt.ylabel("Posterior(" + p + ")") plt.xlabel(p) x1,x2,y1,y2 = plt.axis() plt.axis((-3,3,y1,y2)) if p == 'L' or p == 'T': plt.axis((0,1,y1,y2)) plt.savefig(folder + p + "_" + title) plt.clf() self.params['MAP']['D'] = self.params['D'] if not self.bkt: print("Working on difficulty params...") p = 'D' data = np.array(self.samples[p]) for j in range(self.data['num_problems']): samples = np.array([data[:,j]]).T #print samples a = np.min(samples) b = np.max(samples) band = 0.1 * (b-a + 0.001) kde = KD(kernel='gaussian', bandwidth=band).fit(samples) n, bins, patches = plt.hist(samples, numbins, normed=1) log_dens = kde.score_samples(np.array([bins]).T) plt.plot(bins, np.exp(log_dens), 'r-') MAP = self.get_MAP(kde, a, b) self.params['MAP']['D'][j] = MAP plt.plot([MAP], np.exp(kde.score_samples([MAP])), 'go') plt.title(title + " MAP estimate: " + str(MAP)) plt.ylabel("Posterior(" + p + ")") plt.xlabel("Problem " + str(j)) x1,x2,y1,y2 = plt.axis() plt.axis((-3,3,y1,y2)) plt.savefig(folder + "Difficulty/problem" + str(j) + "_" + title) plt.clf() print("Plots saved!")
class TwoClassKDE(object): """Class for Kernel Density Estimator on two labels. Likelihood ratio at a point is ratio of class-1 likelihood estimate to class-0 likelihood estimate, times the class odds, where this is calculated as the posterior mean estimate under Beta(1, 1) prior, given the observations. If no points are observed for one of the classes, a default (improper) uniform prior is assumed for that class. """ def __init__(self, **kwargs): """Takes same parameters as KernelDensity estimator.""" self.kde0 = KernelDensity(**kwargs) self.kde1 = KernelDensity(**kwargs) def fit(self, X, y): """Fits KDE models on the data. X is array of data points, y is array of 0-1 labels.""" y = np.asarray(y, dtype = int) self.n0, self.n1 = (y == 0).sum(), (y == 1).sum() assert (self.n0 + self.n1 == len(y)), "y must be vector of 1's and 0's." X0, X1 = X[y == 0], X[y == 1] if (self.n0 > 0): self.kde0.fit(X0) if (self.n1 > 0): self.kde1.fit(X1) def fit_with_optimal_bandwidth(self, X, y, gridsize = 101, dynamic_range = 100, cv = 10, verbose = 0, n_jobs = 1): """Determines optimal bandwidth using the following strategy: For each subset (0 or 1) of the dataset, 1) set b = 1.06 * sigma * n^(-1/5), the Silverman's rule of thumb estimate for the optimal bandwidth. sigma is the sample standard deviation of the samples after zero-centering the columns (note: ideally each column will have comparable variance), 2) set up a grid (of size gridsize) of bandwidth values to try, ranging from b / alpha to b * alpha in geometric progression, where alpha = sqrt(dynamic_range), 3) compute average likelihood of the estimator on the data using cv-fold cross-validation, 4) select the bandwidth with the highest likelihood.""" y = np.asarray(y, dtype = int) self.n0, self.n1 = (y == 0).sum(), (y == 1).sum() assert (self.n0 + self.n1 == len(y)), "y must be vector of 1's and 0's." X0, X1 = X[y == 0], X[y == 1] if (self.n0 > 0): log_b0 = np.log(1.06) + np.log((X0 - X0.mean(axis = 0)).std()) - 0.2 * np.log(self.n0) grid0 = GridSearchCV(self.kde0, {'bandwidth' : np.exp(np.linspace(log_b0 - 0.5 * np.log(dynamic_range), log_b0 + 0.5 * np.log(dynamic_range), gridsize))}, cv = cv, verbose = verbose, n_jobs = n_jobs) grid0.fit(X0) self.kde0 = grid0.best_estimator_ if (self.n1 > 0): log_b1 = np.log(1.06) + np.log((X1 - X1.mean(axis = 0)).std()) - 0.2 * np.log(self.n1) grid1 = GridSearchCV(self.kde1, {'bandwidth' : np.exp(np.linspace(log_b1 - 0.5 * np.log(dynamic_range), log_b1 + 0.5 * np.log(dynamic_range), gridsize))}, cv = cv, verbose = verbose, n_jobs = n_jobs) grid1.fit(X1) self.kde1 = grid1.best_estimator_ def get_params(self, **kwargs): return self.kde0.get_params(**kwargs) def set_params(self, **params): self.kde0.set_params(**params) self.kde1.set_params(**params) return self def score_samples(self, X): """Evaluate the density model on the data. Returns vector of log-likelihood ratios of class 1 over class 0.""" p1_est = (self.n1 + 1) / (self.n0 + self.n1 + 2) class_log_odds = np.log(p1_est) - np.log(1 - p1_est) scores0 = self.kde0.score_samples(X) if (self.n0 > 0) else np.zeros(len(X), dtype = float) scores1 = self.kde1.score_samples(X) if (self.n1 > 0) else np.zeros(len(X), dtype = float) return scores1 - scores0 + class_log_odds def score(self, X, y = None): """Compute the overall log-likelihood ratio under the model.""" return self.score_samples(X).sum() def predict_proba(self, X): """Probability estimates.""" scores = self.score_samples(X) p0s = 1 / (1 + np.exp(scores)) return np.array([p0s, 1 - p0s]).transpose() def predict_log_proba(self, X): """Log of probability estimates.""" return np.log(self.predict_proba(X))
def pdf_estimate(images, labels, W, method, t): """ Uses kernel density extimation to the compute the pdf of neural activation data. Args: images (numpy array): input images labels (numpy array): input labels associated with the neuron activations W (numpy array): weights of the hidden neurons method (str): method to approximate the pdf t (float): temperature of the softmax when then network was trained returns: (list of regressor or kde objects): list of marginal pdfs (regressor or kde object): pdf (numpy array): labels of the data points used to compute the pdf (useful to compute prior) """ classes = np.unique(labels) n_classes = len(np.unique(labels)) n_trials = len(labels) """ computes the activation of the hidden neurons for the given input images """ activ = ex.propagate_layerwise(images, W, t=t) n_subsample = 1000 #number of data points to use to compute the pdf in the 'subsample' and 'fit' methods subsample_idx = np.random.choice(n_trials, size=n_subsample, replace=False) activ_subs = activ[subsample_idx, :] n_train_fit = 500 #number of data point to use to fit the pdf in the 'fit' method train_fit_idx = np.random.choice(n_trials, size=n_train_fit, replace=False) activ_fit = activ[train_fit_idx, :] if method=='full': pdf_labels = np.copy(labels) pdf_evidence = KernelDensity(bandwidth=5e-1, kernel='gaussian', rtol=1e-100).fit(activ) pdf_marginals = [] for c in classes: pdf_marginals.append(KernelDensity(bandwidth=5e-1, kernel='gaussian', rtol=1e-100).fit(activ[pdf_labels==c])) if method=='subsample': pdf_labels = labels[subsample_idx] pdf_evidence = KernelDensity(bandwidth=5e-1, kernel='gaussian', rtol=1e-100).fit(activ_subs) pdf_marginals = [] for c in classes: pdf_marginals.append(KernelDensity(bandwidth=5e-1, kernel='gaussian', rtol=1e-100).fit(activ_subs[pdf_labels==c])) if method=='fit': pdf_labels = labels[subsample_idx] pdf_evidence_full = KernelDensity(bandwidth=5e-1, kernel='gaussian', rtol=1e-100).fit(activ_subs) pdf_evidence = KNeighborsRegressor().fit(activ_fit, pdf_evidence_full.score_samples(activ_fit)) pdf_marginals = [] for c in classes: pdf_marginal_full = KernelDensity(bandwidth=5e-1, kernel='gaussian', rtol=1e-100).fit(activ_subs[pdf_labels==c]) pdf_marginals.append(KNeighborsRegressor().fit(activ_fit, pdf_marginal_full.score_samples(activ_fit))) return pdf_marginals, pdf_evidence, pdf_labels
def initialize_optimization_plot(self): if self.dataset is None: print("Set data first before initializing plot options!") return if self.parameters is None: print("Parameter needs to be set!") return ab_list = [ AB_INDICES['A-A'], AB_INDICES['C-C'], AB_INDICES['E-R'], AB_INDICES['R-E'], AB_INDICES['K-E'], AB_INDICES['E-E'], AB_INDICES['K-K'], AB_INDICES['K-R'], AB_INDICES['V-I'], AB_INDICES['I-L'], AB_INDICES['S-T'], AB_INDICES['S-S'], AB_INDICES['K-P'], AB_INDICES['N-N'], AB_INDICES['W-W'], AB_INDICES['G-F'] ] couplings_contacts, couplings_noncontacts, avg_lambda_pair = self.dataset.get_decoy_set(size=self.size_evaluationset) self.evaluation_set['contact'] = np.array(couplings_contacts).transpose() self.evaluation_set['bg'] = np.array(couplings_noncontacts).transpose() bandwidth = 0.01 self.evaluation_set_kde = {} self.evaluation_set_kde['x_grid'] = np.linspace(-0.5, 0.5, 500) self.evaluation_set_kde['contact'] = {} self.evaluation_set_kde['bg'] = {} # kernel density estimate for couplings wijab for ab in ab_list: kde_contact = KernelDensity(kernel='gaussian', bandwidth=bandwidth).fit(self.evaluation_set['contact'][ab].reshape(-1, 1)) kde_bg = KernelDensity(kernel='gaussian', bandwidth=bandwidth).fit(self.evaluation_set['bg'][ab].reshape(-1, 1)) ### add empirical distribution for example data points self.evaluation_set_kde['contact'][ab] = np.exp(kde_contact.score_samples(self.evaluation_set_kde['x_grid'].reshape(-1, 1))) self.evaluation_set_kde['bg'][ab] = np.exp(kde_bg.score_samples(self.evaluation_set_kde['x_grid'].reshape(-1, 1))) #sample points according to regularizer std_dev = np.sqrt(1.0/avg_lambda_pair) regularizer = np.random.normal(scale=std_dev, size=10000) kde_reg = KernelDensity(kernel='gaussian', bandwidth=0.1).fit(regularizer.reshape(-1, 1)) self.evaluation_set_kde['regularizer'] = np.exp(kde_reg.score_samples(self.evaluation_set_kde['x_grid'].reshape(-1, 1)))
def test1(): X = [[1], [2], [4], [3], [2], [8], [8], [9], [10], [12], [11], [9]] kde = KernelDensity(kernel='gaussian', bandwidth=0.4).fit(X) scores = kde.score_samples(X) for x in xrange(len(scores)): scores[x] = math.exp(scores[x]) print scores Y = [[1], [2], [2], [1], [5], [6], [6], [7], [9], [10], [8], [7]] density = kde.score_samples(Y) for x in xrange(len(density)): density[x] = math.exp(density[x]) print density
def get_P_binary_v_tot(proj_sep, delta_v_tot, num_sys=100000): """ This function calculates the probability of a random star having the observed proper motion Parameters ---------- proj_sep : float Projected separation between two stars delta_v_tot : float Total velocity difference between two stars Returns ------- P(proj_sep, delta_v_tot) : float Probability that angular separation, pm+RV difference is due to a genuine binary """ # Catalog check global binary_set if binary_set is None: generate_binary_set(num_sys=num_sys) # Use a Gaussian KDE global binary_v_tot_kde # We work in log space for the set of binaries if binary_v_tot_kde is None: kwargs = {'kernel':'tophat'} binary_v_tot_kde = KernelDensity(bandwidth=0.1, **kwargs) binary_v_tot_kde.fit( np.array([np.log10(binary_set['proj_sep']), np.log10(binary_set['delta_v_tot'])]).T ) if isinstance(delta_v_tot, np.ndarray) and isinstance(proj_sep, np.ndarray): values = np.array([np.log10(proj_sep), np.log10(delta_v_tot)]).T prob_binary = np.exp(binary_v_tot_kde.score_samples(values)) elif isinstance(delta_v_tot, np.ndarray): values = np.array([np.log10(proj_sep)*np.ones(len(delta_v_tot)), np.log10(delta_v_tot)]).T prob_binary = np.exp(binary_v_tot_kde.score_samples(values)) else: prob_binary = np.exp(binary_v_tot_kde.score_samples([np.log10(proj_sep), np.log10(delta_v_tot)])) # Convert back from log10-space to linear-space # the log(10) terms convert from log10 to ln prob_binary = prob_binary / (proj_sep*np.log(10.)) / (delta_v_tot*np.log(10.)) return prob_binary
def plot_agglomerative(): from sklearn.datasets import make_blobs from sklearn.cluster import AgglomerativeClustering from sklearn.neighbors import KernelDensity import matplotlib.pyplot as plt import numpy as np import pandas as pd m = 16 k = 3 X, y = make_blobs(n_samples= m, n_features=2, centers=k, cluster_std=1.3, random_state = 2255) agg = AgglomerativeClustering(n_clusters=3) eps = X.std() / 2. x_min, x_max = X[:, 0].min() - eps, X[:, 0].max() + eps y_min, y_max = X[:, 1].min() - eps, X[:, 1].max() + eps xx, yy = np.meshgrid(np.linspace(x_min, x_max, 100), np.linspace(y_min, y_max, 100)) gridpoints = np.c_[xx.ravel().reshape(-1, 1), yy.ravel().reshape(-1, 1)] ax = plt.gca() for i, x in enumerate(X): ax.text(x[0] + .1, x[1], "%d" % i, horizontalalignment='left', verticalalignment='center') ax.scatter(X[:, 0], X[:, 1], s=20, c='grey') ax.set_xticks(()) ax.set_yticks(()) for i in range((m-1)): agg.n_clusters = X.shape[0] - i agg.fit(X) bins = np.bincount(agg.labels_) for cluster in range(agg.n_clusters): if bins[cluster] > 1: points = X[agg.labels_ == cluster] other_points = X[agg.labels_ != cluster] kde = KernelDensity(bandwidth= 0.9).fit(points) scores = kde.score_samples(gridpoints) score_inside = np.min(kde.score_samples(points)) score_outside = np.max(kde.score_samples(other_points)) levels = .80 * score_inside + .20 * score_outside ax.contour(xx, yy, scores.reshape(100, 100), levels=[levels], colors='k', linestyles='solid', linewidths=0.8) ax.set_xlim(x_min, x_max) ax.set_ylim(y_min, y_max)
def draw_posterior_kld_hist(X_kld, X_vae, f_name, bins=25): """ Plot KDE-smoothed histograms. """ import matplotlib.pyplot as plt # make a figure and configure an axis fig = plt.figure() ax = fig.add_subplot(111) ax.set_xlabel('Posterior KLd Density') ax.set_title('Posterior KLds: Over-regularized vs. Standard') ax.hold(True) for (X, style, label) in [(X_kld, '-', 'ORK'), (X_vae, '--', 'VAR')]: X_samp = X.ravel()[:,np.newaxis] X_min = np.min(X_samp) X_max = np.max(X_samp) X_range = X_max - X_min sigma = X_range / float(bins) plot_min = X_min - (X_range/4.0) plot_max = X_max + (X_range/4.0) plot_X = np.linspace(plot_min, plot_max, 1000)[:,np.newaxis] # make a kernel density estimator for the data in X kde = KernelDensity(kernel='gaussian', bandwidth=sigma).fit(X_samp) ax.plot(plot_X, np.exp(kde.score_samples(plot_X)), linestyle=style, label=label) ax.legend() fig.savefig(f_name, dpi=None, facecolor='w', edgecolor='w', \ orientation='portrait', papertype=None, format='pdf', \ transparent=False, bbox_inches=None, pad_inches=0.1, \ frameon=None) plt.close(fig) return
def kde_opt4(df_cell_train_feats, y_train, df_cell_test_feats): def prepare_feats(df): df_new = pd.DataFrame() df_new["hour"] = df["hour"] df_new["weekday"] = df["weekday"] + df["hour"] / 24. df_new["accuracy"] = df["accuracy"].apply(lambda x: np.log10(x)) df_new["x"] = df["x"] df_new["y"] = df["y"] return df_new logging.info("train kde_opt4 model") df_cell_train_feats_kde = prepare_feats(df_cell_train_feats) df_cell_test_feats_kde = prepare_feats(df_cell_test_feats) n_class = len(np.unique(y_train)) y_test_pred = np.zeros((len(df_cell_test_feats_kde), n_class), "d") for i in range(n_class): X = df_cell_train_feats_kde[y_train == i] y_test_pred_i = np.ones(len(df_cell_test_feats_kde), "d") for feat in df_cell_train_feats_kde.columns.values: X_feat = X[feat].values BGK10_output = kdeBGK10(X_feat) if BGK10_output is None: kde = gaussian_kde(X_feat, "scott") kde = gaussian_kde(X_feat, kde.factor * 0.741379) y_test_pred_i *= kde.evaluate(df_cell_test_feats_kde[feat].values) else: bandwidth, mesh, density = BGK10_output kde = KernelDensity(kernel='gaussian', metric='manhattan', bandwidth=bandwidth) kde.fit(X_feat[:, np.newaxis]) y_test_pred_i *= np.exp(kde.score_samples(df_cell_test_feats_kde[feat].values[:, np.newaxis])) y_test_pred[:, i] += y_test_pred_i return y_test_pred
def kde_sklearn(data, grid, **kwargs): """ Kernel Density Estimation with Scikit-learn Parameters ---------- data : numpy.array Data points used to compute a density estimator. It has `n x p` dimensions, representing n points and p variables. grid : numpy.array Data points at which the desity will be estimated. It has `m x p` dimensions, representing m points and p variables. Returns ------- out : numpy.array Density estimate. Has `m x 1` dimensions """ kde_skl = KernelDensity(**kwargs) kde_skl.fit(data) # score_samples() returns the log-likelihood of the samples log_pdf = kde_skl.score_samples(grid) return np.exp(log_pdf)
def find_kernel(data, numgrid = 1000, bw = 0.002): Xtrain = data[:,0:2] ytrain = data[2] # Set up the data grid for the contour plot xgrid = np.linspace(-74.1, -73.65, numgrid=1000) ygrid = np.linspace(40.5, 40.8, numgrid=1000) X, Y = np.meshgrid(xgrid, ygrid) xy = np.vstack([Y.ravel(), X.ravel()]).T # Plot map of with distributions of each species fig = plt.figure() # construct a kernel density estimate of the distribution kde = KernelDensity(bandwidth=bw, kernel='gaussian') kde.fit(Xtrain, y = ytrain) # evaluate only on the land: -9999 indicates ocean Z = np.exp(kde.score_samples(xy)) Z = Z.reshape(X.shape) # plot contours of the density levels = np.linspace(0, Z.max(), 25) plt.contourf(X, Y, Z, levels=levels, cmap=plt.cm.Reds) plt.title('BK CRIME') plt.show() return Z
def sklearn_density(sample_points, evaluation_points): """ Estimate the probability density function from which a set of sample points was drawn and return the estimated density at the evaluation points. """ from sklearn.neighbors import KernelDensity # Silverman bandwidth estimator n, d = sample_points.shape bandwidth = (n * (d + 2) / 4.)**(-1. / (d + 4)) # Standardize data so that we can use uniform bandwidth. # Note that we will need to scale the resulting density by sigma to # correct the area. mu, sigma = mean(sample_points, axis=0), std(sample_points, axis=0) data, points = (sample_points - mu)/sigma, (evaluation_points - mu)/sigma #print("starting grid search for bandwidth over %d points"%n) #from sklearn.grid_search import GridSearchCV #from numpy import logspace #params = {'bandwidth': logspace(-1, 1, 20)} #fitter = GridSearchCV(KernelDensity(), params) #fitter.fit(data) #kde = fitter.best_estimator_ #print("best bandwidth: {0}".format(kde.bandwidth)) #import time; T0 = time.time() kde = KernelDensity(kernel='gaussian', bandwidth=bandwidth, rtol=1e-6, atol=1e-6) #print("T:%6.3f fitting"%(time.time()-T0)) kde.fit(data) #print("T:%6.3f estimating"%(time.time()-T0)) log_pdf = kde.score_samples(points) #print("T:%6.3f done"%(time.time()-T0)) return exp(log_pdf)/np.prod(sigma) # undo the x scaling on the data points
def sklearn_kde(data, points): from sklearn.neighbors import KernelDensity # Silverman bandwidth estimator n, d = data.shape bandwidth = (n * (d + 2) / 4.)**(-1. / (d + 4)) # standardize data so that we can use uniform bandwidth mu, sigma = mean(data, axis=0), std(data, axis=0) data, points = (data - mu)/sigma, (points - mu)/sigma #print("starting grid search for bandwidth over %d points"%n) #from sklearn.grid_search import GridSearchCV #from numpy import logspace #params = {'bandwidth': logspace(-1, 1, 20)} #fitter = GridSearchCV(KernelDensity(), params) #fitter.fit(data) #kde = fitter.best_estimator_ #print("best bandwidth: {0}".format(kde.bandwidth)) #import time; T0 = time.time() kde = KernelDensity(kernel='gaussian', bandwidth=bandwidth, rtol=1e-6, atol=1e-6) #print("T:%6.3f fitting"%(time.time()-T0)) kde.fit(data) #print("T:%6.3f estimating"%(time.time()-T0)) log_pdf = kde.score_samples(points) #print("T:%6.3f done"%(time.time()-T0)) return exp(log_pdf)
def get_density_based_best_sample(X, known_votes, possibilities): total_votes = sum(map(lambda x: len(x), known_votes)) print total_votes X = X.toarray() current_vectors = numpy.copy(X) #print 'X', X #print 'known_votes ', known_votes original_docs = len(X) possibilities = set([x[0] for x in possibilities]) #print possibilities for i, sample in enumerate(known_votes): for k in range(len(sample)): current_vectors = numpy.append(current_vectors, [X[i]], axis=0) #print 'current_vectors ', current_vectors, len(current_vectors) #assert current_vectors != X model = KernelDensity(kernel='gaussian', bandwidth=0.2).fit(current_vectors) scores = model.score_samples(X) if (total_votes % 3): #Explore low density regions sorted_scores = sorted(enumerate(scores), key = lambda x: x[1], reverse=True) else: #Exploit high density regions 1 times out of 3 sorted_scores = sorted(enumerate(scores), key = lambda x: x[1]) #print sorted_scores for i in range(original_docs): if sorted_scores[i][0] in possibilities: #print sorted_scores[i][0] return sorted_scores[i][0] return None
def plot_kde_histogram2(X1, X2, f_name, bins=25): """ Plot KDE-smoothed histogram of the data in X1/X2. Assume data is 1D. """ import matplotlib.pyplot as plt # make a figure and configure an axis fig = plt.figure() ax = fig.add_subplot(111) ax.hold(True) for (X, style) in [(X1, '-'), (X2, '--')]: X_samp = X.ravel()[:,np.newaxis] X_min = np.min(X_samp) X_max = np.max(X_samp) X_range = X_max - X_min sigma = X_range / float(bins) plot_min = X_min - (X_range/3.0) plot_max = X_max + (X_range/3.0) plot_X = np.linspace(plot_min, plot_max, 1000)[:,np.newaxis] # make a kernel density estimator for the data in X kde = KernelDensity(kernel='gaussian', bandwidth=sigma).fit(X_samp) ax.plot(plot_X, np.exp(kde.score_samples(plot_X)), linestyle=style) fig.savefig(f_name, dpi=None, facecolor='w', edgecolor='w', \ orientation='portrait', papertype=None, format=None, \ transparent=False, bbox_inches=None, pad_inches=0.1, \ frameon=None) plt.close(fig) return
def kde_sklearn(x, x_grid, bandwidth=0.2, **kwargs): """Kernel Density Estimation with Scikit-learn""" kde_skl = KernelDensity(bandwidth=bandwidth, **kwargs) kde_skl.fit(x[:, np.newaxis]) # score_samples() returns the log-likelihood of the samples log_pdf = kde_skl.score_samples(x_grid[:, np.newaxis]) return np.exp(log_pdf)
def xy_kde(xy,bandwidth,N_grid=100,levels=[0.8,0.6,0.4,0.2]): x_edges = np.linspace(np.min(xy[:,0]),np.max(xy[:,0]),N_grid+1) y_edges = np.linspace(np.min(xy[:,1]),np.max(xy[:,1]),N_grid+1) x_centres = np.array([x_edges[b] + (x_edges[b+1]-x_edges[b])/2 for b in range(N_grid)]) y_centres = np.array([y_edges[b] + (y_edges[b+1]-y_edges[b])/2 for b in range(N_grid)]) x_grid, y_grid = np.meshgrid(x_centres,y_centres) xy_grid = np.array([np.ravel(x_grid),np.ravel(y_grid)]).T kde = KernelDensity(kernel='gaussian', bandwidth=bandwidth).fit(xy) H = np.exp(kde.score_samples(xy_grid).reshape(N_grid,N_grid)) # this bit is taken from the corner_plot.py method. ###################################### Hflat = H.flatten() inds = np.argsort(Hflat)[::-1] Hflat = Hflat[inds] sm = np.cumsum(Hflat) sm /= sm[-1] V = np.empty(len(levels)) for i, v0 in enumerate(levels): try: V[i] = Hflat[sm <= v0][-1] except: V[i] = Hflat[0] ##################################### V = np.sort(V) return H, V, x_grid, y_grid, bandwidth
def plot_kde_histogram(X, f_name, bins=25): """ Plot KDE-smoothed histogram of the data in X. Assume data is univariate. """ import matplotlib.pyplot as plt X = X.ravel() np.random.shuffle(X) X = X[0:min(X.shape[0], 1000000)] X_samp = X[:,np.newaxis] X_min = np.min(X_samp) X_max = np.max(X_samp) X_range = X_max - X_min sigma = X_range / float(bins) plot_min = X_min - (X_range/3.0) plot_max = X_max + (X_range/3.0) plot_X = np.linspace(plot_min, plot_max, 1000)[:,np.newaxis] # make a kernel density estimator for the data in X kde = KernelDensity(kernel='gaussian', bandwidth=sigma).fit(X_samp) # make a figure fig = plt.figure() ax = fig.add_subplot(111) ax.plot(plot_X, np.exp(kde.score_samples(plot_X))) fig.savefig(f_name, dpi=None, facecolor='w', edgecolor='w', \ orientation='portrait', papertype=None, format=None, \ transparent=False, bbox_inches=None, pad_inches=0.1, \ frameon=None) plt.close(fig) return
def plot_sklearn_kde(df, support, column='AirTime', bins=50): """ Plots a KDE and a histogram using sklearn.KernelDensity. Uses Gaussian kernels. The optimal bandwidth is calculated according to Silverman's rule of thumb. Parameters ---------- df: A pandas.DataFrame support: A 1-d numpy array. Input data points for the probabilit density function. Returns ------- A matplotlib.axes.Axes instance. """ bw = get_silverman_bandwidth(df, column) kde = KernelDensity(kernel='gaussian', bandwidth=bw) x = df[column] kde.fit(x[:, np.newaxis]) y = kde.score_samples(support[:, np.newaxis]) fig, ax = plt.subplots(figsize=(8, 5)) ax.hist(np.ravel(x), bins=bins, alpha=0.5, color=sns.xkcd_rgb["denim blue"], normed=True) ax.plot(support, np.exp(y)) ax.set_xlabel(column, fontsize=14) ax.set_ylabel('Density', fontsize=14) ax.set_title('Kernel Density Plot', fontsize=14) sns.despine(ax=ax, offset=5, trim=True) return ax
def max_prob(df): df_tmp = df.copy() arr = [] for ind in df_tmp.index: row = df_tmp.loc[ind] d = row.dropna().values # d = d.dropna() if len(d)==0: centre = np.NaN arr.append(centre) continue # arr = vals.sort(axis=0) # df_ordered = pd.DataFrame(vals, index=df.index, columns=df.columns) x_grid = np.linspace(d.min(), d.max(), 50) x_grid = x_grid.reshape(-1,1) d = d.reshape(-1,1) kde = KernelDensity().fit(d) log_dens = kde.score_samples(x_grid) vals = np.exp(log_dens).round(4) centre = x_grid[vals.argmax()][0] centre2 = round(centre, 4) # TODO first element adds unnecessary decimal places (use decimal places class to fix) arr.append(centre2) return arr
def test2(): arr = np.concatenate((np.linspace(0, 10, 10), np.linspace(2, 4, 10), np.linspace(7, 10, 10)))[:, np.newaxis] kde = KernelDensity(kernel='gaussian', bandwidth=0.75).fit(arr) X = np.linspace(0,10,1000)[:, np.newaxis] log_dens = kde.score_samples(X) plt.plot(X, log_dens) plt.show()
def surface_density(c, bandwidth=0.2, grid_step=0.02): """ Given particle positions as a coordinate object, compute the surface density using a kernel density estimate. """ if not HAS_SKLEARN: raise ImportError("scikit-learn is required to use this function.") xgrid = np.arange(2., 9.+0.1, grid_step) # deg ygrid = np.arange(26.5, 33.5+0.1, grid_step) # deg shp = (xgrid.size, ygrid.size) meshies = np.meshgrid(xgrid, ygrid) grid = np.vstack(map(np.ravel, meshies)).T x = c.l.degree y = c.b.degree skypos = np.vstack((x,y)).T kde = KernelDensity(bandwidth=bandwidth, kernel='epanechnikov') kde.fit(skypos) dens = np.exp(kde.score_samples(grid)).reshape(meshies[0].shape) log_dens = np.log10(dens) return grid, log_dens
def kdescatter(xs, ys, log_color=False, atol=1e-4, rtol=1e-4, n_jobs=1, n_samp_scaling=100, n_samp_tuning=1000, ax=None, **kwargs): if ax is None: import matplotlib.pyplot as plt ax = plt kwargs.setdefault('linewidths', 0) kwargs.setdefault('s', 20) kwargs.setdefault('cmap', 'winter') X = np.asarray([xs, ys]).T n = X.shape[0] samp_X = X[np.random.choice(n, min(n_samp_scaling, n), replace=False)] median_sqdist = np.median(euclidean_distances(samp_X, squared=True)) bws = np.logspace(-2, 2, num=10) * np.sqrt(median_sqdist) est = GridSearchCV(KernelDensity(), {'bandwidth': bws}, n_jobs=n_jobs) est.fit(X[np.random.choice(n, min(n_samp_tuning, n), replace=False)]) bw = est.best_params_['bandwidth'] kde = KernelDensity(bandwidth=bw) kde.fit(X) densities = kde.score_samples(X) if not log_color: np.exp(densities, out=densities) ax.scatter(xs, ys, c=densities, **kwargs)
def KDE_plt(categories,inter_arrivals): KDEs = [] for i in range(0,len(categories)): X = np.asarray(extract_cat_samples(inter_arrivals,categories,i))#for single inter-arrivals in a category #X = np_matrix(categories[i][0])#for avg(inter-arrival)/person in a category kde = KernelDensity(kernel='gaussian', bandwidth=4).fit(X) KDEs.append(kde) #to use for prob_return() max_sample = max_interarrival_mean(categories,inter_arrivals,i) X_plot = np.linspace(0,1.5*max_sample,2000)[:, np.newaxis] log_dens = kde.score_samples(X_plot) plt.figure(i) plt.plot(X_plot[:, 0], np.exp(log_dens), '-',label="kernel = '{0}'".format('gaussian')) #plt.draw() #plt.pause(0.001) #plt.title("Non-Parametric Density Estimation for category=%s Visitors"%(i)) plt.hist(combine_inner_lists(extract_cat_samples(inter_arrivals,categories,i)),bins=40,normed=1,color="cyan",alpha=.3,label="histogram") #alpha, from 0 (transparent) to 1 (opaque) # plt.hist(np.asarray(categories[i][0]),bins=40,normed=1,color="cyan",alpha=.3,label="histogram") #alpha, from 0 (transparent) to 1 (opaque) plt.xlabel("inter-arrival time (days)") plt.ylabel("PDF") plt.legend() save_as='./app/static/img/cat_result/kde/kdeplt_cat'+str(i)+'.png' # dump result into kde folder plt.savefig(save_as) plt.show(block=False) plt.close(plt.figure(i)) return KDEs
def EstimateDensity(self,name,df,histogram,f,s,ax): # if the desired output is in Histogram format if(histogram): finRes = [] lab = [] for i in xrange(5): res = np.array(df[ df[f] == i][s]) if(res.shape[0]>0): finRes.append(res) lab.append(name[0]+ ' = ' + str(i)) pl.hist(finRes, bins=2, normed=True, histtype='bar',label = lab) # if the desired output is simple plot else: for i in xrange(5): res = np.array(df[ df[f] == i][s]) if(res.shape[0]>0): res = res.reshape(res.shape[0],1) X_plot = np.array(np.linspace(-1, 5,20)).reshape(20,1) kde= KernelDensity(kernel='exponential', bandwidth=0.05) kde.fit(res) log_dens = kde.score_samples(X_plot) ax.plot(X_plot,np.exp(log_dens),label=name[0]+ ' = ' + str(i)) ax.legend() ax.set_title(name[1] + " distrubution for changing " + name[0])
def kde(self, term, bandwidth=2000, samples=1000, kernel='gaussian'): """ Estimate the kernel density of the instances of term in the text. Args: term (str): A stemmed term. bandwidth (int): The kernel bandwidth. samples (int): The number of evenly-spaced sample points. kernel (str): The kernel function. Returns: np.array: The density estimate. """ # Get the offsets of the term instances. terms = np.array(self.terms[term])[:, np.newaxis] # Fit the density estimator on the terms. kde = KernelDensity(kernel=kernel, bandwidth=bandwidth).fit(terms) # Score an evely-spaced array of samples. x_axis = np.linspace(0, len(self.tokens), samples)[:, np.newaxis] scores = kde.score_samples(x_axis) # Scale the scores to integrate to 1. return np.exp(scores) * (len(self.tokens) / samples)
def art_qi2(img, airmask, min_voxels=int(1e3), max_voxels=int(3e5), save_plot=True): r""" Calculates :math:`\text{QI}_2`, based on the goodness-of-fit of a centered :math:`\chi^2` distribution onto the intensity distribution of non-artifactual background (within the "hat" mask): .. math :: \chi^2_n = \frac{2}{(\sigma \sqrt{2})^{2n} \, (n - 1)!}x^{2n - 1}\, e^{-\frac{x}{2}} where :math:`n` is the number of coil elements. :param numpy.ndarray img: input data :param numpy.ndarray airmask: input air mask without artifacts """ from sklearn.neighbors import KernelDensity from scipy.stats import chi2 from mriqc.viz.misc import plot_qi2 # S. Ogawa was born np.random.seed(1191935) data = img[airmask > 0] data = data[data > 0] # Write out figure of the fitting out_file = op.abspath('error.svg') with open(out_file, 'w') as ofh: ofh.write('<p>Background noise fitting could not be plotted.</p>') if len(data) < min_voxels: return 0.0, out_file modelx = data if len(data) < max_voxels else np.random.choice( data, size=max_voxels) x_grid = np.linspace(0.0, np.percentile(data, 99), 1000) # Estimate data pdf with KDE on a random subsample kde_skl = KernelDensity(bandwidth=0.05 * np.percentile(data, 98), kernel='gaussian').fit(modelx[:, np.newaxis]) kde = np.exp(kde_skl.score_samples(x_grid[:, np.newaxis])) # Find cutoff kdethi = np.argmax(kde[::-1] > kde.max() * 0.5) # Fit X^2 param = chi2.fit(modelx[modelx < np.percentile(data, 95)], 32) chi_pdf = chi2.pdf(x_grid, *param[:-2], loc=param[-2], scale=param[-1]) # Compute goodness-of-fit (gof) gof = float(np.abs(kde[-kdethi:] - chi_pdf[-kdethi:]).mean()) if save_plot: out_file = plot_qi2(x_grid, kde, chi_pdf, modelx, kdethi) return gof, out_file
def test_kde_sample_weights(): n_samples = 400 size_test = 20 weights_neutral = np.full(n_samples, 3.) for d in [1, 2, 10]: rng = np.random.RandomState(0) X = rng.rand(n_samples, d) weights = 1 + (10 * X.sum(axis=1)).astype(np.int8) X_repetitions = np.repeat(X, weights, axis=0) n_samples_test = size_test // d test_points = rng.rand(n_samples_test, d) for algorithm in ['auto', 'ball_tree', 'kd_tree']: for metric in ['euclidean', 'minkowski', 'manhattan', 'chebyshev']: if algorithm != 'kd_tree' or metric in KDTree.valid_metrics: kde = KernelDensity(algorithm=algorithm, metric=metric) # Test that adding a constant sample weight has no effect kde.fit(X, sample_weight=weights_neutral) scores_const_weight = kde.score_samples(test_points) sample_const_weight = kde.sample(random_state=1234) kde.fit(X) scores_no_weight = kde.score_samples(test_points) sample_no_weight = kde.sample(random_state=1234) assert_allclose(scores_const_weight, scores_no_weight) assert_allclose(sample_const_weight, sample_no_weight) # Test equivalence between sampling and (integer) weights kde.fit(X, sample_weight=weights) scores_weight = kde.score_samples(test_points) sample_weight = kde.sample(random_state=1234) kde.fit(X_repetitions) scores_ref_sampling = kde.score_samples(test_points) sample_ref_sampling = kde.sample(random_state=1234) assert_allclose(scores_weight, scores_ref_sampling) assert_allclose(sample_weight, sample_ref_sampling) # Test that sample weights has a non-trivial effect diff = np.max(np.abs(scores_no_weight - scores_weight)) assert diff > 0.001 # Test invariance with respect to arbitrary scaling scale_factor = rng.rand() kde.fit(X, sample_weight=(scale_factor * weights)) scores_scaled_weight = kde.score_samples(test_points) assert_allclose(scores_scaled_weight, scores_weight)
fig.subplots_adjust(left=0.05, right=0.95, wspace=0.05) for i in range(2): plt.subplot(1, 2, i + 1) # construct a kernel density estimate of the distribution print(" - computing KDE in spherical coordinates") kde = KernelDensity(bandwidth=0.04, metric='haversine', kernel='gaussian', algorithm='ball_tree') kde.fit(Xtrain[ytrain == i]) # evaluate only on the land: -9999 indicates ocean Z = -9999 + np.zeros(land_mask.shape[0]) Z[land_mask] = np.exp(kde.score_samples(xy)) Z = Z.reshape(X.shape) # plot contours of the density levels = np.linspace(0, Z.max(), 25) plt.contourf(X, Y, Z, levels=levels, cmap=plt.cm.Reds) if basemap: print(" - plot coastlines using basemap") m = Basemap(projection='cyl', llcrnrlat=Y.min(), urcrnrlat=Y.max(), llcrnrlon=X.min(), urcrnrlon=X.max(), resolution='c') m.drawcoastlines()
#kde = KernelDensity(bandwidth=grid.best_params_['bandwidth'], kde = KernelDensity(bandwidth=1, kernel='gaussian') kde.fit(Zarr[:, None]) ### Plots # Remove large values for ease of plotting Zarr = Zarr[(Zarr < 100) & (Zarr > -100)] x_d = np.linspace(-100,100,1000) cfit = cauchy.pdf(x_d,loc=loc,scale=sca) nfit = norm.pdf(x_d,loc=locnorm,scale=scanorm) tfit = t.pdf(x_d,df=dft,loc=loct,scale=scat) logprob_kde = kde.score_samples(x_d[:, None]) pdf_cmb_array = [] for x in x_d: pdf_cmb_array.append(1/ncomb * pdf_cmb(x)) # pdf_cmb_array.append(pdf_cmb(x)) pdf_cmb_array = np.array(pdf_cmb_array) _ = plt.hist(Zarr,bins=100,normed=True,histtype='step') plt.plot(x_d,cfit,'k-') # Cauchy fit plt.plot(x_d,nfit,'k--') # Normal fit #plt.plot(x_d,tfit,'k-.') # Student-t fit plt.plot(x_d,pdf_cmb_array,'r--') # Mixture plt.fill_between(x_d, np.exp(logprob_kde), alpha=0.5)
plt.savefig(output_dir + 'ws_mean_vs_mld_change.pdf') plt.figure() plt.scatter(mean_highest_max_ws[under_ice], mld_change[under_ice], c='k', s=2) plt.xlabel( 'Average of highest 48 hours of wind speeds between float profile pairs (m/s)' ) plt.ylabel('MLD change (m)') plt.savefig(output_dir + 'ws_highest_vs_mld_change.pdf') plt.figure() mld_axis = arange(-50, 50, 1) kde_without_storms \ = KernelDensity(kernel='gaussian',bandwidth=5.0)\ .fit(mld_change[logical_and(under_ice,num_periods_with_storms == 0)].reshape(-1,1)) log_dens_without_storms = kde_without_storms.score_samples( mld_axis.reshape(-1, 1)) kde_with_storms \ = KernelDensity(kernel='gaussian',bandwidth=5.0)\ .fit(mld_change[logical_and(under_ice,num_periods_with_storms > 0)].reshape(-1,1)) log_dens_with_storms = kde_with_storms.score_samples(mld_axis.reshape(-1, 1)) plt.fill_between(mld_axis, exp(log_dens_without_storms), color='k', alpha=0.5, label='Zero storms', zorder=2) plt.fill_between(mld_axis, exp(log_dens_with_storms), color='r', alpha=0.5, label='One or more storms',
class KDE(KernelDensity, BaseDetector): def __init__(self, bandwidth=1.0, algorithm='auto', kernel='gaussian', metric="euclidean", atol=0, rtol=0, contamination=0.1, breadth_first=True, leaf_size=40, metric_params=None, random_state=42): """Kernel density estimation (KDE) Parameters ---------- bandwidth : float The bandwidth of the kernel. algorithm : str The tree algorithm to use. Valid options are ['kd_tree'|'ball_tree'|'auto']. Default is 'auto'. kernel : str The kernel to use. Valid kernels are ['gaussian'|'tophat'|'epanechnikov'|'exponential'|'linear'|'cosine'] Default is 'gaussian'. metric : str The distance metric to use. atol : float The desired absolute tolerance of the result. A larger tolerance will generally lead to faster execution. Default is 0. rtol : float The desired relative tolerance of the result. breadth_first : bool If true (default), use a breadth-first approach to the problem. Otherwise use a depth-first approach. leaf_size : int Specify the leaf size of the underlying tree. metric_params : dict Additional parameters to be passed to the tree for use with the metric. """ self.algorithm = algorithm self.bandwidth = bandwidth self.kernel = kernel self.metric = metric self.atol = atol self.rtol = rtol self.breadth_first = breadth_first self.leaf_size = leaf_size self.metric_params = metric_params self.contamination = contamination self.random_state = random_state # run the choose algorithm code so that exceptions will happen here # we're using clone() in the GenerativeBayes classifier, # so we can't do this kind of logic in __init__ self._choose_algorithm(self.algorithm, self.metric) if bandwidth <= 0: raise ValueError("bandwidth must be positive") if kernel not in VALID_KERNELS: raise ValueError("invalid kernel: '{0}'".format(kernel)) def fit(self, X_train, y_train=None): """Fit KDE. Parameters ---------- X_train : numpy array of shape (n_samples, n_features) The input samples. y_train : numpy array of shape (n_samples,), optional (default=None) The ground truth of the input samples (labels). Returns ------- self : object the fitted estimator. """ X_train = _check_X(X_train) self.model_ = KernelDensity(bandwidth=self.bandwidth, algorithm=self.algorithm, kernel=self.kernel, metric=self.metric, atol=self.atol, rtol=self.rtol, breadth_first=self.breadth_first, leaf_size=self.leaf_size, metric_params=self.metric_params) self.model_.fit(X_train) return self def decision_function(self, X): """Predict raw anomaly scores of X using the fitted detector. After invert_order(): the higher score, the more probability of x that is predicted as abnormal Parameters ---------- X : numpy array of shape (n_samples, n_features) The input samples. Sparse matrices are accepted only if they are supported by the base estimator. Returns ------- anomaly_scores : numpy array of shape (n_samples,) The anomaly score of the input samples. """ # check_is_fitted(self, ['decision_scores_', 'threshold_', 'labels_']) return invert_order(self.model_.score_samples(X)) def predict_proba(self, X): raise NotImplementedError
{'bandwidth': bandwidths}, cv=5, verbose=1) grid.fit(Pdarr) print('Best params:', grid.best_params_) # Instantiate and fit the KDE model print("Instantiate and fit the KDE model") kde = KernelDensity(bandwidth=grid.best_params_['bandwidth'], kernel='gaussian') kde.fit(Pdarr) # Score_samples returns the log of the probability density x_d = np.linspace(0, 100, 1000) logprob = kde.score_samples(x_d[:, None]) ### CORRECT DOMONKOS for cat in ['SC012', 'EK6', 'AR3']: Ptorr = data.loc[data['cathode'] == cat, 'totalPressure'] do = data.loc[data['cathode'] == cat, 'orificeDiameter'] * 0.1 data.loc[data['cathode'] == cat, 'pressureDiameter'] = Ptorr * do Pdarr_corr = np.array(data.pressureDiameter) Pdarr_corr = Pdarr_corr[~np.isnan(Pdarr_corr)] ## KERNEL DENSITY # Calculate best kernel density bandwidth bandwidths = 10**np.linspace(-1, 1, 200) grid = GridSearchCV(KernelDensity(kernel='gaussian'), {'bandwidth': bandwidths},
def sample_and_visualize(tp): pos = pos_dat[:, :, tp] drn = dir_dat[:, :, tp] vel = vel_dat[:, tp] # convert to spherical coords sph_coords = appendSpherical_np(pos) theta = sph_coords[:, 4] # latitude in [-pi/2, pi/2] phi = sph_coords[:, 5] # longitude in [-pi, pi] # compute KD-Tree for fast neighbor search # note: KD-Tree does not support haversine metric kd = KDTree(pos) theta_idx = np.argsort(theta) theta_sorted = theta[theta_idx] phi_idx = np.argsort(phi) phi_sorted = phi[phi_idx] # compute velocity vector components # reference: https://stackoverflow.com/questions/707985/ # calculate-3d-vector-perpendicular-to-plane-described-by-a-point-and-true-north-h origin = [0, 0, 0] north_vec = [0, 0, sph_rad] east_dir = np.cross(north_vec - pos, pos - origin) # points due east, tangent to the sphere north_dir = np.cross(east_dir, pos - origin) # points due north, tangent to the sphere east_dir = norm_vec_list(east_dir) north_dir = norm_vec_list(north_dir) vec_east = np.multiply(east_dir[:,0], drn[:,0]) + np.multiply(east_dir[:,1], drn[:,1]) + \ np.multiply(east_dir[:,2], drn[:,2]) vec_east = np.multiply(vel, vec_east) vec_north = np.multiply(north_dir[:,0], drn[:,0]) + np.multiply(north_dir[:,1], drn[:,1]) + \ np.multiply(north_dir[:,2], drn[:,2]) vec_north = np.multiply(vel, vec_north) # sampling mesh sample_theta = np.linspace(-np.pi / 2, np.pi / 2, 180) sample_phi = np.linspace(-np.pi, np.pi, 360) lats, lons = np.meshgrid(sample_theta, sample_phi) lats_flat = lats.ravel() lons_flat = lons.ravel() sample_X = sph_rad * np.multiply(np.sin(lats_flat + np.pi / 2), np.cos(lons_flat + np.pi)) sample_Y = sph_rad * np.multiply(np.sin(lats_flat + np.pi / 2), np.sin(lons_flat + np.pi)) sample_Z = sph_rad * np.cos(lats_flat + np.pi / 2) # KDE kde = KernelDensity(bandwidth=0.02, metric='haversine') kde.fit(np.vstack([theta, phi]).T, sample_weight=np.divide(1.0, num_agents)) latlon = np.vstack([lats_flat, lons_flat]).T density_est = np.exp(kde.score_samples(latlon)) density_est = density_est.reshape((360, 180)) # compute velocity at sample points vel_east = np.zeros(np.size(sample_X)) vel_north = np.zeros(np.size(sample_X)) nearest_neighbor_dist = 0.005 ind, dist = kd.query_radius(np.vstack([sample_X, sample_Y, sample_Z]).T, nearest_neighbor_dist, count_only=False, return_distance=True) for i in range(np.size(sample_X)): num_neighbors = np.size(ind[i]) vec_east_sum = np.sum(vec_east[ind[i]]) vec_north_sum = np.sum(vec_north[ind[i]]) if num_neighbors > 0: vel_east[i] = np.divide(vec_east_sum, num_neighbors) vel_north[i] = np.divide(vec_north_sum, num_neighbors) vel_east = vel_east.reshape((360, 180)) vel_north = vel_north.reshape((360, 180)) # shift indices by pi/2 and pi vel_east = np.roll(vel_east, (180, 0), axis=(0, 1)) vel_east = np.fliplr(vel_east) vel_north = np.roll(vel_north, (180, 0), axis=(0, 1)) vel_north = np.fliplr(vel_north) ################################### ## plot maps ################################### ytic_vals = np.array(list(np.linspace(0, 180, 6) - 90)) xtic_vals = np.array(list(np.linspace(-180, 180, 10))) plt.figure(figsize=(8, 3 * 3), dpi=300) plt.subplot(311) plt.imshow(density_est.T) cbar = plt.colorbar() cbar.ax.tick_params(labelsize=6) plt.clim(0, 1) plt.yticks(np.linspace(0, 180, 6), ytic_vals.astype(int), fontsize=7) plt.xticks(np.linspace(0, 360, 10), xtic_vals.astype(int), fontsize=7) plt.xlabel("Longitude", fontsize=8) plt.ylabel("Latitude", fontsize=8) plt.title("Local Density", fontsize=8) plt.subplot(312) plt.imshow(vel_north.T) cbar = plt.colorbar() cbar.ax.tick_params(labelsize=6) plt.clim(-v_max, v_max) plt.yticks(np.linspace(0, 180, 6), ytic_vals.astype(int), fontsize=7) plt.xticks(np.linspace(0, 360, 10), xtic_vals.astype(int), fontsize=7) plt.xlabel("Longitude", fontsize=8) plt.ylabel("Latitude", fontsize=8) plt.title("Velocity (North)", fontsize=8) plt.subplot(313) plt.imshow(vel_east.T) cbar = plt.colorbar() cbar.ax.tick_params(labelsize=6) plt.clim(-v_max, v_max) plt.yticks(np.linspace(0, 180, 6), ytic_vals.astype(int), fontsize=7) plt.xticks(np.linspace(0, 360, 10), xtic_vals.astype(int), fontsize=7) plt.xlabel("Longitude", fontsize=8) plt.ylabel("Latitude", fontsize=8) plt.title("Velocity (East)", fontsize=8) plt.subplots_adjust(wspace=0.2, hspace=0.4) plt.savefig(map_folder + os.sep + repr(tp).zfill(3) + ".png") plt.close() ################################### ## plot agents ################################### plot_quiver_flag = False R = sph_rad theta_val = theta_sorted[ theta_idx.argsort()] + np.pi / 2 # latitude in [0, pi] phi_val = phi_sorted[phi_idx.argsort()] + np.pi # longitude in [0, 2*pi] X_coord = R * np.sin(theta_val) * np.cos(phi_val) Y_coord = R * np.sin(theta_val) * np.sin(phi_val) Z_coord = R * np.cos(theta_val) # sphere parameterization u, v = np.mgrid[0:2 * np.pi:36j, 0:np.pi:18j] xs = R * np.cos(u) * np.sin(v) ys = R * np.sin(u) * np.sin(v) zs = R * np.cos(v) fig = plt.figure(figsize=(3, 3), dpi=300) ax = fig.add_subplot(1, 1, 1, projection='3d') # plot empty plot, with points (without a line) points, = ax.plot([], [], [], 'ro', markersize=0.3, alpha=1.0, fillstyle="full", markerfacecolor="red", markeredgecolor='red', zorder=10) quivers = ax.quiver([], [], [], [], [], [], color='lightblue', linewidth=0.8, normalize=False, zorder=5) # set initial viewing angles azimuth, elev = 75, 21 ax.set_xlim([-R, R]) ax.set_ylim([-R, R]) ax.set_zlim([-R, R]) ax.view_init(elev, azimuth) plot_idx = plot_visible(azimuth, elev, points, X_coord, Y_coord, Z_coord, quivers, drn[:, 0], drn[:, 1], drn[:, 2], vel, plot_quiver_flag) if plot_quiver_flag: rndr = plt.gcf().canvas.get_renderer() quivers.draw(rndr) fig.canvas.draw_idle() ax.plot_surface(xs, ys, zs, linewidth=0.1, zorder=0, edgecolor='gray', color='white', shade=False) plt.axis("off") plt.savefig(abm_folder + os.sep + repr(tp).zfill(3) + ".png") plt.close() ################################### ## plot overlays on sphere ################################### lats_vals = lats + np.pi / 2 lons_vals = lons + np.pi x = R * np.sin(lats_vals) * np.cos(lons_vals) y = R * np.sin(lats_vals) * np.sin(lons_vals) z = R * np.cos(lats_vals) fig = plt.figure(figsize=(4 * 3, 3), dpi=300) # plot density ax1 = fig.add_subplot(1, 3, 1, projection='3d') points, = ax1.plot([], [], [], 'ro', markersize=1.5, alpha=0.5, fillstyle="full", markerfacecolor="red", markeredgecolor='none', zorder=10) azimuth, elev = 75, 21 ax1.set_xlim([-R, R]) ax1.set_ylim([-R, R]) ax1.set_zlim([-R, R]) ax1.view_init(elev, azimuth) ls = LightSource(75, 0) rho_colors = ls.shade(density_est, cmap=cm.viridis, blend_mode='soft', vert_exag=1) rho_plt = ax1.plot_surface(x, y, z, rstride=1, cstride=1, linewidth=0, edgecolor='white', facecolors=rho_colors, antialiased=False, shade=True) cbar1 = fig.colorbar(rho_plt, ax=ax1, shrink=0.75) cbar1.ax.tick_params(labelsize=7) cbar1.mappable.set_clim(0, 1) plt.title("Local Density", fontsize=8) plt.axis("off") # plot velocity (north) ax2 = fig.add_subplot(1, 3, 2, projection='3d') points, = ax2.plot([], [], [], 'ro', markersize=1.5, alpha=0.5, fillstyle="full", markerfacecolor="red", markeredgecolor='none', zorder=10) azimuth, elev = 75, 21 ax2.set_xlim([-R, R]) ax2.set_ylim([-R, R]) ax2.set_zlim([-R, R]) ax2.view_init(elev, azimuth) ls = LightSource(75, 0) vel_north_colors = ls.shade(vel_north, cmap=cm.viridis, blend_mode='soft', vert_exag=1) vel_north_plt = ax2.plot_surface(x, y, z, rstride=1, cstride=1, linewidth=0, edgecolor='white', facecolors=vel_north_colors, antialiased=False, shade=True) cbar2 = fig.colorbar(vel_north_plt, ax=ax2, shrink=0.75) cbar2.ax.tick_params(labelsize=7) cbar2.mappable.set_clim(-v_max, v_max) plt.title("Velocity (North)", fontsize=8) plt.axis("off") # plot velocity (east) ax3 = fig.add_subplot(1, 3, 3, projection='3d') points, = ax3.plot([], [], [], 'ro', markersize=1.5, alpha=0.5, fillstyle="full", markerfacecolor="red", markeredgecolor='none', zorder=10) azimuth, elev = 75, 21 ax3.set_xlim([-R, R]) ax3.set_ylim([-R, R]) ax3.set_zlim([-R, R]) ax3.view_init(elev, azimuth) ls = LightSource(75, 0) vel_east_colors = ls.shade(vel_east, cmap=cm.viridis, blend_mode='soft', vert_exag=1) vel_east_plt = ax3.plot_surface(x, y, z, rstride=1, cstride=1, linewidth=0, edgecolor='white', facecolors=vel_east_colors, antialiased=False, shade=True) cbar3 = fig.colorbar(vel_east_plt, ax=ax3, shrink=0.75) cbar3.ax.tick_params(labelsize=7) cbar3.mappable.set_clim(-v_max, v_max) plt.title("Velocity (East)", fontsize=8) plt.axis("off") # save figure plt.savefig(overlay_folder + os.sep + repr(tp).zfill(3) + ".png") plt.close() ################################### ## save results ################################### feat_vec = np.vstack([ lats.ravel(), lons.ravel(), density_est.ravel(), vel_north.ravel(), vel_east.ravel() ]).T output_fname = npy_folder + os.sep + repr(tp).zfill(3) + ".npy" savedict = {'feat_vec': feat_vec, 'theta': sample_theta, 'phi': sample_phi} np.save(output_fname, savedict) output_fname = mat_folder + os.sep + repr(tp).zfill(3) + ".mat" sio.savemat(output_fname, savedict)
X_plot = np.linspace(-5, 20, 1000)[:, np.newaxis] from collections import Counter fig, ax = plt.subplots() n, bins, patches = ax.hist(X, density=1) c = Counter(x) y = c.values() #ax.plot(bins, y, '--') colors = ['navy', 'cornflowerblue', 'darkorange'] kernels = ['gaussian', 'epanechnikov'] lw = 2 for color, kernel in zip(colors, kernels): kde = KernelDensity(kernel=kernel, bandwidth=0.5).fit(X) log_dens = kde.score_samples(X_plot) ax.plot(X_plot[:, 0], np.exp(log_dens), color=color, lw=lw, linestyle='-', label="kernel = '{0}'".format(kernel)) ax.legend(loc='upper left') ax.plot(X[:, 0], -0.005 - 0.01 * np.random.random(X.shape[0]), '+k') ax.set_xlim(2, 20) ax.set_ylim(-0.02, 1) plt.show() from sklearn.neighbors import KernelDensity
def art_qi2(img, airmask, min_voxels=int(1e3), max_voxels=int(3e5), save_plot=True): r""" Calculates :math:`\text{QI}_2`, based on the goodness-of-fit of a centered :math:`\chi^2` distribution onto the intensity distribution of non-artifactual background (within the "hat" mask): .. math :: \chi^2_n = \frac{2}{(\sigma \sqrt{2})^{2n} \, (n - 1)!}x^{2n - 1}\, e^{-\frac{x}{2}} where :math:`n` is the number of coil elements. :param numpy.ndarray img: input data :param numpy.ndarray airmask: input air mask without artifacts """ from sklearn.neighbors import KernelDensity from scipy.stats import chi2 from mriqc.viz.misc import plot_qi2 # S. Ogawa was born np.random.seed(1191935) data = img[airmask > 0] data = data[data > 0] # Write out figure of the fitting out_file = op.abspath('error.svg') with open(out_file, 'w') as ofh: ofh.write('<p>Background noise fitting could not be plotted.</p>') if len(data) < min_voxels: return 0.0, out_file modelx = data if len(data) < max_voxels else np.random.choice( data, size=max_voxels) x_grid = np.linspace(0.0, np.percentile(data, 99), 1000) # Estimate data pdf with KDE on a random subsample kde_skl = KernelDensity(bandwidth=0.05 * np.percentile(data, 98), kernel='gaussian').fit(modelx[:, np.newaxis]) kde = np.exp(kde_skl.score_samples(x_grid[:, np.newaxis])) # Find cutoff kdethi = np.argmax(kde[::-1] > kde.max() * 0.5) # Fit X^2 param = chi2.fit(modelx[modelx < np.percentile(data, 95)], 32) chi_pdf = chi2.pdf(x_grid, *param[:-2], loc=param[-2], scale=param[-1]) # Compute goodness-of-fit (gof) gof = float(np.abs(kde[-kdethi:] - chi_pdf[-kdethi:]).mean()) if save_plot: out_file = plot_qi2(x_grid, kde, chi_pdf, modelx, kdethi) return gof, out_file
def ShowSingleComponentVariation(self, X, listOfComponents=[0, 1]): #matplotlib.rcParams['font.size'] = 14 showAsTraces = (np.shape(self.objectPixels)[0] == 1) assert (all([(x in range(self.numBasisFunctions)) for x in listOfComponents])) X_rep = self.RepresentUsingModel(X) percentilesToShow = [1, 10, 30, 70, 90, 99] numReadDataSamplePerPercentile = 4 representationPercentiles = [] for percentile in percentilesToShow: representationPercentiles.append( np.percentile(self.dataRepresentation, percentile, axis=0)) medianRepVec = np.percentile(self.dataRepresentation, 50, axis=0) for eigVecInd in listOfComponents: plt.figure() gs = gridspec.GridSpec(numReadDataSamplePerPercentile + 2, len(percentilesToShow)) # calculate the Gaussian smoothed distribution of values along the eignevector direction sigmaOfKDE = 0.12 pdfStart = min(self.dataRepresentation[:, eigVecInd]) - 3 * sigmaOfKDE pdfStop = max(self.dataRepresentation[:, eigVecInd]) + 3 * sigmaOfKDE xAxis = np.linspace(pdfStart, pdfStop, 200) PDF_Model = KernelDensity( kernel='gaussian', bandwidth=sigmaOfKDE).fit( self.dataRepresentation[:, eigVecInd].reshape(-1, 1)) logPDF = PDF_Model.score_samples(xAxis.reshape(-1, 1)) percentileValuesToShow = [ representationPercentiles[x][eigVecInd] for x in range(len(representationPercentiles)) ] percentilesToShowLogPDF = PDF_Model.score_samples( np.array(percentileValuesToShow).reshape(-1, 1)) # show distribution of current component and red dots at the list of precentiles to show plt.subplot(gs[0, :]) plt.fill(xAxis, np.exp(logPDF), fc='b', alpha=0.9) plt.scatter(percentileValuesToShow, np.exp(percentilesToShowLogPDF), c='r', s=300) plt.title( '%.3f%s explained' % (100 * self.PCAModel.explained_variance_ratio_[eigVecInd], '%')) for plotCol, currPrecentile in enumerate(percentilesToShow): currPrecentileRepVec = medianRepVec.copy() currPrecentileRepVec[eigVecInd] = representationPercentiles[ plotCol][eigVecInd] currPrecentileImage = np.zeros(np.shape(self.objectPixels)) currPrecentileImage[ self.objectPixels] = self.ReconstructUsingModel( currPrecentileRepVec).ravel() # show the median image with current precentile as activation of the curr image plt.subplot(gs[1, plotCol]) if showAsTraces: plt.plot(currPrecentileImage) plt.title('precentile: ' + str(percentilesToShow[plotCol]) + '%') elif np.shape(self.objectPixels)[2] == 3: currPrecentileImage[currPrecentileImage > 1] = 1.0 currPrecentileImage[currPrecentileImage < 0] = 0.0 plt.imshow(currPrecentileImage) plt.title('precentile: ' + str(percentilesToShow[plotCol]) + '%') plt.axis('off') else: plt.imshow(currPrecentileImage, cmap='gray') plt.title('precentile: ' + str(percentilesToShow[plotCol]) + '%') plt.axis('off') # find the most suitible candidates in X for current precentile distFromPercentile = abs( X_rep[:, eigVecInd] - representationPercentiles[plotCol][eigVecInd]) X_inds = np.argpartition(distFromPercentile, numReadDataSamplePerPercentile )[:numReadDataSamplePerPercentile] for k, X_ind in enumerate(X_inds): currNearestPrecentileImage = np.zeros( np.shape(self.objectPixels)) currNearestPrecentileImage[self.objectPixels] = X[ X_ind, :].ravel() plt.subplot(gs[2 + k, plotCol]) if showAsTraces: plt.plot(currNearestPrecentileImage) plt.title('Close Neighbor') else: plt.imshow(currNearestPrecentileImage, cmap='gray') plt.title('Close Neighbor') plt.axis('off') plt.tight_layout()
def ShowModelVariations(self, numVariations=6): #matplotlib.rcParams['font.size'] = 14 showAsTraces = (np.shape(self.objectPixels)[0] == 1) numVariations = min(numVariations, self.numBasisFunctions) numVarsPerFigure = min(6, numVariations) numFigures = int(np.ceil(float(numVariations) / numVarsPerFigure)) lowRepVec = np.percentile(self.dataRepresentation, 2, axis=0) medianRepVec = np.percentile(self.dataRepresentation, 50, axis=0) highRepVec = np.percentile(self.dataRepresentation, 98, axis=0) for figureInd in range(numFigures): plt.figure() for plotCol in range(numVarsPerFigure): eigVecInd = numVarsPerFigure * figureInd + plotCol if eigVecInd >= self.numBasisFunctions: break # create the low and high precentile representation activation vectors currLowPrecentileRepVec = medianRepVec.copy() currLowPrecentileRepVec[eigVecInd] = lowRepVec[eigVecInd] currHighPrecentileRepVec = medianRepVec.copy() currHighPrecentileRepVec[eigVecInd] = highRepVec[eigVecInd] # create blank images deltaImage = np.zeros(np.shape(self.objectPixels)) medianImage = np.zeros(np.shape(self.objectPixels)) lowPrecentileImage = np.zeros(np.shape(self.objectPixels)) highPrecentileImage = np.zeros(np.shape(self.objectPixels)) # fill the object pixels with the relevant data deltaImage[self.objectPixels] = self.PCAModel.components_[ eigVecInd, :].ravel() lowPrecentileImage[ self.objectPixels] = self.ReconstructUsingModel( currLowPrecentileRepVec).ravel() medianImage[self.objectPixels] = self.ReconstructUsingModel( medianRepVec).ravel() highPrecentileImage[ self.objectPixels] = self.ReconstructUsingModel( currHighPrecentileRepVec).ravel() # calculate the Gaussian smoothed distribution of values along the eignevector direction sigmaOfKDE = 0.12 pdfStart = min( self.dataRepresentation[:, eigVecInd]) - 3 * sigmaOfKDE pdfStop = max( self.dataRepresentation[:, eigVecInd]) + 3 * sigmaOfKDE xAxis = np.linspace(pdfStart, pdfStop, 200) PDF_Model = KernelDensity( kernel='gaussian', bandwidth=sigmaOfKDE).fit( self.dataRepresentation[:, eigVecInd].reshape(-1, 1)) logPDF = PDF_Model.score_samples(xAxis.reshape(-1, 1)) # show distribution of current component plt.subplot(5, numVarsPerFigure, 0 * numVarsPerFigure + plotCol + 1) plt.fill(xAxis, np.exp(logPDF), fc='b', alpha=0.9) plt.title( '%.3f%s explained' % (100 * self.PCAModel.explained_variance_ratio_[eigVecInd], '%')) # show variance direction (eigenvector) plt.subplot(5, numVarsPerFigure, 1 * numVarsPerFigure + plotCol + 1) if showAsTraces: plt.plot(deltaImage) plt.title('eigenvector ' + str(eigVecInd)) elif np.shape(self.objectPixels)[2] == 3: deltaImage = 0.1 / deltaImage.std() * deltaImage + 0.5 deltaImage[deltaImage > 1] = 1.0 deltaImage[deltaImage < 0] = 0.0 plt.imshow(deltaImage) plt.title('eigenvector ' + str(eigVecInd)) plt.axis('off') else: plt.imshow(deltaImage) plt.title('eigenvector ' + str(eigVecInd)) plt.axis('off') # show 2nd precentile image plt.subplot(5, numVarsPerFigure, 2 * numVarsPerFigure + plotCol + 1) if showAsTraces: plt.plot(lowPrecentileImage) plt.title('2nd precentile') elif np.shape(self.objectPixels)[2] == 3: lowPrecentileImage[lowPrecentileImage > 1] = 1.0 lowPrecentileImage[lowPrecentileImage < 0] = 0.0 plt.imshow(lowPrecentileImage) plt.title('2nd precentile') plt.axis('off') else: plt.imshow(lowPrecentileImage, cmap='gray') plt.title('2nd precentile') plt.axis('off') # show median image plt.subplot(5, numVarsPerFigure, 3 * numVarsPerFigure + plotCol + 1) if showAsTraces: plt.plot(medianImage) plt.title('median') else: plt.imshow(medianImage, cmap='gray') plt.title('median') plt.axis('off') # show 98th precentile image plt.subplot(5, numVarsPerFigure, 4 * numVarsPerFigure + plotCol + 1) if showAsTraces: plt.plot(highPrecentileImage) plt.title('98th precentile') elif np.shape(self.objectPixels)[2] == 3: highPrecentileImage[highPrecentileImage > 1] = 1.0 highPrecentileImage[highPrecentileImage < 0] = 0.0 plt.imshow(highPrecentileImage) plt.title('98th precentile') plt.axis('off') else: plt.imshow(highPrecentileImage, cmap='gray') plt.title('98th precentile') plt.axis('off') plt.tight_layout()
def feature_distribution_plot_mult_modes(feats, feat_names, grp_col, grp_colors, grp_modes, ncol, plot_ranges=None, dens_num=100): """Creats a lattice of histogram plots for the given grouped features, where each group can have a different plot type. Args: feats: A dataframe containing the features (on the columns) and a grouping variable. feat_names: A list of strings of features to plot, matching the column names in feats. grp_col: The name (string) of the column in feats used for grouping. grp_colors: A dict containing the colors (rgb strings) for each group in the grouping column. (E.g. {'group1':'rgb(228,26,28)'}). grp_modes: A dict containing the plotting for each group. Each mode must be one of the following: 'hist' - A histogram 'dens' - A fitted density histogram 'markers' - Scatter plot ncol: The number of columns (int) in the lattice plot. Returns: A plotly figure. """ grps = np.unique(feats[grp_col]) figs = [] cnt = 0 for f in feat_names: if cnt == 0: show_leg = True else: show_leg = False if plot_ranges != None: layout = go.Layout(title=f, titlefont=dict(size=10), autosize=False, xaxis=dict(range=plot_ranges[f])) else: layout = go.Layout(title=f, titlefont=dict(size=10), autosize=False) traces = [] if plot_ranges != None: f_min = plot_ranges[f][0] f_max = plot_ranges[f][1] else: f_min = min(feats.loc[:, f]) f_max = max(feats.loc[:, f]) xpts = np.linspace(f_min, f_max, dens_num) for g in grps: data = feats.loc[feats[grp_col] == g, f] if grp_modes[g] == 'dens': y = np.array(data) y = y[np.isnan(y) == False] kde = KernelDensity(kernel='gaussian', bandwidth=0.2).fit(y[:, np.newaxis]) log_dens = kde.score_samples(xpts[:, np.newaxis]) plt = go.Scatter(x=xpts, y=np.exp(log_dens), mode='lines', line=dict(color=grp_colors[g], width=2), name=g, showlegend=show_leg) elif grp_modes[g] == 'markers': plt = go.Scatter(x=list(data), y=[1.0] * len(data), mode='markers', marker=dict(color=grp_colors[g], size=5), name=g, showlegend=show_leg) else: plt = go.Histogram(x=list(data), marker=Marker(color=grp_colors[g]), name=g, showlegend=show_leg) traces.append(plt) fig = go.Figure(data=traces, layout=layout) figs.append(fig) cnt = cnt + 1 nrow = int(np.ceil(float(len(feat_names)) / float(ncol))) return subplot_helper_fig(nrow, ncol, figs)
def feature_distribution_plot(feats, feat_names, grp_col, grp_colors, ncol, plot_ranges=None, dens_est=False, dens_num=100, title_font=12, all_show_leg=True, lattice=True, titles=None, highlight_samps=[], hl_clr='black'): """Creats a lattice of histogram plots for the given grouped features Args: feats: A dataframe containing the features (on the columns) and a grouping variable. feat_names: A list of strings of features to plot, matching the column names in feats. grp_col: The name (string) of the column in feats used for grouping. grp_colors: A dict containing the colors (rgb strings) for each group in the grouping column. (E.g. {'group1':'rgb(228,26,28)'}). ncol: The number of columns (int) in the lattice plot. Returns: A plotly figure. """ grps = np.unique(feats[grp_col]) figs = [] cnt = 0 for f in feat_names: if cnt == 0: show_leg = True & all_show_leg else: show_leg = False & all_show_leg if titles == None: t = f else: t = titles[f] if plot_ranges != None: layout = go.Layout(title=t, titlefont=dict(size=10), autosize=False, xaxis=dict(range=plot_ranges[f])) else: layout = go.Layout(title=t, titlefont=dict(size=10), autosize=False) traces = [] if plot_ranges != None: f_min = plot_ranges[f][0] f_max = plot_ranges[f][1] else: f_min = 0.7 * min(feats.loc[:, f]) f_max = 1.3 * max(feats.loc[:, f]) xpts = np.linspace(f_min, f_max, dens_num) for g in grps: data = feats.loc[feats[grp_col] == g, f] y = np.array(data) y = y[np.isnan(y) == False] #if (dens_est) & (len(y) > 0) &\ #(len(np.unique(y)) > 3*len(y) / 4): if (dens_est) & (len(y) > 0): kde = KernelDensity(kernel='gaussian', bandwidth=0.2).fit(y[:, np.newaxis]) log_dens = kde.score_samples(xpts[:, np.newaxis]) plt = go.Scatter(x=xpts, y=np.exp(log_dens), mode='lines', line=dict(color=grp_colors[g], width=2), name=g, showlegend=show_leg) traces.append(plt) else: if len(y) > 0: plt = go.Histogram(x=list(data), marker=Marker(color=grp_colors[g]), name=g, showlegend=show_leg) traces.append(plt) if len(highlight_samps) > 0: for hs in highlight_samps: plt = go.Scatter(x=[feats.loc[hs, f]] * 2, y=[0.0, 1.0], line=dict(color=hl_clr, width=2), mode='lines') traces.append(plt) fig = go.Figure(data=traces, layout=layout) fig['layout'].update(titlefont=dict(size=title_font)) figs.append(fig) cnt = cnt + 1 if lattice: nrow = int(np.ceil(float(len(feat_names)) / float(ncol))) return subplot_helper_fig(nrow, ncol, figs) return figs
def plot_scatter(X, out_prefix, title, kde=True): """Draws a 2D scatter plot (png) of the core and accessory distances Also draws contours of kernel density estimare Args: X (numpy.array) n x 2 array of core and accessory distances for n samples. out_prefix (str) Prefix for output plot file (.png will be appended) title (str) The title to display above the plot kde (bool) Whether to draw kernel density estimate contours (default = True) """ # Plot results - max 1M for speed max_plot_samples = 1000000 if X.shape[0] > max_plot_samples: X = utils.shuffle(X, random_state=random.randint( 1, 10000))[0:max_plot_samples, ] # Kernel estimate uses scaled data 0-1 on each axis scale = np.amax(X, axis=0) X /= scale plt.figure(figsize=(11, 8), dpi=160, facecolor='w', edgecolor='k') if kde: xx, yy, xy = get_grid(0, 1, 100) # KDE estimate kde = KernelDensity(bandwidth=0.03, metric='euclidean', kernel='epanechnikov', algorithm='ball_tree') kde.fit(X) z = np.exp(kde.score_samples(xy)) z = z.reshape(xx.shape).T levels = np.linspace(z.min(), z.max(), 10) # Rescale contours plt.contour(xx * scale[0], yy * scale[1], z, levels=levels[1:], cmap='plasma') scatter_alpha = 1 else: scatter_alpha = 0.1 # Plot on correct scale plt.scatter(X[:, 0] * scale[0].flat, X[:, 1] * scale[1].flat, s=1, alpha=scatter_alpha) plt.title(title) plt.xlabel('Core distance (' + r'$\pi$' + ')') plt.ylabel('Accessory distance (' + r'$a$' + ')') plt.savefig(out_prefix + ".png") plt.close()
y = np.array([1, 1, 1, 2, 2, 2]) clf = NearestCentroid() clf.fit(X, y) print(clf.predict([[-0.8, -1]])) from sklearn.neighbors import KernelDensity kde = KernelDensity(bandwidth=0.04, metric='haversine', kernel='gaussian', algorithm='ball_tree', n) #kde.fit(Xtrain[ytrain == i]) kde.fit(X) kde.score_samples(X) from sklearn.datasets import load_iris from sklearn.cluster import KMeans from sklearn.metrics.pairwise import euclidean_distances X, y = load_iris(return_X_y=True) km = KMeans(n_clusters=5, random_state=1).fit(X) dists = euclidean_distances(km.cluster_centers_) import numpy as np tri_dists = dists[np.triu_indices(5, 1)] max_dist, avg_dist, min_dist = tri_dists.max(), tri_dists.mean( ), tri_dists.min()
def get_kde(X_vals, dist, kernel='tophat', bandwidth=0.3): eps=1e-15 scores = np.array(dist) kde = KernelDensity(kernel=kernel, bandwidth=bandwidth).fit(np.log10(1-scores+eps)[:,np.newaxis]) log_dens = kde.score_samples(X_vals) return np.exp(log_dens)
def kl_divergence_error(y, y_hat): kd = KernelDensity(bandwidth=0.75).fit(y.reshape(-1, 1)) yp = kd.score_samples(y.reshape(-1, 1)) kd = KernelDensity(bandwidth=0.75).fit(y_hat.reshape(-1, 1)) ypg = kd.score_samples(y_hat.reshape(-1, 1)) return entropy(yp, ypg)
def main(num_dim, periodic_signal, bw, step, r_fname='training.mat', w_fname="proba.mat"): num_dim = int(num_dim) periodic_signal = int(periodic_signal) bw = float(bw) step = float(step) mat = scipy.io.loadmat(r_fname) keyto_concat = ['X_training', 'Y_training'] to_concat = [] for k in keyto_concat: print mat[k].shape to_concat.append(mat[k]) to_concat[-1] = to_concat[-1][:, 0:num_dim] # select coordinates of Y values = np.hstack(to_concat) print values.shape _, N = values.shape # number of dimensions print 'number of dimensions' print N mins = [] maxs = [] for i in xrange(N): min_tmp = values[:, i].min() max_tmp = values[:, i].max() delta = max_tmp - min_tmp max_tmp = max_tmp + delta / 10. min_tmp = min_tmp - delta / 10. mins.append(min_tmp) maxs.append(max_tmp) mins[0] = 0. maxs[0] = 1. print values # add the same values at X-1 and X+1 to make sure that the estiamted pdf is for a periodic signal if periodic_signal: Xp = values[:, 0] + 1 Xm = values[:, 0] - 1 Y = values[:, 1:] to_concat = [] to_concat.append(Xp) to_concat.append(Y) to_concat = np.column_stack(to_concat) to_concat2 = [] to_concat2.append(to_concat) to_concat2.append(values) values = np.vstack(to_concat2) to_concat = [] to_concat.append(Xm) to_concat.append(Y) to_concat = np.column_stack(to_concat) to_concat2 = [] to_concat2.append(to_concat) to_concat2.append(values) values = np.vstack(to_concat2) print values.shape kde = KernelDensity(bandwidth=bw) kde.fit(values) # generate grid to_exec = "" to_exec += "np.mgrid[" for i in np.arange(N): to_exec += "%f:%f:%fj," % ( mins[i], maxs[i], step ) # select number of samples in each dimension to_exec = to_exec[:-1] to_exec += "]" print to_exec meshes = eval(to_exec) print meshes.shape size_grid = meshes[0].shape print 'size_grid' print size_grid Z = np.vstack([X.reshape(1, X.size) for X in meshes]).transpose() print Z.shape # score_samples() returns the log-likelihood of the samples log_pdf = kde.score_samples(Z) probas = np.exp(log_pdf) print probas.shape probas = probas.transpose().reshape(size_grid) print probas.shape mdict = {'Proba_XY': probas} i = 1 for X in meshes: mdict['X_%d' % i] = X i += 1 print mdict.keys() scipy.io.savemat(w_fname, mdict) proba_plot = probas for i in np.arange(N - 1, 1, -1): proba_plot = proba_plot.sum(i) plotcolormap(np.rot90(proba_plot), extent=[mins[0], maxs[0], mins[1], maxs[1]])
def kde(data): kd = KernelDensity(kernel='tophat', bandwidth=0.02).fit(data) return kd.score_samples(data)
# Note that it is not universal for it to be first minimum. Sometimes # the second minimum is better! print("Positions of the minima: ", min_vals) print("Suggested threshold is the position of the first minimum: ", min_vals[0]) print( "Please verify with the graph. There is a chance subsequent minima may work better." ) print("Elapsed time in seconds:", round(10.0 * (end - beg)) / 10.0) # sklearn, with similar results if use_sklearn: beg2 = time.time() kernel = 'gaussian' kde2 = KernelDensity(kernel=kernel, bandwidth=10).fit(data[:, np.newaxis]) log_dens = kde2.score_samples(xvals[:, np.newaxis]) yvals2 = np.exp(log_dens).reshape(-1) min_pos2 = argrelextrema(yvals2, np.less) min_vals2 = xvals[min_pos2] end2 = time.time() print("Elapsed time for sklearn kernel estimation in seconds:", round(10.0 * (end2 - beg2)) / 10.0) print("Suggested threshold is the position of the first minimum2: ", min_vals2[0]) print("Positions of the minima2: ", min_vals2) # Plot the kernel-density estimate and highlight the minima if not options.no_plot: plt.figure(1) plt.hist(data, bins=100, density=True, label="Data histogram") plt.plot(xvals, yvals, label="KDE", c="red")
def run_benchmark(df_path, n, numScore, tol, cols, bwValue=None, bwMult=1.0, denorm=False, use_std=False): params = { "algorithm": "sklearn", "dataset": df_path, "dim": len(cols), "num_train": n, "num_test": numScore, "train_time": None, "test_time": None, # "num_kernels": None } print(params) data = pd.read_csv(df_path)[cols].iloc[:n].values trainstart = time.time() if bwValue is None: bw = bwMult * estimate_kde_bw(data, use_std=use_std) print("BW: {}".format(bw)) else: bw = bwValue * np.ones(len(cols)) print("BW: {}".format(bwValue)) if numScore is None: numScore = len(data) internal_bw = 1 if denorm: internal_bw = 1.0 / (math.sqrt(2 * math.pi)) scaled_data = (data / bw) * internal_bw # Normalized Computations kde = KernelDensity( bandwidth=internal_bw, kernel='gaussian', algorithm='kd_tree', rtol=tol, ) kde.fit(scaled_data) train_time = time.time() - trainstart params["train_time"] = 1000 * train_time print("Trained in {}".format(train_time), flush=True) scorestart = time.time() scores = np.exp(kde.score_samples(scaled_data[:numScore])) score_time = time.time() - scorestart params["test_time"] = 1000 * score_time print("Scored in {}".format(score_time), flush=True) print("Rate: {}".format(numScore / score_time)) self_density = get_self_density(data.shape[1], data.shape[0]) scores_minus_self = scores - self_density # scale scores back if denorm: final_scores = scores_minus_self else: final_scores = scores_minus_self / np.prod(bw) q = np.percentile(final_scores, 1.0) print("Quantile: {}".format(q)) print("Final Output:") print(params) return final_scores
alpha_cm = plt.cm.Reds alpha_cm._init() alpha_cm._lut[:-3,-1] = abs(np.logspace(0, 1, alpha_cm.N) / 10 - 1)[::-1] aspect = mapdata.shape[0] * 1.0 / mapdata.shape[1] lon_lat_box = (-88, -87.5, 41.6, 42.1) sigthings = traps[traps['WnvPresent'] > 0] sigthings = sigthings.groupby(['Date', 'Trap','Longitude', 'Latitude']).max()['WnvPresent'].reset_index() X = sigthings[['Longitude', 'Latitude']].values kd = KernelDensity(bandwidth=0.02) kd.fit(X) xv,yv = np.meshgrid(np.linspace(-88, -87.5, 100), np.linspace(41.6, 42.1, 100)) gridpoints = np.array([xv.ravel(),yv.ravel()]).T zv = np.exp(kd.score_samples(gridpoints).reshape(100,100)) plt.figure(figsize=(10,14)) plt.imshow(mapdata, cmap=plt.get_cmap('gray'), extent=lon_lat_box, aspect=aspect) plt.imshow(zv, origin='lower', cmap=alpha_cm, extent=lon_lat_box, aspect=aspect) locations = traps[['Longitude', 'Latitude']].values plt.scatter(locations[:,0], locations[:,1], marker='x') plt.savefig('heatmap.png')
print(samples.median()) print(samples.std()) plt.hist(samples['mu_1'], bins=15) plt.hist(samples['mu_2'], bins=15) plt.grid() plt.show() mu1_list = matrix[:, 0] mu2_list = matrix[:, 1] kde1 = KernelDensity(kernel='gaussian', bandwidth=0.5).fit(mu1_list.reshape(-1, 1)) kde2 = KernelDensity(kernel='gaussian', bandwidth=0.5).fit(mu2_list.reshape(-1, 1)) x_plot = np.linspace(0, 10, len(mu1_list)).reshape(-1, 1) prob1 = np.exp(kde1.score_samples(x_plot)) prob2 = np.exp(kde2.score_samples(x_plot)) plt.plot(x_plot, prob1) plt.plot(x_plot, prob2) plt.legend(['mu1', 'mu2']) plt.ylabel('density') plt.title('Kernel Density') plt.grid() plt.show() mixture = [] for i in range(len(mu1_list)): mixture.append(max(prob1[i], prob2[i])) plt.plot(x_plot, mixture) plt.hist(samples['mu_1'], normed=True, bins='fd') plt.hist(samples['mu_2'], normed=True, bins='fd')
def plot(X:'array', threshold_freq:float = 0.0001, nbins:int = None, figsize:tuple = (15,8), supply:bool = False)->tuple: """ Plot histogram with densities estimated by KDE using different kernels. X -- 1D array of data. threshold_freq -- frequency (probability) limit to discard or not a spike as a local maximum (default 0.0001). nbins -- number of bins to be set (default None). In case of not to being included it will be estimated (optimal case). figsize -- figure size (default (15, 8)). supply -- return or not ax object. (default False). return -- ((x/y values of histogram), (x/y values of density for gaussian kernel), list of values where there are local maximums) NOTE - if supply = True, furthermore it will be returned the axis object ax. """ # estimate x limits add = truncate(np.ptp(X)) * 0.5 / 10. xmin = truncate(np.min(X)) - add if truncate(np.min(X)) < 0 else truncate(np.min(X)) + add xmax = truncate(np.max(X)) - add if truncate(np.max(X)) < 0 else truncate(np.max(X)) + add # estimate local maximum steps_avg = local_maximums_kde_gaussian(X.copy(), threshold_freq) #print('local maximums (gaussian): %s'%steps_avg) # number of bins if nbins is None: R = truncate(np.ptp(X)) n = len(X) sigma = np.nanstd(X) nbins = truncate(( R * (n**float(1/3)) ) / 3.49 * sigma ) bins = np.linspace(truncate(np.min(X)), truncate(np.max(X)), nbins) # x for plot X_plot = np.linspace(truncate(np.min(X)), truncate(np.max(X)), 1000)[:, np.newaxis] # create fig/axes fig, ax = plt.subplots(figsize = figsize) # plot the input data distribution h_y, h_x, _ = ax.hist(X , density = True, bins = bins, color = 'grey', label = 'input distribution', alpha = 0.2) # settings colors = ['cornflowerblue', 'darkorange', 'navy'] kernels = ['tophat', 'epanechnikov', 'gaussian'] lw = 2 # calculate kde and plot for color, kernel in zip(colors, kernels): kde = KernelDensity(kernel=kernel, bandwidth=0.5).fit(X) log_dens = kde.score_samples(X_plot) ax.plot(X_plot[:, 0], np.exp(log_dens), color=color, lw=lw, linestyle='-', label="kernel = '{0}'".format(kernel)) # store gaussian results if kernel == 'gaussian': l_y = np.exp(log_dens) l_x = X_plot[:, 0] # set legend ax.legend(loc='upper left') # plot points on the botton if len(X) < 10000: ax.plot(X[:, 0], -0.005 - 0.01 * np.random.random(X.shape[0]), '+k') # plot estimated local maximuns for avg in steps_avg: ax.axvline(avg, color='k', linestyle='--') # set chart limits ax.set_xlim(xmin, xmax) ax.set_ylim(-0.02, np.max(h_y) + 0.05) # set title ax.set_title("%s points / %s bins"%(len(X), nbins)) # set labels ax.set_ylabel("freq") # display / return if supply: return ((h_x, h_y), (l_x, l_y), steps_avg, ax) else: plt.show() return ((h_x, h_y), (l_x, l_y), steps_avg)
])) df2 = train_in.reindex(index=range(0, 5000), columns=list([ 'x_ 1', 'x_ 2', 'x_ 3', 'x_ 4', 'x_ 5', 'x_ 6', 'x_ 7', 'x_ 8', 'x_ 9', 'x_11', 'x_12' ])) df5 = train_in.reindex(index=range(0, 1000), columns=list(['x_ 10', 'x_ 13', 'x_ 14'])) df6 = train_in.reindex(index=range(0, 5000), columns=list(['x_ 10', 'x_ 13', 'x_ 14'])) df3 = train_in.reindex(index=range(0, 2500), columns=list(train_in.columns)) df4 = train_in.reindex(index=range(0, 5000), columns=list(train_in.columns)) #print(len(df1)) #print(len(df2)) kde = KernelDensity(kernel='gaussian', bandwidth=0.75).fit(df3) log_dens = kde.score_samples(df4) print(log_dens) k = np.arange(1, 5001) df = pd.DataFrame({ 'Point_ID': k, 'Output': log_dens }, index=None, columns=['Point_ID', 'Output']) df.to_csv('test_out1.csv', index=False)
def KDE(x, y): kde = KernelDensity(kernel='gaussian', bandwidth=0.2).fit(x) score = sum(kde.score_samples(y[:, ])) return score
def displayFamiliesDistribution(self, directory, label=None): if label is None: self.displayFamiliesDistribution(directory, label=labels_tools.MALICIOUS) self.displayFamiliesDistribution(directory, label=labels_tools.MALICIOUS) return families = self.families[labels_tools.labelBooleanToString(label)] bandwidth = 0.1 num_points = 50 eps = 0.00001 kde = KernelDensity(kernel='gaussian', bandwidth=bandwidth) fig, (ax) = plt.subplots(1, 1) i = 0 for family in families: predictions = families[family] predictions_np = np.asarray(predictions) if i % 3 == 0: linestyle = 'solid' elif i % 3 == 1: linestyle = 'dashed' if i % 3 == 2: linestyle = 'dotted' linewidth = 2 if np.var(predictions_np) < eps: linewidth = 4 mean = np.mean(predictions_np) x = np.arange(0, 1, 0.1) x = np.sort(np.append(x, [mean, mean - eps, mean + eps])) density = [1 if v == mean else 0 for v in x] else: density_predictions = [[x] for x in predictions_np] kde.fit(density_predictions) # Computes the x axis p_max = np.amax(predictions_np) p_min = np.amin(predictions_np) delta = p_max - p_min density_delta = 1.1 * delta x = np.arange(0, 1, density_delta / num_points) x_density = [[y] for y in x] # kde.score_samples returns the 'log' of the density log_density = kde.score_samples(x_density).tolist() density = list(map(math.exp, log_density)) ax.plot(x, density, label=family, linewidth=linewidth, linestyle=linestyle) fig_f, (ax_f) = plt.subplots(1, 1) ax_f.plot(x, density, linewidth=4, color=colors_tools.getLabelColor(label)) ax_f.set_title(family) ax_f.set_xlabel('P(Malicious)') ax_f.set_ylabel('Density') filename = label + '_family_' + family + '_prediction_distributions.png' fig_f.savefig(path.join(directory, filename)) plt.close(fig_f) i += 1 ax.legend(bbox_to_anchor=(0., 0.95, 1., .102), loc=3, ncol=5, mode='expand', borderaxespad=0., fontsize='xx-small') ax.set_xlabel('P(Malicious)') ax.set_ylabel('Density') filename = label + '_families_prediction_distributions.png' fig.savefig(path.join(directory, filename)) plt.close(fig)
class gaussian_kde_wrapper(object): def __init__(self, hyperparameter, param_name, data, oob_strategy='resample', bandwith=0.4): if oob_strategy not in ['resample', 'round', 'ignore']: raise ValueError() self.oob_strategy = oob_strategy self.param_name = param_name self.hyperparameter = hyperparameter reshaped = np.reshape(data, (len(data), 1)) if self.hyperparameter.log: if isinstance(self.hyperparameter, UniformIntegerHyperparameter): # self.probabilities = {val: self.distrib.pdf(np.log2(val)) for val in range(self.hyperparameter.lower, self.hyperparameter.upper)} raise ValueError( 'Log Integer hyperparameter not supported: %s' % param_name) # self.distrib = gaussian_kde(np.log2(data)) # self.distrib = KernelDensity(kernel='gaussian').fit(np.log2(np.reshape(data, (len(data), 1)))) self.distrib = KernelDensity(kernel='gaussian', bandwidth=bandwith).fit( np.log2(reshaped)) else: # self.distrib = gaussian_kde(data) self.distrib = KernelDensity(kernel='gaussian', bandwidth=bandwith).fit(reshaped) pass def pdf(self, x): x = np.reshape(x, (len(x), 1)) if self.hyperparameter.log: x = np.log2(x) log_dens = self.distrib.score_samples(x) return np.exp(log_dens) def rvs(self, *args, **kwargs): # assumes a samplesize of 1, for random search while True: sample = self.distrib.sample( n_samples=1, random_state=kwargs['random_state'])[0][0] if self.hyperparameter.log: value = np.power(2, sample) else: value = sample if isinstance(self.hyperparameter, UniformIntegerHyperparameter): value = int(round(value)) if self.hyperparameter.lower <= value <= self.hyperparameter.upper: return value elif self.oob_strategy == 'ignore': # TODO: hacky fail safe for some hyperparameters if hasattr(self.hyperparameter, 'lower_hard' ) and self.hyperparameter.lower_hard > value: continue if hasattr(self.hyperparameter, 'upper_hard' ) and self.hyperparameter.upper_hard < value: continue return value elif self.oob_strategy == 'round': if value < self.hyperparameter.lower: return self.hyperparameter.lower elif value > self.hyperparameter.upper: return self.hyperparameter.upper
def exe_kde(x_lon, y_lat, year, month, violation_code, kernel, bandwidth, metric): # Build shp shp = get_shp() # Build map fig = plt.figure(dpi = 1000) ax = fig.add_subplot(111) ax.axis('off') map = Basemap(projection = 'cyl', resolution = 'h', lat_0 = 43.0389025, lon_0 = -87.9064736, llcrnrlon = -88.080736, llcrnrlat = 42.917670, urcrnrlon = -87.839722, urcrnrlat = 43.19712) map.readshapefile(shp, name = 'mke_nbhd') patches_mke_nbhd = [] for info, shape in zip(map.mke_nbhd_info, map.mke_nbhd): if info['NEIGHBORHD'] != None: patches_mke_nbhd.append(Polygon(np.array(shape), True)) ax.add_collection(PatchCollection(patches_mke_nbhd, edgecolor = '#000000', facecolor = '#bfbfbf', linewidths = 0.45, zorder = 5)) # Build KDE k, m, kde_bw = kernel.lower(), metric.lower(), float(bandwidth) xy = np.stack([x_lon, y_lat]) d, n = xy.shape[0], xy.shape[1] kde = KernelDensity(kernel = k, bandwidth = kde_bw, metric = m) kde.fit(xy.T) xmin, xmax, ymin, ymax = -88.080736, -87.839722, 42.917670, 43.19712 # For all intents and purposes, this is grid size; in other words, this affects # resolution of the density plots X, Y = np.mgrid[xmin:xmax:1000j, ymin:ymax:1000j] positions = np.vstack([X.ravel(), Y.ravel()]) Z = np.reshape(np.exp(kde.score_samples(positions.T)), X.shape) # Build save cmap = colors.ListedColormap(['#ffffff', '#ffebeb', '#ffd8d8', '#ffc4c4', '#ffb1b1', '#ff9d9d', '#ff8a8a', '#ff7676', '#ff6262', '#ff4f4f', '#ff3b3b', '#ff2828', '#ff1414']) plt.imshow(np.rot90(Z), cmap = cmap, extent = [xmin, xmax, ymin, ymax], alpha = 0.5, zorder = 10) plt.scatter(x_lon, y_lat, c = '#0000ff', s = 1.75, alpha = 0.5, linewidths = 0, edgecolors = None, zorder = 15) s_yr, s_mo, s_vc = str(year), month.lower()[:3], str(violation_code) s_ke, s_bw, s_me = kernel.lower()[:3], str(kde_bw).replace('.', ''), metric.lower()[:3] plot_path_long = path.plot_path_long() plt.savefig(plot_path_long + '%s%s%s%s%s%s.png' % (s_yr, s_mo, s_vc, s_ke, s_bw, s_me), bbox_inches = 'tight', dpi = 1000)
def feature_distribution_plot_users(feats, users, feat_names, grp_col, grp_colors, dens_est=False, dens_num=50): """Creates a lattice of histogram plots for grouped features for many users. Each column contains the histogram feature plots for a single user. Args: feats: A dictionary containing the features (on the columns) and a grouping variable for each user. users: A subset of the keys of 'feats' to plot feat_names: A list of strings of features to plot, matching the column names in feats. grp_col: The name (string) of the column in feats used for grouping. grp_colors: A dict containing the colors (rgb strings) for each group in the grouping column. (E.g. {'group1':'rgb(228,26,28)'}). ncol: The number of columns (int) in the lattice plot. dens_est: A boolean indicating whether or not to show density estimation plot instead of histogram. Returns: A plotly figure. """ grps = np.unique(feats[users[0]][grp_col]) figs = [] cnt = 0 user_plt_cnt = 0 for f in feat_names: vals = [] for u in users: vals.extend(feats[u][f]) fmin = np.nanpercentile(vals, 2) fmax = np.nanpercentile(vals, 98) #fmin = min([min(feats[u][f]) for u in users]) #fmax = max([max(feats[u][f]) for u in users]) feat_plot_cnt = 0 for u in users: if cnt == 0: show_leg = True else: show_leg = False if user_plt_cnt == 0: plt_title = u else: plt_title = '' if feat_plot_cnt == 0: y_title = f else: y_title = '' layout = go.Layout(title=plt_title, titlefont=dict(size=10), yaxis=dict(title=y_title, titlefont=dict(size=18), color='black'), xaxis=dict(range=[fmin, fmax]), autosize=False) f_min = min(feats[u].loc[:, f]) f_max = max(feats[u].loc[:, f]) xpts = np.linspace(f_min, f_max, dens_num) traces = [] for g in grps: data = feats[u].loc[feats[u][grp_col] == g, f] if dens_est: y = np.array(data) y = y[np.isnan(y) == False] kde = KernelDensity(kernel='gaussian', bandwidth=0.75).fit(y[:, np.newaxis]) log_dens = kde.score_samples(xpts[:, np.newaxis]) plt = go.Scatter(x=xpts, y=np.exp(log_dens), mode='lines', line=dict(color=grp_colors[g], width=2), name=g, showlegend=show_leg) else: plt = go.Histogram(x=list(data), marker=Marker(color=grp_colors[g]), name=g, showlegend=show_leg) traces.append(plt) fig = go.Figure(data=traces, layout=layout) figs.append(fig) cnt = cnt + 1 feat_plot_cnt = feat_plot_cnt + 1 user_plt_cnt = user_plt_cnt + 1 ncol = len(users) nrow = len(feat_names) return subplot_helper_fig(nrow, ncol, figs)
def visualize(network, title,pos): """ Visualize the network given an array of posisitons. """ print("-- Starting to Visualize --") colors = [] colori = [] i_edge_colors = [] d_edge_colors = [] default = [] infected = [] nstart = [] ninfect = [] for node in network.nodes(): colorn = network.nodes[node]["color"] if colorn == "#A0C8F0": nstart.append(node) colors.append(network.nodes[node]["color"]) elif colorn == "#30cc1f" or colorn == "red" or colorn == "purple": ninfect.append(node) colori.append(network.nodes[node]["color"]) for i,j in network.edges(): color = network.nodes[i]["color"] if color == "#A0C8F0" or color == "#30cc1f" or color == "purple": color = "#A6A6A6" default.append((i,j)) d_edge_colors.append(color) else: color = "red" infected.append((i,j)) i_edge_colors.append(color) plt.figure(figsize=(30,20)) ax = plt.axes(projection=ccrs.PlateCarree()) ax.coastlines() #make density plot of infection node_positions = {node[0]: (float(node[1]['lon']), float(node[1]['lat'])) for node in network.nodes(data=True)} xp = [] yp = [] for node in network.nodes(): infec = network.nodes[node]["status"] if infec == 'i': xp.append(network.nodes[node]['lon']) yp.append(network.nodes[node]['lat']) if len(xp)>=1: m1, m2 = np.array(xp).astype(np.float), np.array(yp).astype(np.float) xmin = -180 xmax = 180 ymin = -90 ymax = 90 # get the density estimation Xp, Yp = np.mgrid[xmin:xmax:250j, ymin:ymax:250j] XpYp = np.vstack([Xp.ravel(), Yp.ravel()]).T XpYp = np.radians(XpYp) values = np.column_stack((np.array(np.vstack(m1)), np.array(np.vstack(m2)))) kernel = KernelDensity(bandwidth=0.035) kernel.fit(np.radians(values)) #kernel = stats.gaussian_kde(values) Z = np.exp(kernel.score_samples(XpYp)) Z = Z.reshape(Xp.shape) # plot the result cmap = plt.cm.jet cmap.set_under('white') plt.imshow(np.rot90(Z), norm = plt.Normalize(vmin=(Z.max()-(Z.max()*0.9)), vmax=Z.max()), cmap=cmap, extent=[xmin, xmax, ymin, ymax], alpha=0.3, interpolation = 'gaussian') # Fist pass - Gray lines nx.draw_networkx_edges(network,pos=node_positions,edgelist=default, width=0.005, edge_color=d_edge_colors, alpha=0.005, arrows=False) # Second Pass - Colored lines nx.draw_networkx_edges(network,pos=node_positions,edgelist=infected, width=0.1, edge_color=i_edge_colors, alpha=0.25, arrows=False) # first Pass - small nodes nx.draw_networkx_nodes(network, pos=node_positions, nodelist=nstart, linewidths=0.2, node_size=5, with_labels=False, node_color = colors) # # Second Pass - large nodes nx.draw_networkx_nodes(network, pos=node_positions, nodelist=ninfect, linewidths=0.2, node_size=20, with_labels=False, node_color = colori) plt.axis('off') number_files = str(len(os.listdir())) while len(number_files) < 3: number_files = "0" + number_files plt.savefig("infection-{0}.png".format(number_files), bbox_inches='tight', dpi=72 ) plt.show() plt.close()