def _run_gridsearchcv(self, data, param_s, param_c): """ Private method that initializes classifier/clustering with different parameter combinations and :class:`reval.best_nclust_cv.FindBestClustCV`. :param data: training dataset. :type data: numpy array :param param_s: dictionary of classifier parameters. :type: dict :param param_c: dictionary of clustering parameters. :type param_c: dict :return: performance list. :rtype: list """ self.class_method.set_params(**param_s) self.clust_method.set_params(**param_c) findclust = FindBestClustCV(nfold=self.cv, s=self.class_method, c=self.clust_method, nrand=self.nrand, n_jobs=1, nclust_range=self.clust_range) if self.clust_range is not None: metric, nclbest = findclust.best_nclust(data, iter_cv=self.iter_cv, strat_vect=self.strat) tr_lab = None else: try: metric, nclbest, tr_lab = findclust.best_nclust( data, iter_cv=self.iter_cv, strat_vect=self.strat) except TypeError: perf = [(key, val) for key, val in param_s.items()] + \ [(key, val) for key, val in param_c.items()] + \ [('best_nclust', None), ('mean_train_score', None), ('sd_train_score', None), ('mean_val_score', None), ('sd_val_score', None), ('validation_meanerror', None), ('tr_label', None)] return perf perf = [(key, val) for key, val in param_s.items()] + \ [(key, val) for key, val in param_c.items()] + \ [('best_nclust', nclbest), ('mean_train_score', np.mean( findclust.cv_results_.loc[findclust.cv_results_.ncl == nclbest]['ms_tr'])), ('sd_train_score', np.std( findclust.cv_results_.loc[findclust.cv_results_.ncl == nclbest]['ms_tr'])), ('mean_val_score', np.mean( findclust.cv_results_.loc[findclust.cv_results_.ncl == nclbest]['ms_val'])), ('sd_val_score', np.std( findclust.cv_results_.loc[findclust.cv_results_.ncl == nclbest]['ms_val'])), ('validation_meanerror', metric['val'][nclbest]), ('tr_label', tr_lab)] return perf
def setUp(cls): cls.s = CLASSIFIER cls.c = NEW_CLUSTERING cls.c_work = hdbscan.HDBSCAN() cls.nrand = RNDLABELS_ITER cls.nfold = NFOLD cls.nclust_range = NEW_NCLUST_RANGE cls.n_jobs = N_JOBS cls.findbest = FindBestClustCV(cls.s, cls.c, cls.nrand, cls.nfold, cls.n_jobs, cls.nclust_range) cls.findbest_bis = FindBestClustCV(cls.s, cls.c_work, cls.nrand, cls.nfold, cls.n_jobs, cls.nclust_range)
def example_2(): digits_dataset = load_digits() digits_data = digits_dataset['data'] digits_target = digits_dataset['target'] X_tr, X_ts, y_tr, y_ts = train_test_split(digits_data, digits_target, test_size=0.40, random_state=42, stratify=digits_target) transform = UMAP(n_components=2, random_state=42, n_neighbors=30, min_dist=0.0) X_tr = transform.fit_transform(X_tr) X_ts = transform.transform(X_ts) s = KNeighborsClassifier(n_neighbors=30) c = KMeans() reval = FindBestClustCV(s=s, c=c, nfold=5, nclust_range=[2, 15], nrand=100) metrics, nclustbest, _ = reval.best_nclust(X_tr, iter_cv=10, strat_vect=y_tr) plot_metrics(metrics, title='Reval performance digits dataset') out = reval.evaluate(X_tr, X_ts, nclust=nclustbest) perm_lab = kuhn_munkres_algorithm(y_ts, out.test_cllab) print(f"Best number of clusters: {nclustbest}") print(f"Test set prediction ACC: " f"{1 - zero_one_loss(y_ts, perm_lab)}") print(f'AMI (true labels vs predicted labels) = ' f'{adjusted_mutual_info_score(y_ts, out.test_cllab)}') print(f"Validation set normalized stability (misclassification):" f"{metrics['val'][nclustbest]}") print(f'Test set ACC = {out.test_acc} ' f'(true labels vs predicted labels)') plt.figure(figsize=(6, 4)) plt.scatter(X_ts[:, 0], X_ts[:, 1], c=y_ts, cmap='rainbow_r') plt.title("Test set true labels (digits dataset)") plt.show() plt.figure(figsize=(6, 4)) plt.scatter(X_ts[:, 0], X_ts[:, 1], c=perm_lab, cmap='rainbow_r') plt.title("Test set clustering labels (digits dataset)") plt.show()
def example_1(): data = make_blobs(1000, 2, 5, center_box=(-20, 20), random_state=42) plt.figure(figsize=(6, 4)) plt.scatter(data[0][:, 0], data[0][:, 1], c=data[1], cmap='rainbow_r') plt.title("Blobs dataset (N=1000)") plt.show() X_tr, X_ts, y_tr, y_ts = train_test_split(data[0], data[1], test_size=0.30, random_state=42, stratify=data[1]) classifier = KNeighborsClassifier(n_neighbors=5) clustering = KMeans() findbestclust = FindBestClustCV(nfold=10, nclust_range=[2, 7], s=classifier, c=clustering, nrand=100) metrics, nbest, _ = findbestclust.best_nclust(X_tr, iter_cv=10, strat_vect=y_tr) out = findbestclust.evaluate(X_tr, X_ts, nbest) perm_lab = kuhn_munkres_algorithm(y_ts, out.test_cllab) print(f"Best number of clusters: {nbest}") print(f"Test set prediction ACC: " f"{1 - zero_one_loss(y_ts, perm_lab)}") print(f'AMI (true labels vs predicted labels) = ' f'{adjusted_mutual_info_score(y_ts, out.test_cllab)}') print(f"Validation set normalized stability (misclassification):" f"{metrics['val'][nbest]}") print(f'Test set ACC = {out.test_acc} ' f'(true labels vs predicted labels)') plot_metrics(metrics, title="Reval performance blobs dataset", legend_loc=2) plt.figure(figsize=(6, 4)) plt.scatter(X_ts[:, 0], X_ts[:, 1], c=y_ts, cmap='rainbow_r') plt.title("Test set true labels (blobs dataset)") plt.show() plt.figure(figsize=(6, 4)) plt.scatter(X_ts[:, 0], X_ts[:, 1], c=perm_lab, cmap='rainbow_r') plt.title("Test set clustering labels (blobs dataset)") plt.show()
def _run_gridsearchcv(self, data, sc): """ Private function with different initializations of :class:`reval.best_nclust_cv.FindBestClustCV`. :param data: input dataset. :type data: numpy array :param sc: classifier/clustering of the form {'s':, 'c':}. :type sc: dict :return: performance list. :rtype: list """ findclust = FindBestClustCV(s=sc['s'], c=sc['c'], nfold=self.cv, nrand=self.nrand, n_jobs=1, nclust_range=self.clust_range) if 'n_clusters' in sc['c'].get_params().keys(): metric, nclbest = findclust.best_nclust(data, iter_cv=self.iter_cv, strat_vect=self.strat) sc['c'].n_clusters = nclbest tr_lab = None else: try: metric, nclbest, tr_lab = findclust.best_nclust( data, iter_cv=self.iter_cv, strat_vect=self.strat) except TypeError: perf = [('s', sc['s']), ('c', sc['c']), ('best_nclust', None), ('mean_train_score', None), ('sd_train_score', None), ('mean_val_score', None), ('sd_val_score', None), ('validation_meanerror', None), ('tr_label', None)] return perf cv_scores = findclust.cv_results_ perf = [('s', sc['s']), ('c', sc['c']), ('best_nclust', nclbest), ('mean_train_score', np.mean(cv_scores.loc[cv_scores.ncl == nclbest]['ms_tr'])), ('sd_train_score', np.std(cv_scores.loc[cv_scores.ncl == nclbest]['ms_tr'])), ('mean_val_score', np.mean(cv_scores.loc[cv_scores.ncl == nclbest]['ms_val'])), ('sd_val_score', np.std(cv_scores.loc[cv_scores.ncl == nclbest]['ms_val'])), ('validation_meanerror', metric['val'][nclbest]), ('tr_label', tr_lab)] return perf
def setUp(cls): cls.s = CLASSIFIER cls.c = CLUSTERING cls.nrand = RNDLABELS_ITER cls.nfold = NFOLD cls.nclust_range = NCLUST_RANGE cls.findbest = FindBestClustCV(cls.nfold, cls.nclust_range, cls.s, cls.c, cls.nrand)
def setUp(cls): cls.s = KNeighborsClassifier(n_neighbors=5) cls.c = AgglomerativeClustering() cls.nrand = 10 cls.nfold = 2 cls.nclust_range = [2, 4] cls.findbest = FindBestClustCV(cls.nfold, cls.nclust_range, cls.s, cls.c, cls.nrand)
import cProfile import pstats from sklearn.datasets import make_blobs from sklearn.model_selection import train_test_split from sklearn.cluster import KMeans, AgglomerativeClustering from hdbscan import HDBSCAN from sklearn.neighbors import KNeighborsClassifier from reval.best_nclust_cv import FindBestClustCV s = KNeighborsClassifier() # c = KMeans() c = AgglomerativeClustering() findbest = FindBestClustCV(s=s, c=c, nrand=10, nfold=2, n_jobs=1, nclust_range=list(range(2, 7, 1))) data = make_blobs(100, 10, centers=2) data_tr, data_ts, y_tr, y_ts = train_test_split(data[0], data[1], test_size=0.5, stratify=data[1], random_state=42) print("Profiling of algorithm that finds the best number of clusters.") profiler = cProfile.Profile() profiler.enable() findbest.best_nclust(data_tr, iter_cv=10, strat_vect=y_tr) profiler.disable()
# Plot synthetic dataset plt.scatter(data1[0][:, 0], data1[0][:, 1], c=data1[1], cmap='rainbow_r') plt.title('True labels for 10-feature dataset') X_tr, X_ts, y_tr, y_ts = train_test_split(data1[0], data1[1], test_size=0.30, random_state=42, stratify=data1[1]) # Apply relative clustering validation with KNN and Hierarchical clustering classifier = KNeighborsClassifier() clustering = AgglomerativeClustering() findbestclust = FindBestClustCV(nfold=10, nclust_range=[2, 7], s=classifier, c=clustering, nrand=100) metrics, nbest, _ = findbestclust.best_nclust(X_tr, y_tr) out = findbestclust.evaluate(X_tr, X_ts, nbest) plot_metrics(metrics, "Reval performance for synthetic dataset with 10 features") plt.scatter(X_ts[:, 0], X_ts[:, 1], c=out.test_cllab, cmap='rainbow_r') plt.title("Predicted labels for 10-feature dataset") # Compare Reval solution to true labels print(f'AMI test set = {adjusted_mutual_info_score(y_ts, out.test_cllab)}') relabeling = _kuhn_munkres_algorithm(y_ts, out.test_cllab) print(f'ACC test set = {1 - zero_one_loss(y_ts, relabeling)}')
def example_3(): # Classifiers knn = KNeighborsClassifier(n_neighbors=1, metric='euclidean') rf = RandomForestClassifier(n_estimators=100, random_state=42) svm = SVC(C=1, random_state=42) logreg = LogisticRegression(solver='liblinear', random_state=42) classifiers = [knn, logreg, svm, rf] # Clustering hc = AgglomerativeClustering() km = KMeans(random_state=42) sc = SpectralClustering(random_state=42) clustering = [hc, km, sc] # scaler = StandardScaler() transform = UMAP(n_neighbors=30, min_dist=0.0, random_state=42) # Import benchmark datasets uci_data = build_ucidatasets() # Run ensemble learning algorithm for data, name in zip(uci_data, uci_data._fields): nclass = len(np.unique(data['target'])) logging.info(f"Processing dataset {name}") logging.info(f"Number of classes: {nclass}\n") X_tr, X_ts, y_tr, y_ts = train_test_split(data['data'], data['target'], test_size=0.40, random_state=42, stratify=data['target']) X_tr = transform.fit_transform(X_tr) X_ts = transform.transform(X_ts) for s in classifiers: if type(s) == type(svm): svm.gamma = 1 / data['data'].shape[0] for c in clustering: logging.info( f"Clustering algorithm: {c} -- Classification algorithm {s}" ) reval = FindBestClustCV(s=s, c=c, nfold=5, nclust_range=[2, nclass + 3], nrand=100) metrics, nclustbest, _ = reval.best_nclust(X_tr, strat_vect=y_tr) out = reval.evaluate(X_tr, X_ts, nclust=nclustbest) perm_lab = kuhn_munkres_algorithm(y_ts, out.test_cllab) logging.info(f"Best number of clusters: {nclustbest}") logging.info(f"Test set prediction ACC: " f"{1 - zero_one_loss(y_ts, perm_lab)}") logging.info( f'AMI (true labels vs predicted labels) = ' f'{adjusted_mutual_info_score(y_ts, out.test_cllab)}') logging.info( f"Validation set normalized stability (misclassification):" f"{metrics['val'][nclustbest]}") logging.info(f'Test set ACC = {out.test_acc} ' f'(true labels vs predicted labels)\n') logging.info('*' * 100) logging.info('\n\n')
def run_validation(data_tr, data_ts, raw_ts, n_neighbors, period, hierarchy, cl_range=(2, 11)): """Function that performs the relative clustering validation. :param data_tr: training dataset :type data_tr: dataframe :param data_ts: test dataset :type data_ts: dataframe :param raw_ts: original test dataset before imputation :type raw_ts: dataframe :param n_neighbors: number of neighbors to consider for UMAP preprocessing step :type n_neighbors: int :param cl_range: range of number of clusters to consider, default (2, 11) :type cl_range: tuple :param period: interview period :type period: str :param hierarchy: hierarchical feature level :type hierarchy: str """ logging.info(f'Processing Vineland feature level {hierarchy} at period {period}...') transform = umap.UMAP(random_state=42, n_neighbors=n_neighbors, min_dist=0.0) X_tr = transform.fit_transform(data_tr) X_ts = transform.transform(data_ts) # Initialize classes knn = KNeighborsClassifier(n_neighbors=10) clust = AgglomerativeClustering(affinity='euclidean', linkage='ward') relval = FindBestClustCV(s=knn, c=clust, nfold=10, nclust_range=cl_range, nrand=100) # This runs a 10-fold cross validation with number of clusters from 2 to 10 # Run the model metric, ncl, cv_scores = relval.best_nclust(X_tr) # the strat_vect parameter can be used to perform a stratified CV logging.info(f"Best number of clusters: {ncl}") out = relval.evaluate(X_tr, X_ts, ncl) logging.info(f"Training ACC: {out.train_acc}, Test ACC: {out.test_acc}") plot_metrics(metric) unique, counts = np.unique(out.train_cllab, return_counts=True) logging.info(f'Training set (N = {X_tr.shape[0]})\n') for a, b in zip(unique, counts): logging.info(f'N subjects in cluster {a}: {b}') unique, counts = np.unique(out.test_cllab, return_counts=True) logging.info(f'\n\nTest set (N = {X_ts.shape[0]})\n') for a, b in zip(unique, counts): logging.info(f'N subjects in cluster {a}: {b}') umap_tr = X_tr umap_ts = X_ts # umap_tr = transform.fit_transform(X_tr) # umap_ts = transform.fit_transform(X_ts) flatui = ["#1f77b4", "#ff7f0e", "#2ca02c", "#d62728", "#9467bd", "#8c564b", "#e377c2", "#7f7f7f", "#bcbd22", "#17becf", "#8c564b", "#a55194"] _scatter_plot(umap_tr, [(gui, cl) for gui, cl in zip(data_tr.index, out.train_cllab)], flatui, 10, 20, {str(ncl): '-'.join(['cluster', str(ncl)]) for ncl in sorted(np.unique(out.train_cllab))}, title=f'Subgroups of UMAP preprocessed Vineland TRAINING ' f'dataset (period: {period} -- level: {hierarchy})') _scatter_plot(umap_ts, [(gui, cl) for gui, cl in zip(data_ts.index, out[2])], flatui, 10, 20, {str(ncl): '-'.join(['cluster', str(ncl)]) for ncl in sorted(np.unique(out[2]))}, title=f'Subgroups of UMAP preprocessed Vineland TEST ' f'dataset (period: {period} -- level: {hierarchy})') # Plot heatmap raw_ts = raw_ts.loc[data_ts.index] raw_ts['cluster'] = out[2] mis_perc = {} for lab in np.unique(out.test_cllab): ts_rid = raw_ts.loc[raw_ts.cluster == lab].copy() mis_perc[lab] = (sum([ts_rid.iloc[indx].isna().astype(int) for indx in range(ts_rid.shape[0])]) / ts_rid.shape[ 0]) * 100 # Save missingness percentage dataset mis_count_df = pd.DataFrame( [raw_ts[[c for c in raw_ts.columns if c != 'cluster']].iloc[indx].isna().astype(int) for indx in range(raw_ts.shape[0])]) mis_count_df['cluster'] = raw_ts.cluster cl_labels = np.repeat(sorted(raw_ts.cluster.unique().astype(str)), raw_ts.shape[1]) feat = np.array(raw_ts.columns) values = np.array(mis_perc[0]) for lab in range(1, len(raw_ts.cluster.unique())): feat = np.append(feat, np.array(raw_ts.columns)) values = np.append(values, np.array(mis_perc[lab])) plot_miss_heat(raw_ts, cl_labels, feat, values, period=period, hierarchy=hierarchy) return out, mis_count_df
cmap='rainbow_r') plt.title('UMAP-transformed training subsample of MNIST dataset (N=7,000)') plt.scatter(mnist_ts[:, 0], mnist_ts[:, 1], c=label_ts.astype(int), s=0.1, cmap='rainbow_r') plt.title('UMAP-transformed test subsample of MNIST dataset (N=7,000)') classifier = KNeighborsClassifier() clustering = AgglomerativeClustering() findbestclust = FindBestClustCV(nfold=2, nclust_range=list(range(2, 12)), s=classifier, c=clustering, nrand=10, n_jobs=1) metrics, nbest = findbestclust.best_nclust(mnist_tr, iter_cv=10, strat_vect=label_tr) out = findbestclust.evaluate(mnist_tr, mnist_ts, nbest) plot_metrics( metrics, title="Relative clustering validation performance on MNIST dataset") perm_lab = kuhn_munkres_algorithm(label_ts.astype(int), out.test_cllab) plt.scatter(mnist_ts[:, 0],
plt.scatter(mnist_tr[:, 0], mnist_tr[:, 1], c=label_tr.astype(int), s=0.1, cmap='rainbow_r') plt.title('UMAP-transformed training subsample of MNIST dataset (N=7,000)') plt.scatter(mnist_ts[:, 0], mnist_ts[:, 1], c=label_ts.astype(int), s=0.1, cmap='rainbow_r') plt.title('UMAP-transformed test subsample of MNIST dataset (N=7,000)') # Run relative clustering validation classifier = KNeighborsClassifier() clustering = AgglomerativeClustering() findbestclust = FindBestClustCV(nfold=10, nclust_range=[2, 12], s=classifier, c=clustering, nrand=100) metrics, nbest, _ = findbestclust.best_nclust(mnist_tr, label_tr) out = findbestclust.evaluate(mnist_tr, mnist_ts, nbest) plot_metrics(metrics, "Relative clustering validation performance on MNIST dataset") perm_lab = _kuhn_munkres_algorithm(label_ts.astype(int), out.test_cllab) plt.scatter(mnist_ts[:, 0], mnist_ts[:, 1], c=perm_lab, s=0.1, cmap='rainbow_r') plt.title("Predicted labels for MNIST test set") print(f"Best number of clusters: {nbest}") print(f"Test set external ACC: " f"{1 - zero_one_loss(label_ts.astype(int), perm_lab)}")
# ---------------------------------- data = make_blobs(1000, 10, centers=5, random_state=42) plt.scatter(data[0][:, 0], data[0][:, 1], c=data[1], cmap='rainbow_r') classifier = KNeighborsClassifier() clustering = AgglomerativeClustering() X_tr, X_ts, y_tr, y_ts = train_test_split(data[0], data[1], test_size=0.30, random_state=42, stratify=data[1]) findbestclust = FindBestClustCV(nfold=2, nclust_range=list(range(2, 7)), s=classifier, c=clustering, nrand=10) metrics, nbest = findbestclust.best_nclust(X_tr, iter_cv=10, strat_vect=y_tr) out = findbestclust.evaluate(X_tr, X_ts, nbest) plot_metrics(metrics, title="Reval performance") perm_lab = kuhn_munkres_algorithm(y_ts, out.test_cllab) print(f"Best number of clusters: {nbest}") print(f"Test set external ACC: " f"{1 - zero_one_loss(y_ts, perm_lab)}") print(f'AMI = {adjusted_mutual_info_score(y_ts, out.test_cllab)}') print( f"Validation set normalized stability (misclassification): {metrics['val'][nbest]}" ) print(f'Test set ACC = {out.test_acc}')
def gridsearch_cv(self, df, n_neigh, na_perc, cl_range, cv_fold, save=None): """ This function can be performed to decide which percentage of missing information to allow, and what's the best number of neighbors to consider both for the KNNImputer and the KNNClassifier. It takes as input the dataset as output from the function `create_dataset:dataset`. :param df: dataframe of merged instrument versions (longitudinal entries) :type df: pandas dataframe :param n_neigh: number of neighbors for imputation and classification :type n_neigh: tuple :param na_perc: max percentage of missing information allowed :type na_perc: tuple :param cl_range: range of minimum and maximum number of clusters to look for :type cl_range: tuple :param cv_fold: number of cross validation loop for RCV :type cv_fold: tuple :param save: whether to save the performance score table, defaults None Name of the file required :type save: str :return: best number of n_neigh, na_perc and cv_fold, and a summary of all performances The best performance is the one that has highest mean acc scores in both validation and test and the minimum mean amplitude of CIs :rtype: dict, pandas dataframe """ logging.disable(logging.CRITICAL) subdomain_feat = [c for c in df.columns if re.search('vscore', c) and not re.search('written', c)] domain_feat = [c for c in df.columns if re.search('totalb', c) and not re.search('composite', c)] if save is not None: with(open(os.path.join(ut.out_folder, f'{save}.csv'), 'w')) as f: wr = csv.writer(f, delimiter=';', lineterminator='\n') wr.writerow(['cv_fold', 'na_perc_thrs', 'n_neigh', 'period', 'feat_lev', 'N', 'nclust', 'val_acc', 'val_ci', 'test_acc']) transformer = UMAP(n_neighbors=30, min_dist=0.0, n_components=2, random_state=42) scores = {} for k in cv_fold: for nap in na_perc: dict_tr, dict_ts = self.prepare_cs_dataset(prepare_imputation(df, nap)) for n in n_neigh: scores.setdefault('cv', list()).append(k) scores.setdefault('na_perc', list()).append(nap) scores.setdefault('n_neigh', list()).append(n) impute = KNNImputer(n_neighbors=n) dict_imp = {p: _impute(dict_tr[p], dict_ts[p], impute) for p in self.include_age} knn = KNeighborsClassifier(n_neighbors=n) clust = AgglomerativeClustering(affinity='euclidean', linkage='ward') relval = FindBestClustCV(s=knn, c=clust, nfold=k, nclust_range=cl_range, nrand=100) # Run the model val_misclass = [] test_misclass = [] conf_width = [] for p, tup in dict_imp.items(): X_tr = transformer.fit_transform(tup[0][subdomain_feat]) X_ts = transformer.transform(tup[1][subdomain_feat]) metric, ncl, cv_scores = relval.best_nclust(X_tr) out = relval.evaluate(X_tr, X_ts, ncl) ci = (1 - (metric['val'][ncl][1][0] + metric['val'][ncl][1][1]), 1 - (metric['val'][ncl][1][0] - metric['val'][ncl][1][1])) val_misclass.append(metric['val'][ncl][0]) test_misclass.append(1 - out.test_acc) conf_width.append(ci[1] - ci[0]) if save is not None: with open(os.path.join(ut.out_folder, f'{save}.csv'), 'a') as f: wr = csv.writer(f, delimiter=';', lineterminator='\n') wr.writerow([k, nap, n, p, 'subdomain', (X_tr.shape[0], X_ts.shape[0]), ncl, 1 - metric['val'][ncl][0], ci, out.test_acc]) X_tr = transformer.fit_transform(tup[0][domain_feat]) X_ts = transformer.transform(tup[1][domain_feat]) metric, ncl, cv_scores = relval.best_nclust(X_tr) out = relval.evaluate(X_tr, X_ts, ncl) ci = (1 - (metric['val'][ncl][1][0] + metric['val'][ncl][1][1]), 1 - (metric['val'][ncl][1][0] - metric['val'][ncl][1][1])) val_misclass.append(metric['val'][ncl][0]) test_misclass.append(1 - out.test_acc) conf_width.append(ci[1] - ci[0]) if save is not None: with open(os.path.join(ut.out_folder, f'{save}.csv'), 'a') as f: wr = csv.writer(f, delimiter=';', lineterminator='\n') wr.writerow([k, nap, n, p, 'domain', (X_tr.shape[0], X_ts.shape[0]), ncl, 1 - metric['val'][ncl][0], ci, out.test_acc]) scores.setdefault('avg_val_ms', list()).append(np.mean(val_misclass)) scores.setdefault('avg_test_ms', list()).append(np.mean(test_misclass)) scores.setdefault('avg_conf_width', list()).append(np.mean(conf_width)) scores = pd.DataFrame(scores) minval = scores[['avg_val_ms', 'avg_test_ms', 'avg_conf_width']].apply(sum, 1).min() best_param = scores.loc[scores[['avg_val_ms', 'avg_test_ms', 'avg_conf_width']].apply(sum, 1) == minval] logging.disable(logging.NOTSET) logging.info(f"Best parameters selected: {best_param[['cv', 'na_perc', 'n_neigh']].to_dict('records')[0]} " f"-- Scores {best_param[['avg_val_ms', 'avg_test_ms', 'avg_conf_width']].to_dict('records')[0]}") return best_param[['cv', 'na_perc', 'n_neigh']].to_dict('records')[0], scores
def run_rcv(self, df, demo_info, na_perc, n_neigh, cv_fold, cl_range, scatter=False, heatmap=False): """ This function performs RCV method with fixed percentage of missing information, number of neighbors, number of cross validation. The range of clusters to consider still has to vary as desired. It is possible to flag :param scatter: and :param heatmap: to enable the visualization of UMAP scatterplots and the percentage fo missing information per feature for each cluster. The input dataset is a dataframe as returned by `strat:create_dataset:dataset` function (hence also longitudinal datasets). Finally, distance matrices for replication analysis are stored in the output folder. Also imputed datasets are saved to csv files. :param df: dataset obtained by merging different versions of the same instrument :type df: pandas dataframe :param demo_info: demographic information :type demo_info: dict :param na_perc: percentage of missing information :type na_perc: float :param n_neigh: number of neighbors for imputation and classification :type n_neigh: int :param cv_fold: number of cross validation iterations :type cv_fold: int :param cl_range: min/max number of clusters to consider :type cl_range: tuple :param scatter: flag for UMAP scatterplot (for training and test), defaults to False :type scatter: bool :param heatmap: flag for heatmap displaying percentage of missing information per feature for each cluster identified by the RCV method. Defaults to False. :type heatmap: bool :return: imputed datasets with clustering labels :rtype: dict """ flatui = ["#1f77b4", "#ff7f0e", "#2ca02c", "#d62728", "#9467bd", "#8c564b", "#e377c2", "#7f7f7f", "#bcbd22", "#17becf", "#8c564b", "#a55194"] subdomain_feat = [c for c in df.columns if re.search('vscore', c) and not re.search('written', c)] domain_feat = [c for c in df.columns if re.search('totalb', c) and not re.search('composite', c)] dict_tr, dict_ts = self.prepare_cs_dataset(prepare_imputation(df, na_perc)) transformer = UMAP(n_neighbors=30, min_dist=0.0, n_components=2, random_state=42) impute = KNNImputer(n_neighbors=n_neigh) dict_imp = {p: _impute(dict_tr[p], dict_ts[p], impute) for p in self.include_age} knn = KNeighborsClassifier(n_neighbors=n_neigh) clust = AgglomerativeClustering(affinity='euclidean', linkage='ward') relval = FindBestClustCV(s=knn, c=clust, nfold=cv_fold, nclust_range=cl_range, nrand=100) # Run the model for p, tup in dict_imp.items(): dict_imp[p][1]['sex'] = [demo_info[gui].sex for gui in dict_imp[p][1].index] dict_imp[p][1]['phenotype'] = [demo_info[gui].phenotype.replace("'", "") for gui in dict_imp[p][1].index] dict_imp[p][1]['race'] = [demo_info[gui].race for gui in dict_imp[p][1].index] dict_imp[p][1]['collection_id'] = [dict_ts[p].loc[gui].collection_id for gui in dict_imp[p][1].index] dict_imp[p][1]['interview_age'] = [dict_ts[p].loc[gui].interview_age for gui in dict_imp[p][1].index] X_tr = transformer.fit_transform(tup[0][subdomain_feat]) X_ts = transformer.transform(tup[1][subdomain_feat]) metric, ncl, cv_scores = relval.best_nclust(X_tr) out = relval.evaluate(X_tr, X_ts, ncl) logging.info(f"Best number of clusters: {ncl}") logging.info(f"Training ACC: {out.train_acc}, Test ACC: {out.test_acc}") dict_imp[p][0]['cluster_subdomain'], dict_imp[p][1][ 'cluster_subdomain'] = out.train_cllab + 1, out.test_cllab + 1 _, subj_mis = _check_na_perc(dict_ts[p][subdomain_feat]) dict_imp[p][1]['missing_subdomain'] = list(subj_mis.values()) plot_metrics(metric, f'UMAP preprocessed dataset, RCV misclassification performance at {p}, level subdomain') if scatter: _scatter_plot(X_tr, [(gui, cl) for gui, cl in zip(dict_imp[p][0].index, out.train_cllab + 1)], flatui, 10, 15, {str(ncl): '-'.join(['cluster', str(ncl)]) for ncl in sorted(np.unique(out.train_cllab + 1))}, title=f'Subgroups of UMAP preprocessed Vineland TRAINING ' f'dataset (period: {p} -- level: subdomain)') _scatter_plot(X_ts, [(gui, cl) for gui, cl in zip(dict_imp[p][1].index, out.test_cllab + 1)], flatui, 10, 15, {str(ncl): '-'.join(['cluster', str(ncl)]) for ncl in sorted(np.unique(out.test_cllab + 1))}, title=f'Subgroups of UMAP preprocessed Vineland TEST ' f'dataset (period: {p} -- level: subdomain)') if heatmap: dict_ts[p]['cluster'] = out.test_cllab + 1 feat = [] values = [] cl_labels = np.repeat(sorted(dict_ts[p].cluster.unique().astype(str)), len(subdomain_feat)) for lab in np.unique(sorted(out.test_cllab + 1)): na_feat, _ = _check_na_perc(dict_ts[p].loc[dict_ts[p].cluster == lab][subdomain_feat]) feat.extend(list(na_feat.keys())) values.extend(list(na_feat.values())) plot_miss_heat(dict_ts[p], cl_labels, feat, values, period=p, hierarchy='subdomain') X_tr = transformer.fit_transform(tup[0][domain_feat]) X_ts = transformer.transform(tup[1][domain_feat]) metric, ncl, cv_scores = relval.best_nclust(X_tr) plot_metrics(metric, f'UMAP preprocessed dataset, RCV misclassification performance at {p}, level domain') out = relval.evaluate(X_tr, X_ts, ncl) logging.info(f"Best number of clusters: {ncl}") logging.info(f"Training ACC: {out.train_acc}, Test ACC: {out.test_acc}") dict_imp[p][0]['cluster_domain'], dict_imp[p][1]['cluster_domain'] = out.train_cllab + 1, out.test_cllab + 1 _, subj_mis = _check_na_perc(dict_ts[p][domain_feat]) dict_imp[p][1]['missing_domain'] = list(subj_mis.values()) if scatter: _scatter_plot(X_tr, [(gui, cl) for gui, cl in zip(dict_imp[p][0].index, out.train_cllab + 1)], flatui, 10, 15, {str(ncl): '-'.join(['cluster', str(ncl)]) for ncl in sorted(np.unique(out.train_cllab + 1))}, title=f'Subgroups of UMAP preprocessed Vineland TRAINING ' f'dataset (period: {p} -- level: domain)') _scatter_plot(X_ts, [(gui, cl) for gui, cl in zip(dict_imp[p][1].index, out.test_cllab + 1)], flatui, 10, 15, {str(ncl): '-'.join(['cluster', str(ncl)]) for ncl in sorted(np.unique(out.test_cllab + 1))}, title=f'Subgroups of UMAP preprocessed Vineland TEST ' f'dataset (period: {p} -- level: domain)') if heatmap: dict_ts[p]['cluster'] = out.test_cllab + 1 feat = [] values = [] cl_labels = np.repeat(sorted(dict_ts[p].cluster.unique().astype(str)), len(domain_feat)) for lab in np.unique(sorted(out.test_cllab + 1)): na_feat, _ = _check_na_perc(dict_ts[p].loc[dict_ts[p].cluster == lab][domain_feat]) feat.extend(list(na_feat.keys())) values.extend(list(na_feat.values())) plot_miss_heat(dict_ts[p], cl_labels, feat, values, period=p, hierarchy='subdomain') logging.info("Saving train/test datasets with new cluster") new_dict_imp = relabel(dict_imp, plot_scatter=False) for p in new_dict_imp.keys(): new_dict_imp[p][0].to_csv(os.path.join(ut.out_folder, f'imputed_data_{p}_tr.csv'), index_label='subjectkey') new_dict_imp[p][1].to_csv(os.path.join(ut.out_folder, f'imputed_data_{p}.csv'), index_label='subjectkey') logging.info("Building distance matrix...") _build_distmat(imp_dict=dict_imp) return dict_imp
def example1(): # Generate dataset data = make_blobs(1000, 2, centers=5, center_box=(-20, 20), random_state=42) # Visualize dataset plt.figure(figsize=(6, 4)) for i in range(5): plt.scatter(data[0][data[1] == i][:, 0], data[0][data[1] == i][:, 1], label=i, cmap='tab20') plt.title("Blobs dataset") # plt.savefig('./blobs.png', format='png') plt.show() # Create training and test sets X_tr, X_ts, y_tr, y_ts = train_test_split(data[0], data[1], test_size=0.30, random_state=42, stratify=data[1]) # Initialize clustering and classifier classifier = KNeighborsClassifier(n_neighbors=15) clustering = KMeans() # Run relatve validation (repeated CV and testing) findbestclust = FindBestClustCV(nfold=2, nclust_range=list(range(2, 7, 1)), s=classifier, c=clustering, nrand=10, n_jobs=N_JOBS) metrics, nbest = findbestclust.best_nclust(X_tr, iter_cv=10, strat_vect=y_tr) out = findbestclust.evaluate(X_tr, X_ts, nclust=nbest) # Plot CV metrics plot_metrics(metrics, prob_lines=False) logging.info(f"Validation stability: {metrics['val'][nbest]}") perm_lab = kuhn_munkres_algorithm(y_ts, out.test_cllab) logging.info(f"Best number of clusters: {nbest}") logging.info(f'AMI (true labels vs predicted labels) for test set = ' f'{adjusted_mutual_info_score(y_ts, out.test_cllab)}') logging.info('\n\n') # Compute metrics logging.info("Metrics from true label comparisons on test set:") class_scores = compute_metrics(y_ts, perm_lab, perm=False) for k, val in class_scores.items(): if k in ['F1', 'MCC']: logging.info(f"{k}, {val}") logging.info("\n\n") # Internal measures # SILHOUETTE logging.info("Silhouette score based selection") sil_score_tr, sil_best_tr, sil_label_tr = select_best(X_tr, clustering, silhouette_score, select='max', nclust_range=list(range(2, 7, 1))) sil_score_ts, sil_best_ts, sil_label_ts = select_best(X_ts, clustering, silhouette_score, select='max', nclust_range=list(range(2, 7, 1))) sil_eval = evaluate_best(X_ts, clustering, silhouette_score, sil_best_tr) logging.info(f"Best number of clusters (and scores) for tr/ts independent runs: " f"{sil_best_tr}({sil_score_tr})/{sil_best_ts}({sil_score_ts})") logging.info(f"Test set evaluation {sil_eval}") logging.info(f'AMI (true labels vs clustering labels) training = ' f'{adjusted_mutual_info_score(y_tr, kuhn_munkres_algorithm(y_tr, sil_label_tr))}') logging.info(f'AMI (true labels vs clustering labels) test = ' f'{adjusted_mutual_info_score(y_ts, kuhn_munkres_algorithm(y_ts, sil_label_ts))}') logging.info('\n\n') # DAVIES-BOULDIN logging.info("Davies-Bouldin score based selection") db_score_tr, db_best_tr, db_label_tr = select_best(X_tr, clustering, davies_bouldin_score, select='min', nclust_range=list(range(2, 7, 1))) db_score_ts, db_best_ts, db_label_ts = select_best(X_ts, clustering, davies_bouldin_score, select='min', nclust_range=list(range(2, 7, 1))) db_eval = evaluate_best(X_ts, clustering, davies_bouldin_score, db_best_tr) logging.info(f"Best number of clusters (and scores) for tr/ts independent runs: " f"{db_best_tr}({db_score_tr})/{db_best_ts}({db_score_ts})") logging.info(f"Test set evaluation {db_eval}") logging.info(f'AMI (true labels vs clustering labels) training = ' f'{adjusted_mutual_info_score(y_tr, kuhn_munkres_algorithm(y_tr, db_label_tr))}') logging.info(f'AMI (true labels vs clustering labels) test = ' f'{adjusted_mutual_info_score(y_ts, kuhn_munkres_algorithm(y_ts, db_label_ts))}') logging.info('\n\n') # Plot true vs predicted labels for test sets plt.figure(figsize=(6, 4)) for i in range(5): plt.scatter(X_ts[y_ts == i][:, 0], X_ts[y_ts == i][:, 1], label=str(i), cmap='tab20') plt.legend(loc=3) plt.title("Test set true labels") # plt.savefig('./blobs_true.png', format='png') plt.show() plt.figure(figsize=(6, 4)) for i in range(5): plt.scatter(X_ts[perm_lab == i][:, 0], X_ts[perm_lab == i][:, 1], label=str(i), cmap='tab20') plt.legend(loc=3) plt.title("Test set clustering labels") # plt.savefig('./blobs_clustering.png', format='png') plt.show()
def example2(): mnist = fetch_openml('mnist_784', version=1) mnist.target = mnist.target.astype(int) X_tr, y_tr = mnist['data'][:60000], mnist.target[:60000] X_ts, y_ts = mnist['data'][60000::], mnist.target[60000::] transform = UMAP(n_components=2, random_state=42, n_neighbors=30, min_dist=0.0) X_tr = transform.fit_transform(X_tr) X_ts = transform.transform(X_ts) s = KNeighborsClassifier(n_neighbors=30) c = hdbscan.HDBSCAN(min_samples=10, min_cluster_size=200) reval = FindBestClustCV(s=s, c=c, nfold=2, nrand=10, n_jobs=N_JOBS) metrics, nclustbest, tr_lab = reval.best_nclust(X_tr, iter_cv=10, strat_vect=y_tr) plot_metrics(metrics) out = reval.evaluate(X_tr, X_ts, nclust=nclustbest, tr_lab=tr_lab) perm_lab = kuhn_munkres_algorithm(y_ts, out.test_cllab) logging.info(f"Validation stability: {metrics['val'][nclustbest]}") logging.info(f"Best number of clusters during CV: {nclustbest}") logging.info(f"Best number of clusters on test set: " f"{len([lab for lab in np.unique(out.test_cllab) if lab >= 0])}") logging.info(f'AMI (true labels vs predicted labels) = ' f'{adjusted_mutual_info_score(y_ts, out.test_cllab)}') logging.info('\n\n') logging.info("Metrics from true label comparisons on test set:") class_scores = compute_metrics(y_ts, perm_lab) for k, val in class_scores.items(): logging.info(f'{k}, {val}') logging.info('\n\n') # Visualization fig, ax = plt.subplots(figsize=(10, 8)) scatter = ax.scatter(X_tr[:, 0], X_tr[:, 1], c=y_tr, cmap='rainbow_r', s=0.1) legend = ax.legend(*scatter.legend_elements()) ax.add_artist(legend) plt.title("Train set true labels (digits dataset)") plt.show() fig, ax = plt.subplots(figsize=(10, 8)) scatter = ax.scatter(X_tr[:, 0], X_tr[:, 1], c=kuhn_munkres_algorithm(y_tr, tr_lab), cmap='tab20', s=0.1) legend = ax.legend(*scatter.legend_elements()) ax.add_artist(legend) plt.title("Train set predicted labels (digits dataset)") plt.show() fig, ax = plt.subplots(figsize=(10, 8)) scatter = ax.scatter(X_ts[:, 0], X_ts[:, 1], c=y_ts, cmap='tab20', s=0.1) legend = ax.legend(*scatter.legend_elements()) ax.add_artist(legend) plt.title("Test set true labels (digits dataset)") plt.show() fig, ax = plt.subplots(figsize=(10, 8)) scatter = ax.scatter(X_ts[:, 0], X_ts[:, 1], s=0.1, c=perm_lab, cmap='tab20') legend = ax.legend(*scatter.legend_elements()) ax.add_artist(legend) plt.title("Test set clustering labels (digits dataset)") plt.show() # Internal measures # SILHOUETTE logging.info("Silhouette score based selection") sil_score_tr, sil_best_tr, sil_label_tr = select_best(X_tr, c, silhouette_score, select='max') sil_score_ts, sil_best_ts, sil_label_ts = select_best(X_ts, c, silhouette_score, select='max') logging.info( f"Best number of clusters (and scores) for tr/ts independent runs: " f"{sil_best_tr}({sil_score_tr})/{sil_best_ts}({sil_score_ts})") logging.info(f'AMI (true labels vs clustering labels) training = ' f'{adjusted_mutual_info_score(y_tr, kuhn_munkres_algorithm(y_tr, sil_label_tr))}') logging.info(f'AMI (true labels vs clustering labels) test = ' f'{adjusted_mutual_info_score(y_ts, kuhn_munkres_algorithm(y_ts, sil_label_ts))}') logging.info('\n\n') # DAVIES-BOULDIN logging.info("Davies-Bouldin score based selection") db_score_tr, db_best_tr, db_label_tr = select_best(X_tr, c, davies_bouldin_score, select='min') db_score_ts, db_best_ts, db_label_ts = select_best(X_ts, c, davies_bouldin_score, select='min') logging.info( f"Best number of clusters (and scores) for tr/ts independent runs: " f"{db_best_tr}({db_score_tr})/{db_best_ts}({db_score_ts})") logging.info(f'AMI (true labels vs clustering labels) training = ' f'{adjusted_mutual_info_score(y_tr, kuhn_munkres_algorithm(y_tr, db_label_tr))}') logging.info(f'AMI (true labels vs clustering labels) test = ' f'{adjusted_mutual_info_score(y_ts, kuhn_munkres_algorithm(y_ts, db_label_ts))}') logging.info('\n\n') # Visualization fig, ax = plt.subplots(figsize=(10, 8)) scatter = ax.scatter(X_tr[:, 0], X_tr[:, 1], c=sil_label_tr, cmap='tab20', s=0.1) legend = ax.legend(*scatter.legend_elements()) ax.add_artist(legend) plt.title("Train set silhouette labels (digits dataset)") plt.show() fig, ax = plt.subplots(figsize=(10, 8)) scatter = ax.scatter(X_ts[:, 0], X_ts[:, 1], c=sil_label_ts, cmap='tab20', s=0.1) legend = ax.legend(*scatter.legend_elements()) ax.add_artist(legend) legend = ax.legend(*scatter.legend_elements()) ax.add_artist(legend) plt.title("Test set silhouette labels (digits dataset)") plt.show() fig, ax = plt.subplots(figsize=(10, 8)) scatter = ax.scatter(X_tr[:, 0], X_tr[:, 1], c=db_label_tr, cmap='tab20', s=0.1) legend = ax.legend(*scatter.legend_elements()) ax.add_artist(legend) plt.title("Train set Davies-Bouldin labels (digits dataset)") plt.show() fig, ax = plt.subplots(figsize=(10, 8)) scatter = ax.scatter(X_ts[:, 0], X_ts[:, 1], s=0.1, c=db_label_ts, cmap='tab20') legend = ax.legend(*scatter.legend_elements()) ax.add_artist(legend) plt.title("Test set Davies-Bouldin labels (digits dataset)") plt.show()
plt.scatter(data1[0][:, 0], data1[0][:, 1], c=data1[1], cmap='rainbow_r') plt.title('True labels for 10-feature dataset') X_tr, X_ts, y_tr, y_ts = train_test_split(data1[0], data1[1], test_size=0.30, random_state=42, stratify=data1[1]) # Apply relative clustering validation with KNN and Hierarchical clustering classifier = KNeighborsClassifier() clustering = AgglomerativeClustering() findbestclust = FindBestClustCV(nfold=10, nclust_range=list(range(2, 7)), s=classifier, c=clustering, nrand=100) metrics, nbest = findbestclust.best_nclust(data=X_tr, strat_vect=y_tr) out = findbestclust.evaluate(X_tr, X_ts, nbest) plot_metrics(metrics, title="Reval performance for synthetic dataset with 10 features") data2 = make_blobs(1000, 20, centers=5, cluster_std=5, random_state=42) plt.scatter(data2[0][:, 0], data2[0][:, 1], c=data2[1], cmap='rainbow_r') plt.title('True labels for 20-feature dataset') X_tr, X_ts, y_tr, y_ts = train_test_split(data2[0], data2[1],
def blobs_performance(): """ Function performing multiple iterations of reval on simulated 5-blob datasets with varying number of samples and features and 10 repetitions of 10-fold CVs. The function plots the performance (in seconds) of the algorithm for each parameter configuration. """ feat = [10, 100, 500, 1000] samples = [100, 500, 1000, 1500, 2000] perftime = [] for s in samples: perf = [] for f in feat: start = time.time() data = make_blobs(s, f, 5, center_box=(-20, 20), random_state=42) X_tr, X_ts, y_tr, y_ts = train_test_split(data[0], data[1], test_size=0.30, random_state=42, stratify=data[1]) classifier = KNeighborsClassifier(n_neighbors=5) clustering = KMeans() findbestclust = FindBestClustCV(nfold=10, nclust_range=[2, 7], s=classifier, c=clustering, nrand=100) metrics, nbest, _ = findbestclust.best_nclust(X_tr, iter_cv=10, strat_vect=y_tr) tmp_time = time.time() - start perf.append(tmp_time) print( f'Feat {f}, samples {s}: N cluster {nbest}, time: {tmp_time}') perftime.append(perf) perftime = np.array(perftime) fig, ax = plt.subplots() ax.plot(samples, perftime[:, 0], label='10 features', linestyle='--', color='black') ax.plot(samples, perftime[:, 1], label='100 features', color='black') ax.plot(samples, perftime[:, 2], label='500 features', linestyle='-.', color='black') ax.plot(samples, perftime[:, 3], label='1000 features', linestyle=':', color='black') ax.set_xlabel('Number of samples') ax.set_ylabel('Execution time (s)') ax.set_title("") ax.legend() plt.savefig('./performance_blobs.png', dpi=300)
plt.scatter(data[0][:, 0], data[0][:, 1], c=data[1], cmap='rainbow_r') # Split them into training and test set (30% of data) X_tr, X_ts, y_tr, y_ts = train_test_split(data[0], data[1], test_size=0.30, random_state=42, stratify=data[1]) # Apply relative clustering validation with KNN and Hierarchical clustering classifier = KNeighborsClassifier() clustering = AgglomerativeClustering() findbestclust = FindBestClustCV(nfold=10, nclust_range=[2, 7], s=classifier, c=clustering, nrand=100) metrics, nbest, _ = findbestclust.best_nclust(X_tr, y_tr) out = findbestclust.evaluate(X_tr, X_ts, nbest) perm_lab = _kuhn_munkres_algorithm(y_ts, out.test_cllab) print(f"Best number of clusters: {nbest}") print(f"Test set external ACC: " f"{1 - zero_one_loss(y_ts, perm_lab)}") print(f'AMI = {adjusted_mutual_info_score(y_ts, out.test_cllab)}') print( f"Validation set normalized stability (misclassification): {metrics['val'][nbest]}" ) print(f'Test set ACC = {out.test_acc}')
def time_cmplx(n_jobs=1): data = make_blobs(100, 10, centers=2) data_tr, data_ts, y_tr, y_ts = train_test_split(data[0], data[1], test_size=0.5, stratify=data[1], random_state=42) s = [ KNeighborsClassifier(), SVC(), LogisticRegression(), RandomForestClassifier() ] c = [HDBSCAN(), AgglomerativeClustering(), KMeans(), SpectralClustering()] param = itertools.product(s, c) labels = ['KNN'] * 4 + ['SVM'] * 4 + ["LR"] * 4 + ['RF'] * 4 time_cv = {'LR': [], 'KNN': [], 'RF': [], 'SVM': []} time_ev = {'LR': [], 'KNN': [], 'RF': [], 'SVM': []} for idx, mod in enumerate(param): classifier, clustering = mod[0], mod[1] findbest = FindBestClustCV(s=classifier, c=clustering, nrand=10, nfold=2, n_jobs=n_jobs, nclust_range=list(range(2, 7, 1))) if isinstance(clustering, HDBSCAN): start = time.time() _, _, tr_lab = findbest.best_nclust(data_tr, iter_cv=10, strat_vect=y_tr) time_cv[labels[idx]].append(time.time() - start) start = time.time() findbest.evaluate(data_tr, data_ts, nclust=2, tr_lab=tr_lab) time_ev[labels[idx]].append(time.time() - start) else: start = time.time() _, _ = findbest.best_nclust(data_tr, iter_cv=10, strat_vect=y_tr) time_cv[labels[idx]].append(time.time() - start) start = time.time() findbest.evaluate(data_tr, data_ts, nclust=2) time_ev[labels[idx]].append(time.time() - start) pkl.dump(time_cv, open(f'time_cv_njobs{n_jobs}.pkl', 'wb')) pkl.dump(time_ev, open(f'time_ev_njobs{n_jobs}.pkl', 'wb')) clustering = KMeans() classifier = KNeighborsClassifier() time_knnkmeans = {10: [], 100: [], 1000: []} for nsamples, nfeatures in itertools.product( [100, 500, 1000, 1500, 2000, 2500, 3000], [10, 100, 1000]): data = make_blobs(nsamples, nfeatures, centers=2) data_tr, data_ts, y_tr, y_ts = train_test_split(data[0], data[1], test_size=0.5, stratify=data[1], random_state=42) findbest = FindBestClustCV(s=classifier, c=clustering, nrand=10, nfold=2, n_jobs=n_jobs, nclust_range=list(range(2, 7, 1))) start = time.time() _, _ = findbest.best_nclust(data_tr, iter_cv=10, strat_vect=y_tr) findbest.evaluate(data_tr, data_ts, nclust=2) time_knnkmeans[nfeatures].append(time.time() - start) pkl.dump(time_knnkmeans, open(f'time_knnkmeans{n_jobs}.pkl', 'wb'))