def cluster_changepoints_level1(self): print "Level1 : Clustering changepoints in Z(t)" if constants.REMOTE == 1: if self.fit_DPGMM: print "DPGMM L1 - start" # Previously, when L0 was GMM, alpha = 0.4 print "L1 ", str( len(self.list_of_cp) / constants.DPGMM_DIVISOR_L1), " ALPHA ", 10 dpgmm = mixture.DPGMM(n_components=int( len(self.list_of_cp) / 6), covariance_type='diag', n_iter=1000, alpha=10, thresh=1e-7) print "DPGMM L1 - end" if self.fit_GMM: gmm = mixture.GMM(n_components=self.n_components_L1, covariance_type='full', n_iter=1000, thresh=5e-5) print "GMM L1 - end" elif constants.REMOTE == 2: gmm = mixture.GMM(n_components=self.n_components_L1, covariance_type='full') else: gmm = mixture.GMM(n_components=self.n_components_L1, covariance_type='full') if self.fit_GMM: gmm.fit(self.change_pts_Z) Y_gmm = gmm.predict(self.change_pts_Z) Y = Y_gmm if self.fit_DPGMM: Y = [] i = 0 while True: print "In DPGMM Fit loop" dpgmm.fit(self.change_pts_Z) Y = dpgmm.predict(self.change_pts_Z) if len(set(Y)) > 1: break i += 1 if i > 100: break self.save_cluster_metrics(self.change_pts_Z, Y, 'level1') for i in range(len(Y)): label = constants.alphabet_map[Y[i] + 1] self.map_cp2level1[i] = label utils.dict_insert_list(label, i, self.map_level12cp) self.generate_l2_cluster_matrices()
def cluster_changepoints(self): """ Clusters changepoints specified in self.list_of_cp. """ print "Clustering changepoints..." print "L1 ", str(len(self.list_of_cp) / constants.DPGMM_DIVISOR_L1), " ALPHA: ", self.ALPHA_L1 if constants.REMOTE == 1: if self.fit_DPGMM: dpgmm = mixture.DPGMM( n_components=int(len(self.list_of_cp) / constants.DPGMM_DIVISOR_L1), covariance_type="diag", n_iter=10000, alpha=self.ALPHA_L1, thresh=1e-4, ) if self.fit_GMM: gmm = mixture.GMM(n_components=self.n_components_L1, covariance_type="full", n_iter=5000, thresh=0.01) elif constants.REMOTE == 2: gmm = mixture.GMM(n_components=self.n_components_L1, covariance_type="full", thresh=0.01) else: gmm = mixture.GMM(n_components=self.n_components_L1, covariance_type="full") if self.fit_GMM: gmm.fit(self.changepoints) predictions_gmm = gmm.predict(self.changepoints) print "L1: Clusters in GMM", len(set(predictions_gmm)) predictions = predictions_gmm if self.fit_DPGMM: predictions = [] while True: print "Inside loop" dpgmm.fit(self.changepoints) predictions = dpgmm.predict(self.changepoints) if len(set(predictions)) > 1: break print "L1: Clusters in DP-GMM", len(set(predictions)) for i in range(len(predictions)): label = constants.alphabet_map[predictions[i] + 1] self.map_cp2cluster[i] = label utils.dict_insert_list(label, i, self.map_level1_cp) demonstration = self.map_cp2demonstrations[i] frm = self.map_cp2frm[i] try: surgeme = self.map_frm2surgeme[demonstration][frm] except KeyError as e: print e sys.exit() utils.print_and_write(("%3d %s %s %3d %3d\n" % (i, label, demonstration, frm, surgeme)), self.log)
def cluster_changepoints_level1(self): print "Level1 : Clustering changepoints in Z(t)" if constants.REMOTE == 1: if self.fit_DPGMM: print "DPGMM L1 - start" # Previously, when L0 was GMM, alpha = 0.4 print "L1 ", str(len(self.list_of_cp) / constants.DPGMM_DIVISOR_L1), " ALPHA ", 10 dpgmm = mixture.DPGMM( n_components=int(len(self.list_of_cp) / 6), covariance_type="diag", n_iter=1000, alpha=10, thresh=1e-7, ) print "DPGMM L1 - end" if self.fit_GMM: gmm = mixture.GMM(n_components=self.n_components_L1, covariance_type="full", n_iter=1000, thresh=5e-5) print "GMM L1 - end" elif constants.REMOTE == 2: gmm = mixture.GMM(n_components=self.n_components_L1, covariance_type="full") else: gmm = mixture.GMM(n_components=self.n_components_L1, covariance_type="full") if self.fit_GMM: gmm.fit(self.change_pts_Z) Y_gmm = gmm.predict(self.change_pts_Z) Y = Y_gmm if self.fit_DPGMM: Y = [] i = 0 while True: print "In DPGMM Fit loop" dpgmm.fit(self.change_pts_Z) Y = dpgmm.predict(self.change_pts_Z) if len(set(Y)) > 1: break i += 1 if i > 100: break self.save_cluster_metrics(self.change_pts_Z, Y, "level1") for i in range(len(Y)): label = constants.alphabet_map[Y[i] + 1] self.map_cp2level1[i] = label utils.dict_insert_list(label, i, self.map_level12cp) self.generate_l2_cluster_matrices()
def cluster_changepoints(self): """ Clusters changepoints specified in self.list_of_cp. """ print "Clustering changepoints..." print "L1 ", str(len(self.list_of_cp)/constants.DPGMM_DIVISOR_L1)," ALPHA: ", self.ALPHA_L1 if constants.REMOTE == 1: if self.fit_DPGMM: dpgmm = mixture.DPGMM(n_components = int(len(self.list_of_cp)/constants.DPGMM_DIVISOR_L1), covariance_type='diag', n_iter = 10000, alpha = self.ALPHA_L1, thresh= 1e-4) if self.fit_GMM: gmm = mixture.GMM(n_components = self.n_components_L1, covariance_type='full', n_iter=5000, thresh = 0.01) elif constants.REMOTE == 2: gmm = mixture.GMM(n_components = self.n_components_L1, covariance_type='full', thresh = 0.01) else: gmm = mixture.GMM(n_components = self.n_components_L1, covariance_type='full') if self.fit_GMM: gmm.fit(self.changepoints) predictions_gmm = gmm.predict(self.changepoints) print "L1: Clusters in GMM",len(set(predictions_gmm)) predictions = predictions_gmm if self.fit_DPGMM: predictions = [] while True: print "Inside loop" dpgmm.fit(self.changepoints) predictions = dpgmm.predict(self.changepoints) if len(set(predictions)) > 1: break print "L1: Clusters in DP-GMM", len(set(predictions)) for i in range(len(predictions)): label = constants.alphabet_map[predictions[i] + 1] self.map_cp2cluster[i] = label utils.dict_insert_list(label, i, self.map_level1_cp) demonstration = self.map_cp2demonstrations[i] frm = self.map_cp2frm[i] try: surgeme = self.map_frm2surgeme[demonstration][frm] except KeyError as e: print e sys.exit() utils.print_and_write(("%3d %s %s %3d %3d\n" % (i, label, demonstration, frm, surgeme)), self.log)
def cluster_changepoints_level1(self): # print "Level1 : Clustering changepoints in Z(t)" gmm = mixture.GMM(n_components = 10, covariance_type='full') gmm.fit(self.change_pts_Z) Y = gmm.predict(self.change_pts_Z) self.save_cluster_metrics(self.change_pts_Z, Y, gmm.means_, 'level1', gmm) for i in range(len(Y)): label = constants.alphabet_map[Y[i] + 1] self.map_cp_level1[i] = label utils.dict_insert_list(label, i, self.map_level1_cp) self.generate_l2_cluster_matrices()
def get_time_clusters(data, T_COMPONENTS): # k-fold validation (Leave one out) numDemos = len(data.keys()) + 1 sizeTestSet = numDemos - 1 list_of_frms = [] all_frms = [] for key in data.keys(): elem = data[key] list_of_frms.append(elem) all_frms += elem N_COMPONENTS = min(T_COMPONENTS, len(all_frms)) time_cluster = mixture.GMM(n_components=N_COMPONENTS, covariance_type='full', n_iter=50000, thresh=5e-7) X = np.array(all_frms) X = X.reshape(len(all_frms), 1) time_cluster.fit(X) Y = time_cluster.predict(X) means = time_cluster.means_ covars = time_cluster.covars_ #dpgmm = mixture.DPGMM(n_components = numDemos * 3, covariance_type='diag', n_iter = 10000, alpha = 1000, thresh= 1e-10) #dpgmm.fit(X) #Y = dpgmm.predict(X) #means = dpgmm.means_ list_of_elem = [] list_of_means = [] for i in range(len(Y)): list_of_elem.append( (Y[i], X[i], means[Y[i]][0], np.sqrt(covars[Y[i]][0][0]))) list_of_elem = sorted(list_of_elem, key=lambda x: x[1][0]) dict_time_clusters = {} for elem in list_of_elem: utils.dict_insert_list(elem[0], elem[1], dict_time_clusters) list_time_clusters = [] list_time_clusters_noPrune = [] for cluster in dict_time_clusters.keys(): # get all frames in this cluster cluster_frames = dict_time_clusters[cluster] setClusterFrames = set([elem[0] for elem in cluster_frames]) # test if frames in cluster are representative of the test set rep = [] for id in range(sizeTestSet): elemSet = set(list_of_frms[id]) commonElem = elemSet.intersection(setClusterFrames) id_in_cluster = 1. if len(commonElem) > 0 else 0. rep.append(id_in_cluster) pruneCluster = True if sum( rep) / sizeTestSet < constants.PRUNING_FACTOR_T else False min_frm = min(cluster_frames) max_frm = max(cluster_frames) mean = means[cluster][0] list_of_means.append(int(mean)) std = np.sqrt(covars[cluster][0][0]) leftFrame = max(min_frm[0], mean - std) rightFrame = min(max_frm[0], mean + std) list_time_clusters_noPrune.append((leftFrame, rightFrame)) # keep for plotting is pruneFlag = 0 if not (pruneCluster): # list_time_clusters.append((min_frm[0], max_frm[0])) list_time_clusters.append((leftFrame, rightFrame)) print "Number of Clusters pruned in Time Clustering: ", len( list_time_clusters_noPrune) - len(list_time_clusters) labels_automatic, colors_automatic = setup_automatic_labels_2( list_time_clusters, "k") labels_automatic_0, colors_automatic_0 = setup_automatic_labels( list_of_frms[0], "k") return labels_automatic, colors_automatic, labels_automatic_0, colors_automatic_0, list_of_means, list_of_frms
def post_evaluation_multimodal(metrics, file, filename, list_of_demonstrations, feat_fname): mutual_information_1 = [] normalized_mutual_information_1 = [] adjusted_mutual_information_1 = [] homogeneity_1 = [] precision_1_micro = [] recall_1_micro = [] precision_1_macro = [] recall_1_macro = [] precision_1_weighted = [] recall_1_weighted = [] mutual_information_2 = [] normalized_mutual_information_2 = [] adjusted_mutual_information_2 = [] homogeneity_2 = [] precision_2_micro = [] recall_2_micro = [] precision_2_macro = [] recall_2_macro = [] precision_2_weighted = [] recall_2_weighted = [] silhoutte_level_1_global = [] silhoutte_level_1_weighted = [] dunn1_level_1 = [] dunn2_level_1 = [] dunn3_level_1 = [] silhoutte_level_2_global = [] silhoutte_level_2_weighted = [] dunn1_level_2 = [] dunn2_level_2 = [] dunn3_level_2 = [] list_of_frms = {} for elem in metrics: precision_1_micro.append(elem[0]["precision_micro"]) precision_2_micro.append(elem[1]["precision_micro"]) precision_1_macro.append(elem[0]["precision_macro"]) precision_2_macro.append(elem[1]["precision_macro"]) precision_1_weighted.append(elem[0]["precision_weighted"]) precision_2_weighted.append(elem[1]["precision_weighted"]) recall_1_micro.append(elem[0]["recall_micro"]) recall_2_micro.append(elem[1]["recall_micro"]) recall_1_macro.append(elem[0]["recall_macro"]) recall_2_macro.append(elem[1]["recall_macro"]) recall_1_weighted.append(elem[0]["recall_weighted"]) recall_2_weighted.append(elem[1]["recall_weighted"]) mutual_information_1.append(elem[0]["mutual_info_score"]) mutual_information_2.append(elem[1]["mutual_info_score"]) normalized_mutual_information_1.append( elem[0]["normalized_mutual_info_score"]) normalized_mutual_information_2.append( elem[1]["normalized_mutual_info_score"]) adjusted_mutual_information_1.append( elem[0]["adjusted_mutual_info_score"]) adjusted_mutual_information_2.append( elem[1]["adjusted_mutual_info_score"]) homogeneity_1.append(elem[0]["homogeneity_score"]) homogeneity_2.append(elem[1]["homogeneity_score"]) silhoutte_level_1_global.append(elem[2]["level1"]) dunn1_level_1.append(elem[3]["level1"]) dunn2_level_1.append(elem[4]["level1"]) dunn3_level_1.append(elem[5]["level1"]) silhoutte_level_2_global += elem[6] dunn1_level_2 += elem[7] dunn2_level_2 += elem[8] dunn3_level_2 += elem[9] viz = elem[10] for demonstration in viz.keys(): utils.dict_insert_list(demonstration, viz[demonstration], list_of_frms) silhoutte_level_1_weighted.append(elem[11]["level1"]) silhoutte_level_2_weighted += elem[12] utils.print_and_write_2("precision_1_micro", np.mean(precision_1_micro), np.std(precision_1_micro), file) utils.print_and_write_2("precision_2_micro", np.mean(precision_2_micro), np.std(precision_2_micro), file) utils.print_and_write_2("precision_1_macro", np.mean(precision_1_macro), np.std(precision_1_macro), file) utils.print_and_write_2("precision_2_macro", np.mean(precision_2_macro), np.std(precision_2_macro), file) utils.print_and_write_2("precision_1_weighted", np.mean(precision_1_weighted), np.std(precision_1_weighted), file) utils.print_and_write_2("precision_2_weighted", np.mean(precision_2_weighted), np.std(precision_2_weighted), file) utils.print_and_write_2("recall_1_micro", np.mean(recall_1_micro), np.std(recall_1_micro), file) utils.print_and_write_2("recall_2_micro", np.mean(recall_2_micro), np.std(recall_2_micro), file) utils.print_and_write_2("recall_1_macro", np.mean(recall_1_macro), np.std(recall_1_macro), file) utils.print_and_write_2("recall_2_macro", np.mean(recall_2_macro), np.std(recall_2_macro), file) utils.print_and_write_2("recall_1_weighted", np.mean(recall_1_weighted), np.std(recall_1_weighted), file) utils.print_and_write_2("recall_2_weighted", np.mean(recall_2_weighted), np.std(recall_2_weighted), file) utils.print_and_write_2("mutual_info_1", np.mean(mutual_information_1), np.std(mutual_information_1), file) utils.print_and_write_2("mutual_info_2", np.mean(mutual_information_2), np.std(mutual_information_2), file) utils.print_and_write_2("normalized_mutual_info_1", np.mean(normalized_mutual_information_1), np.std(normalized_mutual_information_1), file) utils.print_and_write_2("normalized_mutual_info_2", np.mean(normalized_mutual_information_2), np.std(normalized_mutual_information_2), file) utils.print_and_write_2("adjusted_mutual_info_1", np.mean(adjusted_mutual_information_1), np.std(adjusted_mutual_information_1), file) utils.print_and_write_2("adjusted_mutual_info_2", np.mean(adjusted_mutual_information_2), np.std(adjusted_mutual_information_2), file) utils.print_and_write_2("homogeneity_1", np.mean(homogeneity_1), np.std(homogeneity_1), file) utils.print_and_write_2("homogeneity_2", np.mean(homogeneity_2), np.std(homogeneity_2), file) utils.print_and_write_2("silhoutte_level_1_global", np.mean(silhoutte_level_1_global), np.std(silhoutte_level_1_global), file) utils.print_and_write_2("silhoutte_level_2_global", np.mean(silhoutte_level_2_global), np.std(silhoutte_level_2_global), file) utils.print_and_write_2("silhoutte_level_1_weighted", np.mean(silhoutte_level_1_weighted), np.std(silhoutte_level_1_weighted), file) utils.print_and_write_2("silhoutte_level_2_weighted", np.mean(silhoutte_level_2_weighted), np.std(silhoutte_level_2_weighted), file) utils.print_and_write_2("dunn1_level1", np.mean(dunn1_level_1), np.std(dunn1_level_1), file) utils.print_and_write_2("dunn2_level1", np.mean(dunn2_level_1), np.std(dunn2_level_1), file) utils.print_and_write_2("dunn3_level1", np.mean(dunn3_level_1), np.std(dunn3_level_1), file) utils.print_and_write_2("dunn1_level2", np.mean(dunn1_level_2), np.std(dunn1_level_2), file) utils.print_and_write_2("dunn2_level2", np.mean(dunn2_level_2), np.std(dunn2_level_2), file) utils.print_and_write_2("dunn3_level2", np.mean(dunn3_level_2), np.std(dunn3_level_2), file) list_of_dtw_values = [] list_of_norm_dtw_values = [] list_of_lengths = [] data_cache = {} for demonstration in list_of_demonstrations: try: list_of_frms_demonstration = list_of_frms[demonstration] data = {} for i in range(len(list_of_frms_demonstration)): data[i] = [elem[0] for elem in list_of_frms_demonstration[i]] dtw_score, normalized_dtw_score, length, labels_manual_d, colors_manual_d, labels_automatic_d, colors_automatic_d = broken_barh.plot_broken_barh( demonstration, data, constants.PATH_TO_CLUSTERING_RESULTS + demonstration + "_" + filename + ".jpg", constants.N_COMPONENTS_TIME_ZW) list_of_dtw_values.append(dtw_score) list_of_norm_dtw_values.append(normalized_dtw_score) list_of_lengths.append(length) # Inserting manual and annotations labels into data struct before dumping as pickle file cache_entry = {} cache_entry['changepoints'] = list_of_frms_demonstration cache_entry['plot_labels_manual'] = labels_manual_d cache_entry['plot_colors_manual'] = colors_manual_d cache_entry['plot_labels_automatic'] = labels_automatic_d cache_entry['plot_colors_automatic'] = colors_automatic_d data_cache[demonstration] = cache_entry except: print demonstration continue utils.print_and_write_2("dtw_score", np.mean(list_of_dtw_values), np.std(list_of_dtw_values), file) utils.print_and_write_2("dtw_score_normalized", np.mean(list_of_norm_dtw_values), np.std(list_of_norm_dtw_values), file) utils.print_and_write(str(list_of_lengths), file) pickle.dump( data_cache, open(constants.PATH_TO_CLUSTERING_RESULTS + filename + "_.p", "wb"))
def cluster_metrics(self): labels_true, labels_pred_1, labels_pred_2 = self.prepare_labels() metric_funcs = [ adjusted_rand_score, adjusted_mutual_info_score, normalized_mutual_info_score, mutual_info_score, homogeneity_score, completeness_score ] # ------ Label-based metrics ------ self.file.write( "\n\nPred = L2 Milestones Pred= L1 Labels Metric\n\n") for metric in metric_funcs: try: score_1 = utils.nsf(metric(labels_true, labels_pred_1)) score_2 = utils.nsf(metric(labels_true, labels_pred_2)) except ValueError as e: print e print "Pruning error" sys.exit() key = repr(metric).split()[1] self.label_based_scores_1[key] = score_1 self.label_based_scores_2[key] = score_2 self.file.write("%3.3f %3.3f %s\n" % (score_1, score_2, key)) # ------ Precision & Recall ------ try: for ave in ["micro", "macro", "weighted"]: key = "precision_" + ave score_1 = utils.nsf( precision_score(labels_true, labels_pred_1, average=ave)) score_2 = utils.nsf( precision_score(labels_true, labels_pred_2, average=ave)) self.label_based_scores_1[key] = score_1 self.label_based_scores_2[key] = score_2 self.file.write("%3.3f %3.3f %s\n" % (score_1, score_2, key)) key = "recall_" + ave score_1 = utils.nsf( recall_score(labels_true, labels_pred_1, average=ave)) score_2 = utils.nsf( recall_score(labels_true, labels_pred_2, average=ave)) self.label_based_scores_1[key] = score_1 self.label_based_scores_2[key] = score_2 self.file.write("%3.3f %3.3f %s\n" % (score_1, score_2, key)) except ValueError as e: print e print "Pruning error" sys.exit() self.file.write("\nSilhouette Scores Global\n") # ------ Silhouette Scores ------ for layer in sorted(self.silhouette_scores_global): score = self.silhouette_scores_global[layer] self.file.write("%3.3f %s\n" % (round(Decimal(score), 2), layer)) self.file.write("\nSilhouette Scores Weighted\n") # ------ Silhouette Scores ------ for layer in sorted(self.silhouette_scores_weighted): score = self.silhouette_scores_weighted[layer] self.file.write("%3.3f %s\n" % (round(Decimal(score), 2), layer)) self.file.write("\nDunn Scores1\n") # ------ Dunn Scores ------ for layer in sorted(self.dunn_scores_1): score = self.dunn_scores_1[layer] self.file.write("%3.3f %s\n" % (round(Decimal(score), 2), layer)) self.file.write("\nDunn Scores2\n") # ------ Dunn Scores ------ for layer in sorted(self.dunn_scores_2): score = self.dunn_scores_2[layer] self.file.write("%3.3f %s\n" % (round(Decimal(score), 2), layer)) self.file.write("\nDunn Scores3\n") # ------ Dunn Scores ------ for layer in sorted(self.dunn_scores_3): score = self.dunn_scores_3[layer] self.file.write("%3.3f %s\n" % (round(Decimal(score), 2), layer)) # ------ Visualizing changepoints on broken barh ------ viz = {} for cp in self.list_of_cp: cp_all_data = (self.map_cp2frm[cp], self.map_cp2milestones[cp], self.map_cp2level1[cp], self.map_cp2surgemes[cp], self.map_cp2surgemetransitions[cp]) utils.dict_insert_list(self.map_cp2demonstrations[cp], cp_all_data, viz) data = [ self.label_based_scores_1, self.label_based_scores_2, self.silhouette_scores_global, self.dunn_scores_1, self.dunn_scores_2, self.dunn_scores_3, self.level2_silhoutte_global, self.level2_dunn_1, self.level2_dunn_2, self.level2_dunn_3, viz, self.silhouette_scores_weighted, self.level2_silhoutte_weighted ] pickle.dump(data, open(self.metrics_picklefile, "wb")) return data
def get_time_clusters(data, T_COMPONENTS): # k-fold validation (Leave one out) numDemos = len(data.keys()) + 1 sizeTestSet = numDemos - 1 list_of_frms = [] all_frms = [] for key in data.keys(): elem = data[key] list_of_frms.append(elem) all_frms += elem N_COMPONENTS = min(T_COMPONENTS, len(all_frms)) time_cluster = mixture.GMM(n_components=N_COMPONENTS, covariance_type="full", n_iter=50000, thresh=5e-7) X = np.array(all_frms) X = X.reshape(len(all_frms), 1) time_cluster.fit(X) Y = time_cluster.predict(X) means = time_cluster.means_ covars = time_cluster.covars_ # dpgmm = mixture.DPGMM(n_components = numDemos * 3, covariance_type='diag', n_iter = 10000, alpha = 1000, thresh= 1e-10) # dpgmm.fit(X) # Y = dpgmm.predict(X) # means = dpgmm.means_ list_of_elem = [] list_of_means = [] for i in range(len(Y)): list_of_elem.append((Y[i], X[i], means[Y[i]][0], np.sqrt(covars[Y[i]][0][0]))) list_of_elem = sorted(list_of_elem, key=lambda x: x[1][0]) dict_time_clusters = {} for elem in list_of_elem: utils.dict_insert_list(elem[0], elem[1], dict_time_clusters) list_time_clusters = [] list_time_clusters_noPrune = [] for cluster in dict_time_clusters.keys(): # get all frames in this cluster cluster_frames = dict_time_clusters[cluster] setClusterFrames = set([elem[0] for elem in cluster_frames]) # test if frames in cluster are representative of the test set rep = [] for id in range(sizeTestSet): elemSet = set(list_of_frms[id]) commonElem = elemSet.intersection(setClusterFrames) id_in_cluster = 1.0 if len(commonElem) > 0 else 0.0 rep.append(id_in_cluster) pruneCluster = True if sum(rep) / sizeTestSet < constants.PRUNING_FACTOR_T else False min_frm = min(cluster_frames) max_frm = max(cluster_frames) mean = means[cluster][0] list_of_means.append(int(mean)) std = np.sqrt(covars[cluster][0][0]) leftFrame = max(min_frm[0], mean - std) rightFrame = min(max_frm[0], mean + std) list_time_clusters_noPrune.append((leftFrame, rightFrame)) # keep for plotting is pruneFlag = 0 if not (pruneCluster): # list_time_clusters.append((min_frm[0], max_frm[0])) list_time_clusters.append((leftFrame, rightFrame)) print "Number of Clusters pruned in Time Clustering: ", len(list_time_clusters_noPrune) - len(list_time_clusters) labels_automatic, colors_automatic = setup_automatic_labels_2(list_time_clusters, "k") labels_automatic_0, colors_automatic_0 = setup_automatic_labels(list_of_frms[0], "k") return labels_automatic, colors_automatic, labels_automatic_0, colors_automatic_0, list_of_means, list_of_frms
def cluster_metrics(self): """ Computes metrics and returns as single dictionary. """ labels_true, labels_pred = self.prepare_labels() metric_funcs = [adjusted_rand_score, adjusted_mutual_info_score, normalized_mutual_info_score, mutual_info_score, homogeneity_score, completeness_score] # ------ Label-based metrics ------ utils.print_and_write("\n\nPred= L1 Labels Metric\n\n", self.log) for metric in metric_funcs: score_1 = round(Decimal(metric(labels_true, labels_pred)), 2) key = repr(metric).split()[1] self.label_based_scores_1[key] = score_1 utils.print_and_write(("%3.3f %s\n" % (score_1, key)), self.log) # ------ Precision & Recall ------ utils.print_and_write("\n Precision & Recall scores \n", self.log) for ave in ["micro", "macro", "weighted"]: key = "precision_" + ave score_1 = utils.nsf(precision_score(labels_true, labels_pred, average = ave)) self.label_based_scores_1[key] = score_1 utils.print_and_write("%3.3f %s\n" % (round(Decimal(score_1), 2), key), self.log) key = "recall_" + ave score_1 = utils.nsf(recall_score(labels_true, labels_pred, average = ave)) self.label_based_scores_1[key] = score_1 utils.print_and_write("%3.3f %s\n" % (round(Decimal(score_1), 2), key), self.log) utils.print_and_write("\nDunn Scores1\n", self.log) # ------ Dunn Scores # 1------ for layer in sorted(self.dunn_scores_1): score = self.dunn_scores_1[layer] utils.print_and_write("%3.3f %s\n" % (round(Decimal(score), 2), layer), self.log) utils.print_and_write("\nDunn Scores2\n", self.log) # ------ Dunn Scores # 2------ for layer in sorted(self.dunn_scores_2): score = self.dunn_scores_2[layer] utils.print_and_write("%3.3f %s\n" % (round(Decimal(score), 2), layer), self.log) utils.print_and_write("\nDunn Scores3\n", self.log) # ------ Dunn Scores #3 ------ for layer in sorted(self.dunn_scores_3): score = self.dunn_scores_3[layer] utils.print_and_write("%3.3f %s\n" % (round(Decimal(score), 2), layer), self.log) # ------ Visualizing changepoints on broken barh ------ viz = {} for cp in self.list_of_cp: cp_all_data = (self.map_cp2frm[cp], self.map_cp2cluster[cp], self.map_cp2surgemetransitions[cp], self.map_cp2surgemes[cp]) utils.dict_insert_list(self.map_cp2demonstrations[cp], cp_all_data, viz) data = [self.label_based_scores_1, self.silhouette_score_global, self.dunn_scores_1, self.dunn_scores_2, self.dunn_scores_3, viz, self.silhouette_score_weighted] return data
def evaluate_multi_modal(metrics, file): mutual_information_1 = [] normalized_mutual_information_1 = [] adjusted_mutual_information_1 = [] homogeneity_1 = [] precision_1_micro = [] recall_1_micro = [] precision_1_macro = [] recall_1_macro = [] precision_1_weighted = [] recall_1_weighted = [] mutual_information_2 = [] normalized_mutual_information_2 = [] adjusted_mutual_information_2 = [] homogeneity_2 = [] precision_2_micro = [] recall_2_micro = [] precision_2_macro = [] recall_2_macro = [] precision_2_weighted = [] recall_2_weighted = [] silhoutte_level_1 = [] dunn1_level_1 = [] dunn2_level_1 = [] dunn3_level_1 = [] silhoutte_level_2 = [] dunn1_level_2 = [] dunn2_level_2 = [] dunn3_level_2 = [] list_of_frms = {} for elem in metrics: precision_1_micro.append(elem[0]["precision_micro"]) precision_2_micro.append(elem[1]["precision_micro"]) precision_1_macro.append(elem[0]["precision_macro"]) precision_2_macro.append(elem[1]["precision_macro"]) precision_1_weighted.append(elem[0]["precision_weighted"]) precision_2_weighted.append(elem[1]["precision_weighted"]) recall_1_micro.append(elem[0]["recall_micro"]) recall_2_micro.append(elem[1]["recall_micro"]) recall_1_macro.append(elem[0]["recall_macro"]) recall_2_macro.append(elem[1]["recall_macro"]) recall_1_weighted.append(elem[0]["recall_weighted"]) recall_2_weighted.append(elem[1]["recall_weighted"]) mutual_information_1.append(elem[0]["mutual_info_score"]) mutual_information_2.append(elem[1]["mutual_info_score"]) normalized_mutual_information_1.append(elem[0]["normalized_mutual_info_score"]) normalized_mutual_information_2.append(elem[1]["normalized_mutual_info_score"]) adjusted_mutual_information_1.append(elem[0]["adjusted_mutual_info_score"]) adjusted_mutual_information_2.append(elem[1]["adjusted_mutual_info_score"]) homogeneity_1.append(elem[0]["homogeneity_score"]) homogeneity_2.append(elem[1]["homogeneity_score"]) silhoutte_level_1.append(elem[2]["level1"]) dunn1_level_1.append(elem[3]["level1"]) dunn2_level_1.append(elem[4]["level1"]) dunn3_level_1.append(elem[5]["level1"]) silhoutte_level_2 += elem[6] dunn1_level_2 += elem[7] dunn2_level_2 += elem[8] dunn3_level_2 += elem[9] viz = elem[10] for demonstration in viz.keys(): utils.dict_insert_list(demonstration, viz[demonstration], list_of_frms) utils.print_and_write_2("precision_1_micro", np.mean(precision_1_micro), np.std(precision_1_micro), file) utils.print_and_write_2("precision_2_micro", np.mean(precision_2_micro), np.std(precision_2_micro), file) utils.print_and_write_2("precision_1_macro", np.mean(precision_1_macro), np.std(precision_1_macro), file) utils.print_and_write_2("precision_2_macro", np.mean(precision_2_macro), np.std(precision_2_macro), file) utils.print_and_write_2("precision_1_weighted", np.mean(precision_1_weighted), np.std(precision_1_weighted), file) utils.print_and_write_2("precision_2_weighted", np.mean(precision_2_weighted), np.std(precision_2_weighted), file) utils.print_and_write_2("recall_1_micro", np.mean(recall_1_micro), np.std(recall_1_micro), file) utils.print_and_write_2("recall_2_micro", np.mean(recall_2_micro), np.std(recall_2_micro), file) utils.print_and_write_2("recall_1_macro", np.mean(recall_1_macro), np.std(recall_1_macro), file) utils.print_and_write_2("recall_2_macro", np.mean(recall_2_macro), np.std(recall_2_macro), file) utils.print_and_write_2("recall_1_weighted", np.mean(recall_1_weighted), np.std(recall_1_weighted), file) utils.print_and_write_2("recall_2_weighted", np.mean(recall_2_weighted), np.std(recall_2_weighted), file) utils.print_and_write_2("mutual_info_1", np.mean(mutual_information_1), np.std(mutual_information_1), file) utils.print_and_write_2("mutual_info_2", np.mean(mutual_information_2), np.std(mutual_information_2), file) utils.print_and_write_2("normalized_mutual_info_1", np.mean(normalized_mutual_information_1), np.std(normalized_mutual_information_1), file) utils.print_and_write_2("normalized_mutual_info_2", np.mean(normalized_mutual_information_2), np.std(normalized_mutual_information_2), file) utils.print_and_write_2("adjusted_mutual_info_1", np.mean(adjusted_mutual_information_1), np.std(adjusted_mutual_information_1), file) utils.print_and_write_2("adjusted_mutual_info_2", np.mean(adjusted_mutual_information_2), np.std(adjusted_mutual_information_2), file) utils.print_and_write_2("homogeneity_1", np.mean(homogeneity_1), np.std(homogeneity_1), file) utils.print_and_write_2("homogeneity_2", np.mean(homogeneity_2), np.std(homogeneity_2), file) utils.print_and_write_2("silhoutte_level_1", np.mean(silhoutte_level_1), np.std(silhoutte_level_1), file) utils.print_and_write_2("silhoutte_level_2", np.mean(silhoutte_level_2), np.std(silhoutte_level_2), file) utils.print_and_write_2("dunn1_level1", np.mean(dunn1_level_1), np.std(dunn1_level_1), file) utils.print_and_write_2("dunn2_level1", np.mean(dunn2_level_1), np.std(dunn2_level_1), file) utils.print_and_write_2("dunn3_level1", np.mean(dunn3_level_1), np.std(dunn3_level_1), file) utils.print_and_write_2("dunn1_level2", np.mean(dunn1_level_2), np.std(dunn1_level_2), file) utils.print_and_write_2("dunn2_level2", np.mean(dunn2_level_2), np.std(dunn2_level_2), file) utils.print_and_write_2("dunn3_level2", np.mean(dunn3_level_2), np.std(dunn3_level_2), file) return list_of_frms
def post_evaluation(metrics, file, fname, vision_mode): mutual_information_1 = [] normalized_mutual_information_1 = [] adjusted_mutual_information_1 = [] homogeneity_1 = [] precision_1_micro = [] recall_1_micro = [] precision_1_macro = [] recall_1_macro = [] precision_1_weighted = [] recall_1_weighted = [] silhouette_scores_global = [] silhouette_scores_weighted = [] dunn1_level_1 = [] dunn2_level_1 = [] dunn3_level_1 = [] list_of_frms = {} for elem in metrics: mutual_information_1.append(elem[0]["mutual_info_score"]) normalized_mutual_information_1.append(elem[0]["normalized_mutual_info_score"]) adjusted_mutual_information_1.append(elem[0]["adjusted_mutual_info_score"]) homogeneity_1.append(elem[0]["homogeneity_score"]) silhouette_scores_global.append(elem[1]) silhouette_scores_weighted.append(elem[6]) dunn1_level_1.append(elem[2]["level1"]) dunn2_level_1.append(elem[3]["level1"]) dunn3_level_1.append(elem[4]["level1"]) precision_1_micro.append(elem[0]["precision_micro"]) precision_1_macro.append(elem[0]["precision_macro"]) precision_1_weighted.append(elem[0]["precision_weighted"]) recall_1_micro.append(elem[0]["recall_micro"]) recall_1_macro.append(elem[0]["recall_macro"]) recall_1_weighted.append(elem[0]["recall_weighted"]) viz = elem[5] for demonstration in viz.keys(): utils.dict_insert_list(demonstration, viz[demonstration], list_of_frms) utils.print_and_write_2("precision_1_micro", np.mean(precision_1_micro), np.std(precision_1_micro), file) utils.print_and_write_2("precision_1_macro", np.mean(precision_1_macro), np.std(precision_1_macro), file) utils.print_and_write_2("precision_1_weighted", np.mean(precision_1_weighted), np.std(precision_1_weighted), file) utils.print_and_write_2("recall_1_micro", np.mean(recall_1_micro), np.std(recall_1_micro), file) utils.print_and_write_2("recall_1_macro", np.mean(recall_1_macro), np.std(recall_1_macro), file) utils.print_and_write_2("recall_1_weighted", np.mean(recall_1_weighted), np.std(recall_1_weighted), file) utils.print_and_write_2("mutual_info", np.mean(mutual_information_1), np.std(mutual_information_1), file) utils.print_and_write_2("normalized_mutual_info", np.mean(normalized_mutual_information_1), np.std(normalized_mutual_information_1), file) utils.print_and_write_2("adjusted_mutual_info", np.mean(adjusted_mutual_information_1), np.std(adjusted_mutual_information_1), file) utils.print_and_write_2("silhouette_scores_global", np.mean(silhouette_scores_global), np.std(silhouette_scores_global), file) utils.print_and_write_2("silhouette_scores_weighted", np.mean(silhouette_scores_weighted), np.std(silhouette_scores_weighted), file) utils.print_and_write_2("homogeneity", np.mean(homogeneity_1), np.std(homogeneity_1), file) utils.print_and_write_2("dunn1", np.mean(dunn1_level_1), np.std(dunn1_level_1), file) utils.print_and_write_2("dunn2", np.mean(dunn2_level_1), np.std(dunn2_level_1), file) utils.print_and_write_2("dunn3", np.mean(dunn3_level_1), np.std(dunn3_level_1), file) list_of_dtw_values = [] list_of_norm_dtw_values = [] list_of_lengths = [] if vision_mode: T = constants.N_COMPONENTS_TIME_Z else: T = constants.N_COMPONENTS_TIME_W pickle.dump(list_of_frms, open(constants.PATH_TO_CLUSTERING_RESULTS + fname + "_.p", "wb")) for demonstration in list_of_demonstrations: try: list_of_frms_demonstration = list_of_frms[demonstration] data = {} for i in range(len(list_of_frms_demonstration)): data[i] = [elem[0] for elem in list_of_frms_demonstration[i]] dtw_score, normalized_dtw_score, length = broken_barh.plot_broken_barh(demonstration, data, constants.PATH_TO_CLUSTERING_RESULTS + demonstration +"_" + fname + ".jpg", T) list_of_dtw_values.append(dtw_score) list_of_norm_dtw_values.append(normalized_dtw_score) list_of_lengths.append(length) except: print demonstration pass utils.print_and_write_2("dtw_score", np.mean(list_of_dtw_values), np.std(list_of_dtw_values), file) utils.print_and_write_2("dtw_score_normalized", np.mean(list_of_norm_dtw_values), np.std(list_of_norm_dtw_values), file) utils.print_and_write(str(list_of_lengths), file)
def cluster_metrics(self): labels_true, labels_pred_1, labels_pred_2 = self.prepare_labels() metric_funcs = [adjusted_rand_score, adjusted_mutual_info_score, normalized_mutual_info_score, mutual_info_score, homogeneity_score, completeness_score] # ------ Label-based metrics ------ self.file.write("\n\nPred = L2 Milestones Pred= L1 Labels Metric\n\n") for metric in metric_funcs: try: score_1 = utils.nsf(metric(labels_true, labels_pred_1)) score_2 = utils.nsf(metric(labels_true, labels_pred_2)) except ValueError as e: print e print "Pruning error" sys.exit() key = repr(metric).split()[1] self.label_based_scores_1[key] = score_1 self.label_based_scores_2[key] = score_2 self.file.write("%3.3f %3.3f %s\n" % (score_1, score_2, key)) # ------ Precision & Recall ------ try: for ave in ["micro", "macro", "weighted"]: key = "precision_" + ave score_1 = utils.nsf(precision_score(labels_true, labels_pred_1, average = ave)) score_2 = utils.nsf(precision_score(labels_true, labels_pred_2, average = ave)) self.label_based_scores_1[key] = score_1 self.label_based_scores_2[key] = score_2 self.file.write("%3.3f %3.3f %s\n" % (score_1, score_2, key)) key = "recall_" + ave score_1 = utils.nsf(recall_score(labels_true, labels_pred_1, average = ave)) score_2 = utils.nsf(recall_score(labels_true, labels_pred_2, average = ave)) self.label_based_scores_1[key] = score_1 self.label_based_scores_2[key] = score_2 self.file.write("%3.3f %3.3f %s\n" % (score_1, score_2, key)) except ValueError as e: print e print "Pruning error" sys.exit() self.file.write("\nSilhouette Scores Global\n") # ------ Silhouette Scores ------ for layer in sorted(self.silhouette_scores_global): score = self.silhouette_scores_global[layer] self.file.write("%3.3f %s\n" % (round(Decimal(score), 2), layer)) self.file.write("\nSilhouette Scores Weighted\n") # ------ Silhouette Scores ------ for layer in sorted(self.silhouette_scores_weighted): score = self.silhouette_scores_weighted[layer] self.file.write("%3.3f %s\n" % (round(Decimal(score), 2), layer)) self.file.write("\nDunn Scores1\n") # ------ Dunn Scores ------ for layer in sorted(self.dunn_scores_1): score = self.dunn_scores_1[layer] self.file.write("%3.3f %s\n" % (round(Decimal(score), 2), layer)) self.file.write("\nDunn Scores2\n") # ------ Dunn Scores ------ for layer in sorted(self.dunn_scores_2): score = self.dunn_scores_2[layer] self.file.write("%3.3f %s\n" % (round(Decimal(score), 2), layer)) self.file.write("\nDunn Scores3\n") # ------ Dunn Scores ------ for layer in sorted(self.dunn_scores_3): score = self.dunn_scores_3[layer] self.file.write("%3.3f %s\n" % (round(Decimal(score), 2), layer)) # ------ Visualizing changepoints on broken barh ------ viz = {} for cp in self.list_of_cp: cp_all_data = (self.map_cp2frm[cp], self.map_cp2milestones[cp], self.map_cp2level1[cp], self.map_cp2surgemes[cp], self.map_cp2surgemetransitions[cp]) utils.dict_insert_list(self.map_cp2demonstrations[cp], cp_all_data, viz) data = [self.label_based_scores_1, self.label_based_scores_2, self.silhouette_scores_global, self.dunn_scores_1, self.dunn_scores_2, self.dunn_scores_3, self.level2_silhoutte_global, self.level2_dunn_1, self.level2_dunn_2, self.level2_dunn_3, viz, self.silhouette_scores_weighted, self.level2_silhoutte_weighted] pickle.dump(data, open(self.metrics_picklefile, "wb")) return data
def post_evaluation_multimodal(metrics, file, filename, list_of_demonstrations, feat_fname): mutual_information_1 = [] normalized_mutual_information_1 = [] adjusted_mutual_information_1 = [] homogeneity_1 = [] precision_1_micro = [] recall_1_micro = [] precision_1_macro = [] recall_1_macro = [] precision_1_weighted = [] recall_1_weighted = [] mutual_information_2 = [] normalized_mutual_information_2 = [] adjusted_mutual_information_2 = [] homogeneity_2 = [] precision_2_micro = [] recall_2_micro = [] precision_2_macro = [] recall_2_macro = [] precision_2_weighted = [] recall_2_weighted = [] silhoutte_level_1_global = [] silhoutte_level_1_weighted = [] dunn1_level_1 = [] dunn2_level_1 = [] dunn3_level_1 = [] silhoutte_level_2_global = [] silhoutte_level_2_weighted = [] dunn1_level_2 = [] dunn2_level_2 = [] dunn3_level_2 = [] list_of_frms = {} for elem in metrics: precision_1_micro.append(elem[0]["precision_micro"]) precision_2_micro.append(elem[1]["precision_micro"]) precision_1_macro.append(elem[0]["precision_macro"]) precision_2_macro.append(elem[1]["precision_macro"]) precision_1_weighted.append(elem[0]["precision_weighted"]) precision_2_weighted.append(elem[1]["precision_weighted"]) recall_1_micro.append(elem[0]["recall_micro"]) recall_2_micro.append(elem[1]["recall_micro"]) recall_1_macro.append(elem[0]["recall_macro"]) recall_2_macro.append(elem[1]["recall_macro"]) recall_1_weighted.append(elem[0]["recall_weighted"]) recall_2_weighted.append(elem[1]["recall_weighted"]) mutual_information_1.append(elem[0]["mutual_info_score"]) mutual_information_2.append(elem[1]["mutual_info_score"]) normalized_mutual_information_1.append(elem[0]["normalized_mutual_info_score"]) normalized_mutual_information_2.append(elem[1]["normalized_mutual_info_score"]) adjusted_mutual_information_1.append(elem[0]["adjusted_mutual_info_score"]) adjusted_mutual_information_2.append(elem[1]["adjusted_mutual_info_score"]) homogeneity_1.append(elem[0]["homogeneity_score"]) homogeneity_2.append(elem[1]["homogeneity_score"]) silhoutte_level_1_global.append(elem[2]["level1"]) dunn1_level_1.append(elem[3]["level1"]) dunn2_level_1.append(elem[4]["level1"]) dunn3_level_1.append(elem[5]["level1"]) silhoutte_level_2_global += elem[6] dunn1_level_2 += elem[7] dunn2_level_2 += elem[8] dunn3_level_2 += elem[9] viz = elem[10] for demonstration in viz.keys(): utils.dict_insert_list(demonstration, viz[demonstration], list_of_frms) silhoutte_level_1_weighted.append(elem[11]["level1"]) silhoutte_level_2_weighted += elem[12] utils.print_and_write_2("precision_1_micro", np.mean(precision_1_micro), np.std(precision_1_micro), file) utils.print_and_write_2("precision_2_micro", np.mean(precision_2_micro), np.std(precision_2_micro), file) utils.print_and_write_2("precision_1_macro", np.mean(precision_1_macro), np.std(precision_1_macro), file) utils.print_and_write_2("precision_2_macro", np.mean(precision_2_macro), np.std(precision_2_macro), file) utils.print_and_write_2("precision_1_weighted", np.mean(precision_1_weighted), np.std(precision_1_weighted), file) utils.print_and_write_2("precision_2_weighted", np.mean(precision_2_weighted), np.std(precision_2_weighted), file) utils.print_and_write_2("recall_1_micro", np.mean(recall_1_micro), np.std(recall_1_micro), file) utils.print_and_write_2("recall_2_micro", np.mean(recall_2_micro), np.std(recall_2_micro), file) utils.print_and_write_2("recall_1_macro", np.mean(recall_1_macro), np.std(recall_1_macro), file) utils.print_and_write_2("recall_2_macro", np.mean(recall_2_macro), np.std(recall_2_macro), file) utils.print_and_write_2("recall_1_weighted", np.mean(recall_1_weighted), np.std(recall_1_weighted), file) utils.print_and_write_2("recall_2_weighted", np.mean(recall_2_weighted), np.std(recall_2_weighted), file) utils.print_and_write_2("mutual_info_1", np.mean(mutual_information_1), np.std(mutual_information_1), file) utils.print_and_write_2("mutual_info_2", np.mean(mutual_information_2), np.std(mutual_information_2), file) utils.print_and_write_2("normalized_mutual_info_1", np.mean(normalized_mutual_information_1), np.std(normalized_mutual_information_1), file) utils.print_and_write_2("normalized_mutual_info_2", np.mean(normalized_mutual_information_2), np.std(normalized_mutual_information_2), file) utils.print_and_write_2("adjusted_mutual_info_1", np.mean(adjusted_mutual_information_1), np.std(adjusted_mutual_information_1), file) utils.print_and_write_2("adjusted_mutual_info_2", np.mean(adjusted_mutual_information_2), np.std(adjusted_mutual_information_2), file) utils.print_and_write_2("homogeneity_1", np.mean(homogeneity_1), np.std(homogeneity_1), file) utils.print_and_write_2("homogeneity_2", np.mean(homogeneity_2), np.std(homogeneity_2), file) utils.print_and_write_2("silhoutte_level_1_global", np.mean(silhoutte_level_1_global), np.std(silhoutte_level_1_global), file) utils.print_and_write_2("silhoutte_level_2_global", np.mean(silhoutte_level_2_global), np.std(silhoutte_level_2_global), file) utils.print_and_write_2("silhoutte_level_1_weighted", np.mean(silhoutte_level_1_weighted), np.std(silhoutte_level_1_weighted), file) utils.print_and_write_2("silhoutte_level_2_weighted", np.mean(silhoutte_level_2_weighted), np.std(silhoutte_level_2_weighted), file) utils.print_and_write_2("dunn1_level1", np.mean(dunn1_level_1), np.std(dunn1_level_1), file) utils.print_and_write_2("dunn2_level1", np.mean(dunn2_level_1), np.std(dunn2_level_1), file) utils.print_and_write_2("dunn3_level1", np.mean(dunn3_level_1), np.std(dunn3_level_1), file) utils.print_and_write_2("dunn1_level2", np.mean(dunn1_level_2), np.std(dunn1_level_2), file) utils.print_and_write_2("dunn2_level2", np.mean(dunn2_level_2), np.std(dunn2_level_2), file) utils.print_and_write_2("dunn3_level2", np.mean(dunn3_level_2), np.std(dunn3_level_2), file) list_of_dtw_values = [] list_of_norm_dtw_values = [] list_of_lengths = [] data_cache = {} for demonstration in list_of_demonstrations: try: list_of_frms_demonstration = list_of_frms[demonstration] data = {} for i in range(len(list_of_frms_demonstration)): data[i] = [elem[0] for elem in list_of_frms_demonstration[i]] dtw_score, normalized_dtw_score, length, labels_manual_d, colors_manual_d, labels_automatic_d, colors_automatic_d = broken_barh.plot_broken_barh(demonstration, data, constants.PATH_TO_CLUSTERING_RESULTS + demonstration +"_" + filename + ".jpg", constants.N_COMPONENTS_TIME_ZW) list_of_dtw_values.append(dtw_score) list_of_norm_dtw_values.append(normalized_dtw_score) list_of_lengths.append(length) # Inserting manual and annotations labels into data struct before dumping as pickle file cache_entry = {} cache_entry['changepoints'] = list_of_frms_demonstration cache_entry['plot_labels_manual'] = labels_manual_d cache_entry['plot_colors_manual'] = colors_manual_d cache_entry['plot_labels_automatic'] = labels_automatic_d cache_entry['plot_colors_automatic'] = colors_automatic_d data_cache[demonstration] = cache_entry except: print demonstration continue utils.print_and_write_2("dtw_score", np.mean(list_of_dtw_values), np.std(list_of_dtw_values), file) utils.print_and_write_2("dtw_score_normalized", np.mean(list_of_norm_dtw_values), np.std(list_of_norm_dtw_values), file) utils.print_and_write(str(list_of_lengths), file) pickle.dump(data_cache, open(constants.PATH_TO_CLUSTERING_RESULTS + filename + "_.p", "wb"))
def evaluate_multi_modal(metrics, file): mutual_information_1 = [] normalized_mutual_information_1 = [] adjusted_mutual_information_1 = [] homogeneity_1 = [] precision_1_micro = [] recall_1_micro = [] precision_1_macro = [] recall_1_macro = [] precision_1_weighted = [] recall_1_weighted = [] mutual_information_2 = [] normalized_mutual_information_2 = [] adjusted_mutual_information_2 = [] homogeneity_2 = [] precision_2_micro = [] recall_2_micro = [] precision_2_macro = [] recall_2_macro = [] precision_2_weighted = [] recall_2_weighted = [] silhoutte_level_1 = [] dunn1_level_1 = [] dunn2_level_1 = [] dunn3_level_1 = [] silhoutte_level_2 = [] dunn1_level_2 = [] dunn2_level_2 = [] dunn3_level_2 = [] list_of_frms = {} for elem in metrics: precision_1_micro.append(elem[0]["precision_micro"]) precision_2_micro.append(elem[1]["precision_micro"]) precision_1_macro.append(elem[0]["precision_macro"]) precision_2_macro.append(elem[1]["precision_macro"]) precision_1_weighted.append(elem[0]["precision_weighted"]) precision_2_weighted.append(elem[1]["precision_weighted"]) recall_1_micro.append(elem[0]["recall_micro"]) recall_2_micro.append(elem[1]["recall_micro"]) recall_1_macro.append(elem[0]["recall_macro"]) recall_2_macro.append(elem[1]["recall_macro"]) recall_1_weighted.append(elem[0]["recall_weighted"]) recall_2_weighted.append(elem[1]["recall_weighted"]) mutual_information_1.append(elem[0]["mutual_info_score"]) mutual_information_2.append(elem[1]["mutual_info_score"]) normalized_mutual_information_1.append( elem[0]["normalized_mutual_info_score"]) normalized_mutual_information_2.append( elem[1]["normalized_mutual_info_score"]) adjusted_mutual_information_1.append( elem[0]["adjusted_mutual_info_score"]) adjusted_mutual_information_2.append( elem[1]["adjusted_mutual_info_score"]) homogeneity_1.append(elem[0]["homogeneity_score"]) homogeneity_2.append(elem[1]["homogeneity_score"]) silhoutte_level_1.append(elem[2]["level1"]) dunn1_level_1.append(elem[3]["level1"]) dunn2_level_1.append(elem[4]["level1"]) dunn3_level_1.append(elem[5]["level1"]) silhoutte_level_2 += elem[6] dunn1_level_2 += elem[7] dunn2_level_2 += elem[8] dunn3_level_2 += elem[9] viz = elem[10] for demonstration in viz.keys(): utils.dict_insert_list(demonstration, viz[demonstration], list_of_frms) utils.print_and_write_2("precision_1_micro", np.mean(precision_1_micro), np.std(precision_1_micro), file) utils.print_and_write_2("precision_2_micro", np.mean(precision_2_micro), np.std(precision_2_micro), file) utils.print_and_write_2("precision_1_macro", np.mean(precision_1_macro), np.std(precision_1_macro), file) utils.print_and_write_2("precision_2_macro", np.mean(precision_2_macro), np.std(precision_2_macro), file) utils.print_and_write_2("precision_1_weighted", np.mean(precision_1_weighted), np.std(precision_1_weighted), file) utils.print_and_write_2("precision_2_weighted", np.mean(precision_2_weighted), np.std(precision_2_weighted), file) utils.print_and_write_2("recall_1_micro", np.mean(recall_1_micro), np.std(recall_1_micro), file) utils.print_and_write_2("recall_2_micro", np.mean(recall_2_micro), np.std(recall_2_micro), file) utils.print_and_write_2("recall_1_macro", np.mean(recall_1_macro), np.std(recall_1_macro), file) utils.print_and_write_2("recall_2_macro", np.mean(recall_2_macro), np.std(recall_2_macro), file) utils.print_and_write_2("recall_1_weighted", np.mean(recall_1_weighted), np.std(recall_1_weighted), file) utils.print_and_write_2("recall_2_weighted", np.mean(recall_2_weighted), np.std(recall_2_weighted), file) utils.print_and_write_2("mutual_info_1", np.mean(mutual_information_1), np.std(mutual_information_1), file) utils.print_and_write_2("mutual_info_2", np.mean(mutual_information_2), np.std(mutual_information_2), file) utils.print_and_write_2("normalized_mutual_info_1", np.mean(normalized_mutual_information_1), np.std(normalized_mutual_information_1), file) utils.print_and_write_2("normalized_mutual_info_2", np.mean(normalized_mutual_information_2), np.std(normalized_mutual_information_2), file) utils.print_and_write_2("adjusted_mutual_info_1", np.mean(adjusted_mutual_information_1), np.std(adjusted_mutual_information_1), file) utils.print_and_write_2("adjusted_mutual_info_2", np.mean(adjusted_mutual_information_2), np.std(adjusted_mutual_information_2), file) utils.print_and_write_2("homogeneity_1", np.mean(homogeneity_1), np.std(homogeneity_1), file) utils.print_and_write_2("homogeneity_2", np.mean(homogeneity_2), np.std(homogeneity_2), file) utils.print_and_write_2("silhoutte_level_1", np.mean(silhoutte_level_1), np.std(silhoutte_level_1), file) utils.print_and_write_2("silhoutte_level_2", np.mean(silhoutte_level_2), np.std(silhoutte_level_2), file) utils.print_and_write_2("dunn1_level1", np.mean(dunn1_level_1), np.std(dunn1_level_1), file) utils.print_and_write_2("dunn2_level1", np.mean(dunn2_level_1), np.std(dunn2_level_1), file) utils.print_and_write_2("dunn3_level1", np.mean(dunn3_level_1), np.std(dunn3_level_1), file) utils.print_and_write_2("dunn1_level2", np.mean(dunn1_level_2), np.std(dunn1_level_2), file) utils.print_and_write_2("dunn2_level2", np.mean(dunn2_level_2), np.std(dunn2_level_2), file) utils.print_and_write_2("dunn3_level2", np.mean(dunn3_level_2), np.std(dunn3_level_2), file) return list_of_frms