def expectation_maximization(X,y,dataset_name): X_train, X_test, y_train, y_test = train_test_split(X,y, random_state=65) train_scores = [] train_homo = [] train_completeness = [] train_v_score = [] test_scores = [] test_homo = [] test_completeness = [] test_v_score = [] kvals = [x for x in range(2,51)] for k in range(2, 51): print("k= {}".format(k)) clf = GaussianMixture(n_components=k, max_iter=1000) # Train on train data, recording accuracy, homogeneity, completeness, and v_measure train_pred = clf.fit_predict(X_train) train_score = fowlkes_mallows_score(y_train, train_pred) train_scores.append(train_score) homogeneity, completeness, v_measure = homogeneity_completeness_v_measure(y_train, train_pred) train_homo.append(homogeneity) train_completeness.append(completeness) train_v_score.append(v_measure) # Evaluate same metrics on test set test_pred = clf.predict(X_test) test_score = fowlkes_mallows_score(y_test, test_pred) test_scores.append(test_score) homogeneity, completeness, v_measure = homogeneity_completeness_v_measure(y_test, test_pred) test_homo.append(homogeneity) test_completeness.append(completeness) test_v_score.append(v_measure) print("done") print("generating plots") plt.figure() plt.title('Folkes-Mallows Score of Expectation Maximization on {} Dataset'.format(dataset_name)) plt.xlabel('Number of Components') plt.ylabel('Folkes-Mallows Score') plt.plot(kvals, train_scores, label='Training Score') plt.plot(kvals, test_scores, label='Test Score') plt.legend(loc='upper left') plt.show(block=False) plt.figure() plt.title('Performance Metrics of Expectation Maximization on {} Dataset'.format(dataset_name)) plt.xlabel('K Value (Number of Clusters)') plt.ylabel('Score (Range 0.0 to 1.0)') plt.plot(kvals, train_homo, label='Training Homogeneity') plt.plot(kvals, test_homo, label='Test Homogeneity') plt.plot(kvals, train_completeness, label='Training Completeness') plt.plot(kvals, test_completeness, label='Test Completeness') plt.plot(kvals, train_v_score, label='Training V-Measure') plt.plot(kvals, test_v_score, label='Test V-Measure') plt.legend(loc='upper left') plt.show(block=False)
def comparison_table(df, clusterers, scaler, ae_dims=[]): global AE table = [] if is_pretty_table: table = PrettyTable(['Method', 'Mean silhouette score', 'V Score', 'Fowlkes Score']) lanes_df = df['lane'] # indices of elements who are from MID, TOP or JUNGLE known_label_indices = [i for i in range(len(lanes_df)) if lanes_df[i] in ['MIDDLE', 'TOP', 'JUNGLE']] true_labels = lanes_df[known_label_indices] data_no_labels = df.drop(['lane', 'championId'], axis=1) data = scaleColumns(data_no_labels, scaler=scaler) scaled_data = data.copy() if not ae_dims: for clusterer_name in clusterers: clusterer = clusterers[clusterer_name] labels = clusterer.predict(data) sil_score = silhouette_score(data, labels) # labels for elements from MID TOP or JUNGLE known_lane_labels = labels[known_label_indices] v_score = v_measure_score(true_labels, known_lane_labels) fowlkes_score = fowlkes_mallows_score(true_labels, known_lane_labels) if is_pretty_table: table.add_row([f'{clusterer_name}', "{:.3f}".format(sil_score), "{:.3f}".format(v_score), "{:.3f}".format(fowlkes_score)]) else: print(f'{clusterer_name}: {sil_score}, {v_score}, {fowlkes_score}') else: for ae_dim in ae_dims: if AE.is_fit is True and AE.mid_size == ae_dim: pass else: AE = AutoEncoder(54, ae_dim) data = AE.fit_transform(data) for clusterer_name in clusterers: clusterer = clusterers[clusterer_name] labels = clusterer.predict(data) sil_score = silhouette_score(scaled_data, labels) # labels for elements from MID TOP or JUNGLE known_lane_labels = labels[known_label_indices] v_score = v_measure_score(true_labels, known_lane_labels) fowlkes_score = fowlkes_mallows_score(true_labels, known_lane_labels) if is_pretty_table: table.add_row([f'AE {ae_dim}-{clusterer_name}', "{:.3f}".format(sil_score), "{:.3f}".format(v_score), "{:.3f}".format(fowlkes_score)]) else: print(f'AE {ae_dim}-{clusterer_name}: {sil_score:.3f}, {v_score:.3f}, {fowlkes_score:.3f}') if is_pretty_table: print(table)
def compare(self, model, X, y=None): """ Compares the score of a sample in two models. Returns a crossvalidation of metrics, predictions and score. More info: https://machinelearningmastery.com/compare-machine-learning-algorithms-python-scikit-learn/ :param model: model :param X: data :type model: LinearModel :type X: ndarray or scipy.sparse matrix, (n_samples, n_features) """ metrict_dict = {} #Adjusted Rand Index, Mutual Information labels_true = y labels_pred = self.model_.predict(X) labels_pred_other = model.predict(X) labels_other = model.get_model().labels_ labels = model_.labels_ if y != None: #These metrics require the knowledge of the ground truth classes metrict_dict['ARI'] = (metrics.adjusted_rand_score( labels_true, labels_pred), metrics.adjusted_rand_score( labels_true, labels_pred_other)) metrict_dict['MIS'] = (metrics.adjusted_mutual_info_score( labels_true, labels_pred), metrics.adjusted_mutual_info_score( labels_true, labels_pred_other)) metrict_dict['homogeneity'] = (metrics.homogeneity_score( labels_true, labels_pred), metrics.homogeneity_score( labels_true, labels_pred_other)) metrict_dict['completeness'] = (metrics.completeness_score( labels_true, labels_pred), metrics.completeness_score( labels_true, labels_pred_other)) metrict_dict['v_measure'] = (metrics.v_measure_score( labels_true, labels_pred), metrics.v_measure_score( labels_true, labels_pred_other)) metrict_dict['FMS'] = (metrics.fowlkes_mallows_score( labels_true, labels_pred), metrics.fowlkes_mallows_score( labels_true, labels_pred_other)) #These metrics DON'T require the knowledge of the ground truth classes metrict_dict['SS'] = (metrics.silhouette_score(X, labels, metric='euclidean'), metrics.silhouette_score(X, labels_other, metric='euclidean')) metrict_dict['CHI'] = (metrics.calinski_harabaz_score(X, labels), metrics.calinski_harabaz_score(X, labels_other)) return metrict_dict
def external_validation(pred_, moa_, treat_, comp_, random_, same_): metrices_dict = dict() # Completeness metrices_dict['comple_moa-pred'] = [completeness_score(moa_, pred_)] metrices_dict['comple_treat-moa'] = [completeness_score(treat_, moa_)] metrices_dict['comple_treat-pred'] = [completeness_score(treat_, pred_)] metrices_dict['comple_treat-rand'] = [completeness_score(treat_, random_)] metrices_dict['comple_treat-same'] = [completeness_score(treat_, same_)] metrices_dict['comple_comp-moa'] = [completeness_score(comp_, moa_)] metrices_dict['comple_comp_pred'] = [completeness_score(comp_, pred_)] metrices_dict['comple_comp_rand'] = [completeness_score(comp_, random_)] metrices_dict['comple_comp_same'] = [completeness_score(comp_, same_)] # Jaccard similarity coefficient metrices_dict['jaccard_moa-pred'] = [jaccard(moa_, pred_)] metrices_dict['jaccard_treat-moa'] = [jaccard(treat_, moa_)] metrices_dict['jaccard_treat-pred'] = [jaccard(treat_, pred_)] metrices_dict['jaccard_treat-rand'] = [jaccard(treat_, random_)] metrices_dict['jaccard_treat-same'] = [jaccard(treat_, same_)] metrices_dict['jaccard_comp-moa'] = [jaccard(comp_, moa_)] metrices_dict['jaccard_comp_pred'] = [jaccard(comp_, pred_)] metrices_dict['jaccard_comp_rand'] = [jaccard(comp_, random_)] metrices_dict['jaccard_comp_same'] = [jaccard(comp_, same_)] # Adjusted Rand index metrices_dict['adj-rand_moa-pred'] = [adjusted_rand_score(moa_, pred_)] metrices_dict['adj-rand_treat-moa'] = [adjusted_rand_score(treat_, moa_)] metrices_dict['adj-rand_treat-pred'] = [adjusted_rand_score(treat_, pred_)] metrices_dict['adj-rand_treat-rand'] = [adjusted_rand_score(treat_, random_)] metrices_dict['adj-rand_treat-same'] = [adjusted_rand_score(treat_, same_)] metrices_dict['adj-rand_comp-moa'] = [adjusted_rand_score(comp_, moa_)] metrices_dict['adj-rand_comp_pred'] = [adjusted_rand_score(comp_, pred_)] metrices_dict['adj-rand_comp_rand'] = [adjusted_rand_score(comp_, random_)] metrices_dict['adj-rand_comp_same'] = [adjusted_rand_score(comp_, same_)] # Fowlkes-Mallows index metrices_dict['fow-mal_moa-pred'] = [fowlkes_mallows_score(moa_, pred_)] metrices_dict['fow-mal_treat-moa'] = [fowlkes_mallows_score(treat_, moa_)] metrices_dict['fow-mal_treat-pred'] = [fowlkes_mallows_score(treat_, pred_)] metrices_dict['fow-mal_treat-rand'] = [fowlkes_mallows_score(treat_, random_)] metrices_dict['fow-mal_treat-same'] = [fowlkes_mallows_score(treat_, same_)] metrices_dict['fow-mal_comp-moa'] = [fowlkes_mallows_score(comp_, moa_)] metrices_dict['fow-mal_comp_pred'] = [fowlkes_mallows_score(comp_, pred_)] metrices_dict['fow-mal_comp_rand'] = [fowlkes_mallows_score(comp_, random_)] metrices_dict['fow-mal_comp_same'] = [fowlkes_mallows_score(comp_, same_)] # Adjusted mutual information metrices_dict['adj-mut_moa-pred'] = [adjusted_mutual_info_score(moa_, pred_)] metrices_dict['adj-mut_treat-moa'] = [adjusted_mutual_info_score(treat_, moa_)] metrices_dict['adj-mut_treat-pred'] = [adjusted_mutual_info_score(treat_, pred_)] metrices_dict['adj-mut_treat-rand'] = [adjusted_mutual_info_score(treat_, random_)] metrices_dict['adj-mut_treat-same'] = [adjusted_mutual_info_score(treat_, same_)] metrices_dict['adj-mut_comp-moa'] = [adjusted_mutual_info_score(comp_, moa_)] metrices_dict['adj-mut_comp_pred'] = [adjusted_mutual_info_score(comp_, pred_)] metrices_dict['adj-mut_comp_rand'] = [adjusted_mutual_info_score(comp_, random_)] metrices_dict['adj-mut_comp_same'] = [adjusted_mutual_info_score(comp_, same_)] return pd.DataFrame(metrices_dict)
def evaluate(self, y_gt,y_est): y_gt = self.type_format(y_gt) y_est = self.type_format(y_est) scores = {} if '#Err' in self.criteria: scores['#Err'] = self.ErrNumClusters(y_gt,y_est) if 'AMI' in self.criteria: scores['AMI'] = sm.adjusted_mutual_info_score(y_gt,y_est) if 'ARI' in self.criteria: scores['ARI'] = sm.adjusted_rand_score(y_gt,y_est) if 'COMP' in self.criteria: scores['COMP'] = sm.completeness_score(y_gt,y_est) if 'FMS' in self.criteria: scores['FMS'] = sm.fowlkes_mallows_score(y_gt,y_est) #if 'HCVM' in self.criteria: # scores['HCVM'] = sm.homogeneity_completeness_v_measure(y_gt,y_est) if 'HS' in self.criteria: scores['HS'] = sm.homogeneity_score(y_gt,y_est) if 'MI' in self.criteria: scores['MI'] = sm.mutual_info_score(y_gt,y_est) if 'NMI' in self.criteria: scores['NMI'] = sm.normalized_mutual_info_score(y_gt,y_est) if 'VMS' in self.criteria: scores['VMS'] = sm.v_measure_score(y_gt,y_est) if 'PURITY' in self.criteria: scores['PURITY'] = self.purity(y_gt,y_est) # for inlier/outlier classification if 'F1' in self.criteria: scores['F1'] = self.F1score(y_gt,y_est) return self.conv_return_type(scores)
def evaluateClustering(X_true, y_true, y_pred, model, classes): print('\n', model.upper()) ars = adjusted_rand_score(y_true, y_pred) print('Adjusted random score: ', ars) ss = silhouette_score(X_true, y_pred) print('Silhouette score: ', ss) hs = homogeneity_score(y_true, y_pred) print('Homogeneity score: ', hs) cs = completeness_score(y_true, y_pred) print('Completeness score: ', cs) vms = v_measure_score(y_true, y_pred) print('V-measure score: ', vms) fms = fowlkes_mallows_score(y_true, y_pred) print('Fowlkes-Mallows score: ', fms) chs = calinski_harabaz_score(X_true, y_pred) print('Calinski-Harabaz score: ', chs) cm = contingency_matrix(y_true, y_pred) orderContingencyMatrix(cm) acc = accuracy(cm) print('Supervised-like Accuracy: ', acc) mean = meanScore(X_true, y_true, y_pred) print('Mean Score: ', mean) plotContingencyMatrix(cm, model, classes)
def compute_results(G_data, B_data, name, encoder,r_state=0,only_kmeans=False): B_data_X = encoder.detach().numpy() kmeans = KMeans(n_clusters=get_clusters(G_data,name),init='k-means++',random_state=r_state) if not only_kmeans: kmeans.fit(B_data_X) else: kmeans.fit(B_data) X_ae = kmeans.labels_ # Calculated labels # Finding truth values if name =='karate': c_groups=[0,0,0,0,0,0,0,0,1,1,0,0,0,0,1,1,0,0,1,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1] elif name == 'email': c_groups=[1, 1, 21, 21, 21, 25, 25, 14, 14, 14, 9, 14, 14, 26, 4, 17, 34, 1, 1, 14, 9, 9, 9, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 5, 34, 14, 14, 17, 17, 10, 10, 36, 37, 5, 7, 4, 22, 22, 21, 21, 21, 21, 7, 7, 36, 21, 25, 4, 8, 15, 15, 15, 37, 37, 9, 1, 1, 10, 10, 3, 3, 3, 29, 15, 36, 36, 37, 1, 36, 34, 20, 20, 8, 15, 9, 4, 5, 4, 20, 16, 16, 16, 16, 16, 38, 7, 7, 34, 38, 36, 8, 27, 8, 8, 8, 10, 10, 13, 13, 6, 26, 10, 1, 36, 0, 13, 16, 16, 22, 6, 5, 4, 0, 28, 28, 4, 2, 13, 13, 21, 21, 17, 17, 14, 36, 8, 40, 35, 15, 23, 0, 0, 7, 10, 37, 27, 35, 35, 0, 0, 19, 19, 36, 14, 37, 24, 17, 13, 36, 4, 4, 13, 13, 10, 4, 38, 32, 32, 4, 1, 0, 0, 0, 7, 7, 4, 15, 16, 40, 15, 15, 15, 15, 0, 21, 21, 21, 21, 5, 4, 4, 4, 4, 4, 4, 4, 5, 5, 4, 4, 22, 19, 19, 22, 34, 14, 0, 1, 17, 37, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 10, 23, 0, 4, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 10, 14, 14, 1, 14, 7, 13, 20, 31, 40, 6, 4, 0, 8, 9, 9, 10, 0, 10, 14, 14, 14, 14, 39, 17, 4, 28, 17, 17, 17, 4, 4, 0, 0, 23, 4, 21, 36, 36, 0, 22, 21, 15, 37, 0, 4, 4, 4, 14, 4, 7, 7, 1, 15, 15, 38, 26, 20, 20, 20, 21, 9, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 10, 19, 7, 7, 17, 16, 14, 9, 9, 9, 8, 8, 13, 39, 14, 10, 17, 17, 13, 13, 13, 13, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 16, 27, 8, 8, 14, 14, 14, 10, 14, 35, 37, 14, 36, 10, 7, 20, 10, 16, 36, 36, 14, 8, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 4, 9, 4, 0, 4, 16, 38, 14, 14, 21, 26, 27, 28, 21, 4, 1, 1, 9, 10, 15, 4, 26, 14, 35, 10, 34, 4, 4, 12, 17, 17, 14, 37, 37, 37, 34, 6, 13, 13, 13, 13, 4, 14, 10, 10, 10, 3, 17, 17, 17, 1, 4, 14, 14, 6, 27, 22, 21, 4, 4, 1, 34, 17, 30, 30, 4, 23, 14, 15, 1, 22, 12, 31, 6, 15, 15, 8, 15, 8, 8, 1, 15, 22, 2, 3, 4, 10, 4, 14, 14, 25, 6, 6, 40, 4, 36, 23, 14, 3, 14, 14, 14, 14, 14, 14, 14, 14, 14, 31, 15, 15, 14, 0, 23, 35, 8, 4, 1, 1, 35, 23, 21, 2, 4, 4, 9, 14, 4, 10, 25, 14, 14, 3, 21, 35, 4, 9, 15, 6, 9, 3, 15, 23, 4, 4, 4, 11, 35, 10, 6, 15, 15, 15, 22, 2, 2, 14, 4, 3, 14, 27, 31, 34, 4, 4, 19, 14, 14, 4, 4, 14, 14, 21, 4, 14, 4, 0, 4, 27, 27, 17, 3, 15, 2, 4, 4, 21, 21, 11, 23, 11, 23, 17, 5, 36, 15, 23, 23, 2, 19, 4, 36, 14, 1, 22, 1, 21, 34, 14, 13, 6, 4, 37, 6, 24, 35, 6, 17, 16, 6, 4, 0, 21, 4, 26, 21, 4, 15, 7, 1, 20, 19, 7, 21, 21, 21, 19, 38, 19, 16, 23, 6, 37, 25, 1, 22, 6, 14, 1, 26, 8, 37, 4, 0, 17, 6, 17, 14, 16, 4, 32, 14, 15, 0, 23, 21, 29, 14, 14, 1, 17, 26, 15, 0, 0, 0, 22, 34, 21, 6, 16, 4, 15, 21, 0, 36, 4, 1, 1, 22, 14, 14, 30, 4, 9, 10, 4, 4, 14, 16, 16, 15, 21, 0, 4, 15, 29, 24, 21, 14, 11, 11, 9, 13, 10, 31, 4, 22, 14, 23, 1, 4, 9, 17, 27, 28, 22, 14, 20, 7, 23, 1, 6, 15, 15, 23, 4, 20, 5, 36, 10, 21, 39, 41, 31, 17, 7, 21, 34, 1, 14, 2, 18, 16, 27, 16, 38, 7, 38, 21, 1, 9, 15, 15, 15, 0, 6, 23, 28, 11, 23, 34, 24, 4, 4, 4, 24, 23, 17, 10, 17, 1, 1, 15, 15, 4, 21, 14, 14, 20, 28, 20, 22, 26, 3, 32, 4, 0, 21, 13, 4, 15, 17, 5, 4, 14, 0, 9, 21, 14, 38, 4, 14, 31, 21, 14, 6, 4, 4, 6, 17, 0, 4, 7, 16, 4, 4, 21, 1, 10, 3, 21, 4, 0, 1, 7, 17, 15, 14, 0, 9, 32, 13, 5, 2, 21, 28, 21, 22, 22, 7, 7, 33, 0, 1, 15, 4, 31, 30, 15, 11, 19, 21, 9, 21, 13, 21, 9, 32, 9, 32, 38, 9, 38, 38, 14, 9, 10, 38, 10, 22, 21, 13, 21, 4, 0, 1, 1, 23, 0, 5, 4, 4, 15, 14, 14, 13, 11, 1, 5, 5, 10, 23, 21, 14, 9, 20, 10, 19, 19, 21, 17, 19, 19, 36, 17, 35, 16, 4, 16, 4, 6, 4, 41, 6, 7, 23, 9, 23, 7, 6, 22, 36, 14, 15, 11, 35, 5, 14, 14, 15, 4, 6, 4, 9, 19, 11, 4, 29, 14, 15, 15, 5, 32, 15, 14, 5, 9, 10, 19, 13, 23, 12, 10, 21, 10, 35, 7, 22, 22, 22, 8, 21, 32, 4, 21, 21, 6, 14, 11, 14, 15, 4, 21, 1, 6, 22] else: c_attributes = nx.get_node_attributes(G_data, 'value') c_groups = [] for i, val in enumerate(c_attributes.values()): c_groups.append(val) X_gt = np.array(c_groups) # print(X_ae) # print(X_gt) return metrics.fowlkes_mallows_score(X_gt, X_ae)
def KmeansCluster(TrainMalSet, labelcsv, FeatureOption): Logger.debug("Loading Malware and Goodware Sample Data for training and testing") TrainMalSamples = getApkList(TrainMalSet, ".data") Logger.info("Loaded Samples") FeatureVectorizer = TF(input="filename", tokenizer=lambda x: x.split('\n'), token_pattern=None, binary=FeatureOption) X = FeatureVectorizer.fit_transform(TrainMalSamples) get_family_dict(labelcsv) y_train = [] for file in TrainMalSamples: if "amd" in labelcsv: sha256 = os.path.split(file)[-1][:-5].split('_')[-1] else: sha256 = os.path.split(file)[-1][:-5] if sha256 in family_dict: y_train.append(family_dict[sha256]) else: y_train.append(-1) # test print(y_train[:20]) kmeans = KMeans(n_clusters=family_count, random_state=10).fit(X) labels = kmeans.labels_ score = fowlkes_mallows_score(y_train, labels) print(labels[:20]) print(family_count, score)
def test_compare_example_with_sklearn(self): # Act metrics = similarity_metrics(self.large_A, self.large_B) ar_similarity = metrics.adjusted_rand() fm_similarity = metrics.fowlkes_mallows() ar_sklearn = [] fm_sklearn = [] for i in range(9, 1, -1): fcluster_a = fcluster(self.large_A, i, 'maxclust') fcluster_b = fcluster(self.large_B, i, 'maxclust') ar = adjusted_rand_score(fcluster_a, fcluster_b) fm = fowlkes_mallows_score(fcluster_a, fcluster_b) ar_sklearn.append(ar) fm_sklearn.append(fm) # Assert assert_almost_equal(ar_similarity, ar_sklearn) assert_almost_equal(fm_similarity, fm_sklearn)
def eval_2(labels_true, labels_pred, is_show=True): """ 有监督的评估 评价指标 越接近 1 越好 :param labels_true: :param labels_pred: :param is_show: 是否显示结果 :return: """ if labels_true == []: info = f"cluster: img_sum:{len(labels_pred)}, id_sum:{len(set(labels_pred))}" return [], info nmi = 0 # metrics.normalized_mutual_info_score(labels_true, labels_pred) # 归一化互信息 ari = metrics.adjusted_rand_score(labels_true, labels_pred) # 调整兰德指数 # 纯度,散度, v_measure homogeneity, completeness, v_measure_score = metrics.homogeneity_completeness_v_measure( labels_true, labels_pred) fmi = metrics.fowlkes_mallows_score(labels_true, labels_pred) # 几何平均数 avg_pre, avg_rec, fscore = fowlkes_mallows_score( labels_true, labels_pred) # 调和平均数 ***** k = 0.5 fscore_2 = 2. * avg_pre * k * avg_rec / (avg_pre * k + avg_rec) s_1 = f"gt: img_sum:{len(labels_true)}, id_sum:{len(set(labels_true))}" s_2 = f"cluster: img_sum:{len(labels_pred)}, id_sum:{len(set(labels_pred))}" s_3 = "有监督: 纯度, 散度, nmi, v_measure, ari:" + f"{r(homogeneity)}, {r(completeness)}, {r(nmi)}, {r(v_measure_score)}, {r(ari)}" s_4 = 'avg_pre, avg_rec, fscore, fmi:' + f"{r(avg_pre)}, {r(avg_rec)}, {r(fscore)}, {r(fmi)}" info = f"{s_1}\n{s_2}\n{s_3}\n{s_4}" if is_show: print(info) metric = [avg_pre, avg_rec, fscore, fmi] return metric, info
def _compute_coref_metrics_threshold(gold_clustering, mst, threshold): # get the true_labels true_labels = [(i, x) for i, l in enumerate(gold_clustering) for x in l] true_labels = sorted(true_labels, key=lambda x: x[1]) true_labels, true_midxs = zip(*true_labels) # prune mst (make sure to deepcopy!!!!) pruned_mst = deepcopy(mst) pruned_mask = pruned_mst.data > threshold pruned_mst.row = pruned_mst.row[pruned_mask] pruned_mst.col = pruned_mst.col[pruned_mask] pruned_mst.data = pruned_mst.data[pruned_mask] # get connected components to get clusters of `pruned_mst` n_components, labels = connected_components(csgraph=pruned_mst, directed=False, return_labels=True) pred_midxs = np.arange(labels.size) label_mask = np.isin(pred_midxs, true_midxs) pred_labels = zip(labels[label_mask], pred_midxs[label_mask]) pred_labels = sorted(pred_labels, key=lambda x: x[1]) pred_labels, _ = zip(*pred_labels) return { 'fmi': fowlkes_mallows_score(true_labels, pred_labels), 'rand_index': adjusted_rand_score(true_labels, pred_labels), 'pred_labels': pred_labels, 'true_labels': true_labels, 'midxs': true_midxs }
def kmeans(): from sklearn.datasets import load_digits data, target = load_digits(return_X_y=True) clustering = KMeans(n_clusters=10).fit(data) results = [[0 for _ in range(10)] for __ in range(10)] for i, val in enumerate(clustering.labels_): results[val - 1][target[i]] += 1 print( 'Kmeans Clustering Confusion Matrix \n Cluster on X axis, Target on Y axis', end='\n ') [print("{:3d}".format(i), end=' ') for i in range(10)] print('', end='\n ') [print("_ _ ", end='') for i in range(10)] print('') for x in range(10): print(x, end=' | ') for y in range(10): print("{:3d}".format(results[x][y]), end=' ') print('') # Calculating the Fowlkes-Mallows scores for Kmeans print('For K-Means, Fowlkes-Mallows Score is: ' + str( metrics.fowlkes_mallows_score(target[:500], clustering.labels_[:500])) + '\n' + '\n')
def evaluate(communities1, communities2, number_of_algorithms): if not communities2: return None c1 = communities1.copy() c2 = communities2.copy() # Making sure they have the same size c1_keys = list(c1.keys()) c2_keys = list(c2.keys()) for key in c1_keys: if not key in c2: del c1[key] for key in c2_keys: if not key in c1: del c2[key] od1 = collections.OrderedDict(sorted(c1.items())) od2 = collections.OrderedDict(sorted(c2.items())) evaluations = [] for i in range(0,number_of_algorithms): cluster1_list = list(map(lambda x: x[i], od1.values())) cluster2_list = list(map(lambda x: x[i], od2.values())) ari = metrics.adjusted_rand_score(cluster1_list, cluster2_list) ami = metrics.adjusted_mutual_info_score(cluster1_list, cluster2_list) v_score = metrics.v_measure_score(cluster1_list, cluster2_list) fmc = metrics.fowlkes_mallows_score(cluster1_list, cluster2_list) evaluations.append((ari,ami,v_score,fmc)) return evaluations
def test_given_disjoint_graphs_embeddings_cluster(self): """ Embedding disjoint subgraphs should cluster correctly """ n_clusters = 5 graph_size = 150 G = nx.complete_graph(graph_size) for i in range(1, n_clusters): G = nx.disjoint_union(G, nx.complete_graph(graph_size)) labels = [] for i in range(n_clusters): labels.append([i] * graph_size) labels = sum(labels, []) # Embed Graph and cluster around it G = cg.csrgraph(G) v = G.ggvec(n_components=4, tol=0.1, max_epoch=550, learning_rate=0.05, max_loss=1., verbose=False) cluster_hat = cluster.AgglomerativeClustering( n_clusters=n_clusters, affinity='cosine', linkage='average').fit(v).labels_ r1 = metrics.adjusted_mutual_info_score(cluster_hat, labels) r2 = metrics.adjusted_rand_score(cluster_hat, labels) r3 = metrics.fowlkes_mallows_score(cluster_hat, labels) self.assertGreaterEqual(r1, 0.65) self.assertGreaterEqual(r2, 0.65) self.assertGreaterEqual(r3, 0.65)
def test_mlops_fowlkes_mallows_score_apis(): mlops.init(ctx=None, mlops_mode=MLOpsMode.STAND_ALONE) labels_pred = [1, 0, 1, 2, 3, 0] labels_actual = [0, 1, 0, 1, 3, 1] fms = metrics.fowlkes_mallows_score(labels_actual, labels_pred) # first way mlops.set_stat(ClusteringMetrics.FOWLKES_MALLOWS_SCORE, fms) # second way mlops.metrics.fowlkes_mallows_score(labels_true=labels_actual, labels_pred=labels_pred) # should throw error if not numeric number is provided with pytest.raises(MLOpsStatisticsException): mlops.set_stat(ClusteringMetrics.FOWLKES_MALLOWS_SCORE, [1, 2, 3]) # should throw error if labels predicted is different length than actuals with pytest.raises(ValueError): labels_pred_missing_values = [0, 0, 0, 1] mlops.metrics.fowlkes_mallows_score( labels_true=labels_actual, labels_pred=labels_pred_missing_values) mlops.done()
def generate_cluster_labels(classes, K): #def generate_cluster_labels(classes,K): #Each cluster is defined by the majority represented cluster class_labels = [] confusion_mtx = [] i = 0 for c in classes: class_labels.append([]) confusion_mtx.append(np.zeros(K)) confusion_mtx[i][i] = -1 i += 1 #tally the votes for the highest number number of votes for a class. for c in classes: votes = np.zeros(K) for i in range(0, len(c)): votes[c[i].label] += 1 dc = np.where(votes == max(votes)) confusion_mtx[dc[0][0]] += votes print( 'Cluster: %d has %d votes. This cluster has been labeled: C-%d ' % (i, votes[dc[0]], dc[0])) print('Cluster Confusion Matrix:') confusion_mtx = np.array(confusion_mtx) print(confusion_mtx) #Generate the accuracy matrix to get scores print('Fowlkes-Mallows Scores') labels_true = (np.identity(10) * confusion_mtx).flatten() labels_pred = confusion_mtx.flatten() score = metrics.fowlkes_mallows_score(labels_true, labels_pred) print('Accuracy Score:%f' % (score * 100))
def dimReducedClusters(x_data, y_data, x, n): kmeans = KMeans(n_clusters=2) kmeans.fit(x_data) kmeans.predict(x_data) kLabels = kmeans.labels_ if x == 0: nData = pd.DataFrame(x_data) nData['cluster'] = kLabels nNetwork(nData, y_data, n + ': After K-Means') em = GaussianMixture(n_components=2) em.fit(x_data) eLabels = em.predict(x_data) if x == 0: nData = pd.DataFrame(x_data) nData['cluster'] = eLabels nNetwork(nData, y_data, n + ': After EM') a = silhouette_score(x_data, kLabels) b = adjusted_rand_score(y_data, kLabels) c = adjusted_mutual_info_score(y_data, kLabels) d = homogeneity_score(y_data, kLabels) e = completeness_score(y_data, kLabels) f = fowlkes_mallows_score(y_data, kLabels) g = em.bic(x_data) h = em.aic(x_data) return a, b, c, d, e, f, g, h
def evaluate(label, pred): nmi = metrics.normalized_mutual_info_score(label, pred) ari = metrics.adjusted_rand_score(label, pred) f = metrics.fowlkes_mallows_score(label, pred) pred_adjusted = get_y_preds(label, pred, len(set(label))) acc = metrics.accuracy_score(pred_adjusted, label) return nmi, ari, f, acc
def get_performance_metrics(y_pred_num, y_actual): from sklearn import metrics adjusted_rand_score = metrics.adjusted_rand_score(y_actual, y_pred_num) adjusted_mutual_info_score = metrics.adjusted_mutual_info_score( y_actual, y_pred_num) normalized_mutual_info_score = metrics.normalized_mutual_info_score( y_actual, y_pred_num) homogeneity_score = metrics.homogeneity_score(y_actual, y_pred_num) completeness_score = metrics.completeness_score(y_actual, y_pred_num) v_measure_score = metrics.v_measure_score(y_actual, y_pred_num) fowlkes_mallows_score = metrics.fowlkes_mallows_score(y_actual, y_pred_num) metric_names = [ "adjusted_rand_score", "adjusted_mutual_info_score", "normalized_mutual_info_score", "homogeneity_score", "completeness_score", "v_measure_score", "fowlkes_mallows_score" ] metric_values = [ adjusted_rand_score, adjusted_mutual_info_score, normalized_mutual_info_score, homogeneity_score, completeness_score, v_measure_score, fowlkes_mallows_score ] print "Metrics" print "\t".join(metric_names) print('\t'.join([str(x) for x in metric_values])) return metric_names, metric_values
def kmeans_clustering(X_train, y_train, X_test, y_test, genre_list): scalar = StandardScaler() scalar.fit(X_train, y_train) new_data = scalar.transform(X_train) kmeans = KMeans(init='k-means++', n_init=10, n_clusters=4, max_iter=300) rVal = kmeans.fit(X_train, y_train) kmeans_predictions = kmeans.predict(X_test) print("the randomized score is : ", metrics.adjusted_rand_score(y_test, kmeans_predictions)) print("the normalized mutual info score is : ", metrics.normalized_mutual_info_score(y_test, kmeans_predictions)) print("the mutual info score is : ", metrics.mutual_info_score(y_test, kmeans_predictions)) print( "the homogenity, completeness and v measure score is : ", metrics.homogeneity_completeness_v_measure(y_test, kmeans_predictions)) print("the fowlkes mallows score is : ", metrics.fowlkes_mallows_score(y_test, kmeans_predictions)) labels = kmeans.labels_ print( "the silhouette score is :", metrics.silhouette_score(X_test, kmeans_predictions, metric='euclidean')) print(kmeans_predictions) print(y_test) centers = rVal.cluster_centers_ distances = pairwise_distances(new_data, centers, metric='euclidean') clusters = np.argmin(distances, axis=1) print(len(clusters)) plotSamples = PCA(n_components=2).fit_transform(new_data) plotClusters(plotSamples, clusters, kmeans) joblib.dump(kmeans, 'saved_models/model_kmeans.pkl')
def evaluate(self, X, y=None): """ Depending on the clustering task, evaluate the predictor and store its performance. Args: X (array-like): Test samples y (array-like, optional): True labels Returns: eval_dict: Dictionary containing evaluations """ # Start by predicting clusters y_pred = self.predict(X) # Result dict eval_dict = dict() # Check if ground truth is available: if y is not None: eval_dict['rand_index'] = m.adjusted_rand_score(y, y_pred) eval_dict['mutual_information'] = m.adjusted_mutual_info_score( y, y_pred) eval_dict['fowlkes_mallows_score'] = m.fowlkes_mallows_score( y, y_pred) # Compute label-agnostic metrics: eval_dict['silhouette_score'] = m.silhouette_score(X, y_pred) eval_dict['calinski_harabaz_score'] = m.calinski_harabaz_score( X, y_pred) return eval_dict
def agglo_clustering(ctx): """ Agglomerative clustering function to be run on dataset. """ clusters = 10 linkage = 'ward' print("loading data...") x_train, _, y_train = load_data(ctx.obj["data_folder"], shuffle_seed=ctx.obj["seed"]) print( "Running agglomerative clustering where linkage = {} and n_clusters = {}" .format(linkage, clusters)) model = AgglomerativeClustering(linkage=linkage, n_clusters=clusters) labels_predicted = model.fit_predict(x_train) y_train = column_or_1d(y_train) score = metrics.adjusted_rand_score(y_train, labels_predicted) print(f"Accuracy: {score}.") score = metrics.homogeneity_score(y_train, labels_predicted) print(f"Homogeneity Score: {score}.") score = metrics.completeness_score(y_train, labels_predicted) print(f"Completeness Score: {score}.") score = metrics.v_measure_score(y_train, labels_predicted) print(f"V Measure Score: {score}.") score = metrics.fowlkes_mallows_score(y_train, labels_predicted) print(f"Fowlkes Mallows Score: {score}.")
def compareAB(A, B, X): #measures the similarity of the two assignments, ignoring permutations and with chance normalization ars = metrics.adjusted_rand_score(A, B) ars_str = '%17.3f' % ars # each cluster contains only members of a single class hs = homogeneity_score(A, B) hs_str = '%17.3f' % hs #all members of a given class are assigned to the same cluster cs = completeness_score(A, B) cs_str = '%17.3f' % cs vms = metrics.v_measure_score(A, B) vms_str = '%17.3f' % vms # geometric mean of the pairwise precision and recall fowlkes_mallows_score = metrics.fowlkes_mallows_score(A, B) fms_str = '%17.3f' % fowlkes_mallows_score sc = metrics.silhouette_score(X, B, metric='euclidean') sc_str = '%17.3f' % sc my_str = ars_str + "&" + hs_str + "&" + cs_str + "&" + vms_str + "&" + fms_str + "&" + sc_str return my_str
def em_clustering(ctx): """ Gaussian Mixture function to be run on dataset. """ covariance_type = 'spherical' n_components = 10 print("loading data...") x_train, _, y_train = load_data(ctx.obj["data_folder"], shuffle_seed=ctx.obj["seed"]) print("Running Gaussian Mixture...") model = GaussianMixture(n_components=n_components, covariance_type=covariance_type, verbose=2) labels_predicted = model.fit_predict(x_train) y_train = column_or_1d(y_train) score = metrics.adjusted_rand_score(y_train, labels_predicted) print(f"Accuracy: {score}.") score = metrics.homogeneity_score(y_train, labels_predicted) print(f"Homogeneity Score: {score}.") score = metrics.completeness_score(y_train, labels_predicted) print(f"Completeness Score: {score}.") score = metrics.v_measure_score(y_train, labels_predicted) print(f"V Measure Score: {score}.") score = metrics.fowlkes_mallows_score(y_train, labels_predicted) print(f"Fowlkes Mallows Score: {score}.")
def bench_kmeans(estimator, name, k, x_train, y_train, x_test, y_test): bench_kmeans_format = '% 9s %.2i %.3f %.2fs %.3f %.2fs %i %.3f %.3f %.3f %.3f %.3f %.3f' #%.3f %.3f' # Train. start = time.time() estimator.fit(x_train) train_time = time.time() - start # Test. start = time.time() predicted = estimator.predict(x_test) test_time = time.time() - start #print('% 9s' % 'init k' # ' time acc inertia h**o compl v-meas ARI AMI FMI') results = (name, k, train_time, metrics.accuracy_score(y_train, estimator.labels_), test_time, metrics.accuracy_score(y_test, predicted), estimator.inertia_, metrics.homogeneity_score(y_train, estimator.labels_), metrics.completeness_score(y_train, estimator.labels_), metrics.v_measure_score(y_train, estimator.labels_), metrics.adjusted_rand_score(y_train, estimator.labels_), metrics.adjusted_mutual_info_score(y_train, estimator.labels_), metrics.fowlkes_mallows_score(y_train, estimator.labels_)) #metrics.calinski_harabaz_score(x_train, estimator.labels_), #metrics.silhouette_score(x_train, estimator.labels_, metric='euclidean'))) #print(bench_kmeans_format % (results)) # TODO why does silhouette score throw an error about label size? # TODO all x_train, estimator.labels_, and y_train all have same size (and its not 1) # TODO should we pass explicit sample size to silhouette score? len(x_train) return list(results)
def evaluate_clustering_performance(clusters, labels): set_of_dimensionality = set() for cluster in clusters: set_of_dimensionality.add(frozenset(cluster.dimensions)) # Evaluating performance in all dimensionality for dim in set_of_dimensionality: print("\nEvaluating clusters in dimension: ", list(dim)) # Finding clusters with same dimensions clusters_in_dim = [] for c in clusters: if c.dimensions == dim: clusters_in_dim.append(c) clustering_labels = np.zeros(np.shape(labels)) for i, c in enumerate(clusters_in_dim): clustering_labels[list(c.data_point_ids)] = i + 1 print("Number of clusters: ", len(clusters_in_dim)) print("Adjusted Rand index: ", metrics.adjusted_rand_score(labels, clustering_labels)) print("Mutual Information: ", metrics.adjusted_mutual_info_score(labels, clustering_labels)) print( "Homogeneity, completeness, V-measure: ", metrics.homogeneity_completeness_v_measure(labels, clustering_labels)) print("Fowlkes-Mallows: ", metrics.fowlkes_mallows_score(labels, clustering_labels))
def fowlkes_mallows_index(df: pd.DataFrame, column_true: str = "GroundTruth", column_pred: str = "label") -> float: """ Measure the similarity of two clusterings of a set of points. The Fowlkes-Mallows index (FMI) is defined as the geometric mean between of the precision and recall: FMI = TP / sqrt((TP + FP) * (TP + FN)) TThe score ranges from 0 to 1. A high value indicates a good similarity between two clusters. Parameters ---------- df : DataFrame Input DataFrame with at least two columns: True and Predicted ones column_true : str, default if 'GroundTruth' The title for column with True labels column_pred : str, default is 'label' The title for column with Predicted labels Returns ------- The FMI returns a value of from 0.0 to 1.0: Perfect labeling is scored 1.0, while Bad (e.g. independent labelings) have zero scores: """ str_labels = list(df[column_true].unique()) real_labels = dict(zip(str_labels, range(len(str_labels)))) return fowlkes_mallows_score( df[column_true].apply(lambda x: real_labels[x]).values, df[column_pred].values)
def record_sampling_clustermultimeasure_vs_opnum(wn, samplemth): cora_true = models.CoraPerformanceLog.objects.filter( explorationMethod='groundtruth').order_by('cora_id') aa_true = [item.clusterid for item in cora_true] cluster_membership = models.CoraPerformanceLog.objects.filter( explorationMethod=samplemth, workerOperationNum=wn).order_by('cora_id') bb_pred = [item.clusterid for item in cluster_membership] nmi = metrics.normalized_mutual_info_score(aa_true, bb_pred) ari = metrics.adjusted_rand_score(aa_true, bb_pred) ami = metrics.adjusted_mutual_info_score(aa_true, bb_pred) homogeneity = metrics.homogeneity_score(aa_true, bb_pred) completeness = metrics.completeness_score(aa_true, bb_pred) v_measure = metrics.v_measure_score(aa_true, bb_pred) fmi = metrics.fowlkes_mallows_score(aa_true, bb_pred) dict = { 'RecordSamplingMethod': samplemth.name, 'workerOperationNum': wn, 'nmi': nmi, 'ari': ari, 'ami': ami, 'homogeneity': homogeneity, 'completeness': completeness, 'v_measure': v_measure, 'fmi': fmi } for k, v in dict.items(): print(k, ' ', v) return dict
def clusterMeasureSet(aa_true, bb_pred, RecordSamplingMethod, workerOperationNum): nmi = metrics.normalized_mutual_info_score(aa_true, bb_pred) ari = metrics.adjusted_rand_score(aa_true, bb_pred) ami = metrics.adjusted_mutual_info_score(aa_true, bb_pred) homogeneity = metrics.homogeneity_score(aa_true, bb_pred) completeness = metrics.completeness_score(aa_true, bb_pred) v_measure = metrics.v_measure_score(aa_true, bb_pred) fmi = metrics.fowlkes_mallows_score(aa_true, bb_pred) dict = { 'RecordSamplingMethod': RecordSamplingMethod, 'workerOperationNum': workerOperationNum, 'nmi': nmi, 'ari': ari, 'ami': ami, 'homogeneity': homogeneity, 'completeness': completeness, 'v_measure': v_measure, 'fmi': fmi } for k, v in dict.items(): print(k, ' ', v) for k, v in dict.items(): print(v) return dict
def prin_clustering(test_rep, test_label, NUM_OF_CLASS): # 聚类 km = KMeans(n_clusters=NUM_OF_CLASS) #km.fit_transform(test_rep) cls_rs = km.fit_predict(test_rep) # ARI ari = metrics.adjusted_rand_score(test_label, cls_rs) # AMI ami = metrics.adjusted_mutual_info_score(test_label, cls_rs) # H,C,V H, C, V = metrics.homogeneity_completeness_v_measure(test_label, cls_rs) # FMI fmi = metrics.fowlkes_mallows_score(test_label, cls_rs) # s # s = metrics.silhouette_score(test_label, cls_rs) # DBI # dbi = metrics.davies_bouldin_score(test_label, cls_rs) # nmi nmi = metrics.normalized_mutual_info_score(test_label, cls_rs) d = dict() d['ari'] = ari d['ami'] = ami d['nmi'] = nmi d['fmi'] = fmi d['H'] = H d['C'] = C d['V'] = V print('ARI:%.4f,AMI:%.4f,HCV:%.4f %.4f %.4f FMI:%.4f NMI:%.4f' % (ari, ami, H, C, V, fmi, nmi)) return d
def get_fowlkes_mallows_score(standard_file, prediction_file, isjson=False, isint=False): if isjson: standard_data = AbstractionUtility.read_json(standard_file) standard_labels = standard_data.values() else: standard_labels = ExternalEvaluation.get_evaluated(standard_file) prediction_labels = ExternalEvaluation.get_evaluated(prediction_file, isint=isint) fowlkes_mallows_score = metrics.fowlkes_mallows_score(standard_labels, prediction_labels) return fowlkes_mallows_score
def compute_stability_score(estimator, X, y=None, n_draws=10, p_samples=0.8, random_state=None): if estimator.n_clusters == 1: return 0.0 # Generate data rng = np.random.RandomState(random_state) n_samples, _ = X.shape data = rng.uniform(size=(n_draws, 2 * n_samples)) < p_samples score = np.empty(n_draws) for k, d in enumerate(data): p1, p2 = np.split(d, 2) labels1 = estimator.fit_predict(X[p1]) labels2 = estimator.fit_predict(X[p2]) score[k] = fowlkes_mallows_score(labels1[p2[p1]], labels2[p1[p2]]) return score.mean()
def get_metrics(): dict_lable = {'eco': 0, 'env': 1, 'sports': 2, 'other': 3} data = pd.read_table('id2class.txt', header=None, delim_whitespace=True) data2 = pd.read_table('result.txt', header=None, delim_whitespace=True) list_true = [] list_pred = [] for i in range(len(data)): data.iloc[i, 1] = dict_lable[data.iloc[i, 1]] list_true.append(data.iloc[i, 1]) list_pred.append(data2.iloc[i, 1]) # 文档链接 http://scikit-learn.org/stable/modules/clustering.html#clustering-performance-evaluation # 2.3.9.1 Adjusted Rand index (RI) # 2.3.9.2. Mutual Information based scores(NMI) # 2.3.9.4. Fowlkes-Mallows scores(FMI) # 章节号为文档里面的章节号 print (metrics.adjusted_rand_score(list_true, list_pred)) # RI指数,越接近1越好 print (metrics.adjusted_mutual_info_score(list_true, list_pred)) # NMI指数,越接近1越好 print (metrics.fowlkes_mallows_score(list_true, list_pred)) # FMI指数,越接近1越好
def compare_clusters(prog, argv): parser = argparse.ArgumentParser(prog=prog, description='Compare Equivalence Classes') parser.add_argument('classes', metavar='eqclass', type=str, nargs=2, help='Ground Truth / Prediction') parser.add_argument('-ar', action='store_true', default=False, help='Adjusted rand score') parser.add_argument('-mi', action='store_true', default=False, help='Mutual info score') parser.add_argument('-ami', action='store_true', default=False, help='Adjusted mutual info score') parser.add_argument('-nmi', action='store_true', default=False, help='Normalised mutual info score') parser.add_argument('-pur', action='store_true', default=False, help='Purity') parser.add_argument('-pr', action='store_true', default=False, help='Classic Precision/Recall') parser.add_argument('-fm', action='store_true', default=False, help='Fowlkes-Mallow score') parser.add_argument('-remove-identical', action='store_true', default=False, help='Remove identical clusters before comparing') parser.add_argument('-f', type=str, help='Write results to filename') parser.add_argument('-test', action='store_true', default=False, help='run tests') args = parser.parse_args(argv) # These are the converted example from: # https://nlp.stanford.edu/IR-book/html/htmledition/evaluation-of-clustering-1.html if args.test: ground_truth = Cluster() prediction = Cluster() prediction.insert(0,1,2,3,4,5) prediction.insert(6,7,8,9,10,11) prediction.insert(12, 13, 14, 15, 16) ground_truth.insert(0,2,3,4,5,6,12,14) ground_truth.insert(1, 7, 8, 9, 11) ground_truth.insert(10,13,15,16) else: ground_truth = Cluster.from_file(args.classes[0], must_exist=True) prediction = Cluster.from_file(args.classes[1], must_exist=True) # remove identical clusters, if desired if args.remove_identical: identical = list() for cluster in ground_truth: cand = prediction.get_cluster(list(cluster)[0]) if not cand: continue elif cand == cluster: identical.append(cand) log.info('Removing %d identical clusters (%d elements)...' % (len(identical), sum([len(x) for x in identical]))) for cluster in identical: for element in cluster: ground_truth.remove_key(element) prediction.remove_key(element) ground_truth.optimize() prediction.optimize() if (args.pr): prec_rec(ground_truth, prediction) # intermix all keys ground_truth_keys = ground_truth.get_keys() prediction_keys = prediction.get_keys() missing = ground_truth_keys - prediction_keys log.info('%d keys missing in prediction' % len(missing)) for key in missing: prediction.insert_single(key) missing = prediction_keys - ground_truth_keys log.info('%d keys missing in ground truth' % len(missing)) for key in missing: ground_truth.insert_single(key) gt = list(sorted(ground_truth.lookup.items())) t = list(sorted(prediction.lookup.items())) gt = [x[1] for x in gt] t = [x[1] for x in t] log.info('Number of equiv classes: %d' % len(ground_truth)) h**o, comp, vm = metrics.homogeneity_completeness_v_measure(gt, t) log.info("Homogeneity: %0.3f" % h**o) log.info("Completeness: %0.3f" % comp) log.info("V-measure: %0.3f" % vm) if args.ar: ar = metrics.adjusted_rand_score(gt, t) log.info("Adjusted rand score: %0.3f" % ar) if args.mi: mi = metrics.mutual_info_score(gt, t) log.info("Mutual info score: %0.3f" % mi) if args.ami: ami = metrics.adjusted_mutual_info_score(gt, t) log.info("Adjusted mutual info score: %0.3f" % ami) if args.nmi: nmi = metrics.normalized_mutual_info_score(gt, t) log.info("Normalised mutual info score: %0.3f" % nmi) if args.pur: elements = len(gt) hits = 0 for w in ground_truth: this = 0 for element in w: tmp = prediction[element] foo = len(w & tmp) if foo > this: this = foo hits += this purity = hits / elements log.info('Purity: %0.3f' % purity) if args.fm: fm = metrics.fowlkes_mallows_score(gt, t) log.info("Fowlkes-Mallows score: %0.3f" % fm) if args.f: with open(args.f, 'w') as f: f.write("h**o: %0.3f\n" % h**o) f.write("comp: %0.3f\n" % comp) f.write("vm: %0.3f\n" % vm) if args.ar: f.write("ar: %0.3f\n" % ar) if args.mi: f.write("mi: %0.3f\n" % mi) if args.nmi: f.write("nmi: %0.3f\n" % nmi) if args.ami: f.write("ami: %0.3f\n" % ami) if args.pur: f.write("pur: %0.3f\n" % purity) if args.fm: f.write("fm: %0.3f\n" % fm) return 0