def expectation_maximization(X,y,dataset_name):
X_train, X_test, y_train, y_test = train_test_split(X,y, random_state=65)
train_scores = []
train_homo = []
train_completeness = []
train_v_score = []
test_scores = []
test_homo = []
test_completeness = []
test_v_score = []
kvals = [x for x in range(2,51)]
for k in range(2, 51):
print("k= {}".format(k))
clf = GaussianMixture(n_components=k, max_iter=1000)
# Train on train data, recording accuracy, homogeneity, completeness, and v_measure
train_pred = clf.fit_predict(X_train)
train_score = fowlkes_mallows_score(y_train, train_pred)
train_scores.append(train_score)
homogeneity, completeness, v_measure = homogeneity_completeness_v_measure(y_train, train_pred)
train_homo.append(homogeneity)
train_completeness.append(completeness)
train_v_score.append(v_measure)
# Evaluate same metrics on test set
test_pred = clf.predict(X_test)
test_score = fowlkes_mallows_score(y_test, test_pred)
test_scores.append(test_score)
homogeneity, completeness, v_measure = homogeneity_completeness_v_measure(y_test, test_pred)
test_homo.append(homogeneity)
test_completeness.append(completeness)
test_v_score.append(v_measure)
print("done")
print("generating plots")
plt.figure()
plt.title('Folkes-Mallows Score of Expectation Maximization on {} Dataset'.format(dataset_name))
plt.xlabel('Number of Components')
plt.ylabel('Folkes-Mallows Score')
plt.plot(kvals, train_scores, label='Training Score')
plt.plot(kvals, test_scores, label='Test Score')
plt.legend(loc='upper left')
plt.show(block=False)
plt.figure()
plt.title('Performance Metrics of Expectation Maximization on {} Dataset'.format(dataset_name))
plt.xlabel('K Value (Number of Clusters)')
plt.ylabel('Score (Range 0.0 to 1.0)')
plt.plot(kvals, train_homo, label='Training Homogeneity')
plt.plot(kvals, test_homo, label='Test Homogeneity')
plt.plot(kvals, train_completeness, label='Training Completeness')
plt.plot(kvals, test_completeness, label='Test Completeness')
plt.plot(kvals, train_v_score, label='Training V-Measure')
plt.plot(kvals, test_v_score, label='Test V-Measure')
plt.legend(loc='upper left')
plt.show(block=False)
def comparison_table(df, clusterers, scaler, ae_dims=[]):
global AE
table = []
if is_pretty_table:
table = PrettyTable(['Method', 'Mean silhouette score', 'V Score', 'Fowlkes Score'])
lanes_df = df['lane']
# indices of elements who are from MID, TOP or JUNGLE
known_label_indices = [i for i in range(len(lanes_df)) if lanes_df[i] in ['MIDDLE', 'TOP', 'JUNGLE']]
true_labels = lanes_df[known_label_indices]
data_no_labels = df.drop(['lane', 'championId'], axis=1)
data = scaleColumns(data_no_labels, scaler=scaler)
scaled_data = data.copy()
if not ae_dims:
for clusterer_name in clusterers:
clusterer = clusterers[clusterer_name]
labels = clusterer.predict(data)
sil_score = silhouette_score(data, labels)
# labels for elements from MID TOP or JUNGLE
known_lane_labels = labels[known_label_indices]
v_score = v_measure_score(true_labels, known_lane_labels)
fowlkes_score = fowlkes_mallows_score(true_labels, known_lane_labels)
if is_pretty_table:
table.add_row([f'{clusterer_name}',
"{:.3f}".format(sil_score),
"{:.3f}".format(v_score),
"{:.3f}".format(fowlkes_score)])
else:
print(f'{clusterer_name}: {sil_score}, {v_score}, {fowlkes_score}')
else:
for ae_dim in ae_dims:
if AE.is_fit is True and AE.mid_size == ae_dim:
pass
else:
AE = AutoEncoder(54, ae_dim)
data = AE.fit_transform(data)
for clusterer_name in clusterers:
clusterer = clusterers[clusterer_name]
labels = clusterer.predict(data)
sil_score = silhouette_score(scaled_data, labels)
# labels for elements from MID TOP or JUNGLE
known_lane_labels = labels[known_label_indices]
v_score = v_measure_score(true_labels, known_lane_labels)
fowlkes_score = fowlkes_mallows_score(true_labels, known_lane_labels)
if is_pretty_table:
table.add_row([f'AE {ae_dim}-{clusterer_name}',
"{:.3f}".format(sil_score),
"{:.3f}".format(v_score),
"{:.3f}".format(fowlkes_score)])
else:
print(f'AE {ae_dim}-{clusterer_name}: {sil_score:.3f}, {v_score:.3f}, {fowlkes_score:.3f}')
if is_pretty_table:
print(table)
def compare(self, model, X, y=None):
"""
Compares the score of a sample in two models.
Returns a crossvalidation of metrics, predictions and score.
More info: https://machinelearningmastery.com/compare-machine-learning-algorithms-python-scikit-learn/
:param model: model
:param X: data
:type model: LinearModel
:type X: ndarray or scipy.sparse matrix, (n_samples, n_features)
"""
metrict_dict = {} #Adjusted Rand Index, Mutual Information
labels_true = y
labels_pred = self.model_.predict(X)
labels_pred_other = model.predict(X)
labels_other = model.get_model().labels_
labels = model_.labels_
if y != None:
#These metrics require the knowledge of the ground truth classes
metrict_dict['ARI'] = (metrics.adjusted_rand_score(
labels_true, labels_pred),
metrics.adjusted_rand_score(
labels_true, labels_pred_other))
metrict_dict['MIS'] = (metrics.adjusted_mutual_info_score(
labels_true, labels_pred),
metrics.adjusted_mutual_info_score(
labels_true, labels_pred_other))
metrict_dict['homogeneity'] = (metrics.homogeneity_score(
labels_true, labels_pred),
metrics.homogeneity_score(
labels_true, labels_pred_other))
metrict_dict['completeness'] = (metrics.completeness_score(
labels_true, labels_pred),
metrics.completeness_score(
labels_true,
labels_pred_other))
metrict_dict['v_measure'] = (metrics.v_measure_score(
labels_true, labels_pred),
metrics.v_measure_score(
labels_true, labels_pred_other))
metrict_dict['FMS'] = (metrics.fowlkes_mallows_score(
labels_true, labels_pred),
metrics.fowlkes_mallows_score(
labels_true, labels_pred_other))
#These metrics DON'T require the knowledge of the ground truth classes
metrict_dict['SS'] = (metrics.silhouette_score(X,
labels,
metric='euclidean'),
metrics.silhouette_score(X,
labels_other,
metric='euclidean'))
metrict_dict['CHI'] = (metrics.calinski_harabaz_score(X, labels),
metrics.calinski_harabaz_score(X, labels_other))
return metrict_dict
def external_validation(pred_, moa_, treat_, comp_, random_, same_):
metrices_dict = dict()
# Completeness
metrices_dict['comple_moa-pred'] = [completeness_score(moa_, pred_)]
metrices_dict['comple_treat-moa'] = [completeness_score(treat_, moa_)]
metrices_dict['comple_treat-pred'] = [completeness_score(treat_, pred_)]
metrices_dict['comple_treat-rand'] = [completeness_score(treat_, random_)]
metrices_dict['comple_treat-same'] = [completeness_score(treat_, same_)]
metrices_dict['comple_comp-moa'] = [completeness_score(comp_, moa_)]
metrices_dict['comple_comp_pred'] = [completeness_score(comp_, pred_)]
metrices_dict['comple_comp_rand'] = [completeness_score(comp_, random_)]
metrices_dict['comple_comp_same'] = [completeness_score(comp_, same_)]
# Jaccard similarity coefficient
metrices_dict['jaccard_moa-pred'] = [jaccard(moa_, pred_)]
metrices_dict['jaccard_treat-moa'] = [jaccard(treat_, moa_)]
metrices_dict['jaccard_treat-pred'] = [jaccard(treat_, pred_)]
metrices_dict['jaccard_treat-rand'] = [jaccard(treat_, random_)]
metrices_dict['jaccard_treat-same'] = [jaccard(treat_, same_)]
metrices_dict['jaccard_comp-moa'] = [jaccard(comp_, moa_)]
metrices_dict['jaccard_comp_pred'] = [jaccard(comp_, pred_)]
metrices_dict['jaccard_comp_rand'] = [jaccard(comp_, random_)]
metrices_dict['jaccard_comp_same'] = [jaccard(comp_, same_)]
# Adjusted Rand index
metrices_dict['adj-rand_moa-pred'] = [adjusted_rand_score(moa_, pred_)]
metrices_dict['adj-rand_treat-moa'] = [adjusted_rand_score(treat_, moa_)]
metrices_dict['adj-rand_treat-pred'] = [adjusted_rand_score(treat_, pred_)]
metrices_dict['adj-rand_treat-rand'] = [adjusted_rand_score(treat_, random_)]
metrices_dict['adj-rand_treat-same'] = [adjusted_rand_score(treat_, same_)]
metrices_dict['adj-rand_comp-moa'] = [adjusted_rand_score(comp_, moa_)]
metrices_dict['adj-rand_comp_pred'] = [adjusted_rand_score(comp_, pred_)]
metrices_dict['adj-rand_comp_rand'] = [adjusted_rand_score(comp_, random_)]
metrices_dict['adj-rand_comp_same'] = [adjusted_rand_score(comp_, same_)]
# Fowlkes-Mallows index
metrices_dict['fow-mal_moa-pred'] = [fowlkes_mallows_score(moa_, pred_)]
metrices_dict['fow-mal_treat-moa'] = [fowlkes_mallows_score(treat_, moa_)]
metrices_dict['fow-mal_treat-pred'] = [fowlkes_mallows_score(treat_, pred_)]
metrices_dict['fow-mal_treat-rand'] = [fowlkes_mallows_score(treat_, random_)]
metrices_dict['fow-mal_treat-same'] = [fowlkes_mallows_score(treat_, same_)]
metrices_dict['fow-mal_comp-moa'] = [fowlkes_mallows_score(comp_, moa_)]
metrices_dict['fow-mal_comp_pred'] = [fowlkes_mallows_score(comp_, pred_)]
metrices_dict['fow-mal_comp_rand'] = [fowlkes_mallows_score(comp_, random_)]
metrices_dict['fow-mal_comp_same'] = [fowlkes_mallows_score(comp_, same_)]
# Adjusted mutual information
metrices_dict['adj-mut_moa-pred'] = [adjusted_mutual_info_score(moa_, pred_)]
metrices_dict['adj-mut_treat-moa'] = [adjusted_mutual_info_score(treat_, moa_)]
metrices_dict['adj-mut_treat-pred'] = [adjusted_mutual_info_score(treat_, pred_)]
metrices_dict['adj-mut_treat-rand'] = [adjusted_mutual_info_score(treat_, random_)]
metrices_dict['adj-mut_treat-same'] = [adjusted_mutual_info_score(treat_, same_)]
metrices_dict['adj-mut_comp-moa'] = [adjusted_mutual_info_score(comp_, moa_)]
metrices_dict['adj-mut_comp_pred'] = [adjusted_mutual_info_score(comp_, pred_)]
metrices_dict['adj-mut_comp_rand'] = [adjusted_mutual_info_score(comp_, random_)]
metrices_dict['adj-mut_comp_same'] = [adjusted_mutual_info_score(comp_, same_)]
return pd.DataFrame(metrices_dict)
def evaluate(self, y_gt,y_est):
y_gt = self.type_format(y_gt)
y_est = self.type_format(y_est)
scores = {}
if '#Err' in self.criteria:
scores['#Err'] = self.ErrNumClusters(y_gt,y_est)
if 'AMI' in self.criteria:
scores['AMI'] = sm.adjusted_mutual_info_score(y_gt,y_est)
if 'ARI' in self.criteria:
scores['ARI'] = sm.adjusted_rand_score(y_gt,y_est)
if 'COMP' in self.criteria:
scores['COMP'] = sm.completeness_score(y_gt,y_est)
if 'FMS' in self.criteria:
scores['FMS'] = sm.fowlkes_mallows_score(y_gt,y_est)
#if 'HCVM' in self.criteria:
# scores['HCVM'] = sm.homogeneity_completeness_v_measure(y_gt,y_est)
if 'HS' in self.criteria:
scores['HS'] = sm.homogeneity_score(y_gt,y_est)
if 'MI' in self.criteria:
scores['MI'] = sm.mutual_info_score(y_gt,y_est)
if 'NMI' in self.criteria:
scores['NMI'] = sm.normalized_mutual_info_score(y_gt,y_est)
if 'VMS' in self.criteria:
scores['VMS'] = sm.v_measure_score(y_gt,y_est)
if 'PURITY' in self.criteria:
scores['PURITY'] = self.purity(y_gt,y_est)
# for inlier/outlier classification
if 'F1' in self.criteria:
scores['F1'] = self.F1score(y_gt,y_est)
return self.conv_return_type(scores)
def evaluateClustering(X_true, y_true, y_pred, model, classes):
print('\n', model.upper())
ars = adjusted_rand_score(y_true, y_pred)
print('Adjusted random score: ', ars)
ss = silhouette_score(X_true, y_pred)
print('Silhouette score: ', ss)
hs = homogeneity_score(y_true, y_pred)
print('Homogeneity score: ', hs)
cs = completeness_score(y_true, y_pred)
print('Completeness score: ', cs)
vms = v_measure_score(y_true, y_pred)
print('V-measure score: ', vms)
fms = fowlkes_mallows_score(y_true, y_pred)
print('Fowlkes-Mallows score: ', fms)
chs = calinski_harabaz_score(X_true, y_pred)
print('Calinski-Harabaz score: ', chs)
cm = contingency_matrix(y_true, y_pred)
orderContingencyMatrix(cm)
acc = accuracy(cm)
print('Supervised-like Accuracy: ', acc)
mean = meanScore(X_true, y_true, y_pred)
print('Mean Score: ', mean)
plotContingencyMatrix(cm, model, classes)
def compute_results(G_data, B_data, name, encoder,r_state=0,only_kmeans=False):
B_data_X = encoder.detach().numpy()
kmeans = KMeans(n_clusters=get_clusters(G_data,name),init='k-means++',random_state=r_state)
if not only_kmeans:
kmeans.fit(B_data_X)
else:
kmeans.fit(B_data)
X_ae = kmeans.labels_ # Calculated labels
# Finding truth values
if name =='karate':
c_groups=[0,0,0,0,0,0,0,0,1,1,0,0,0,0,1,1,0,0,1,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1]
elif name == 'email':
c_groups=[1, 1, 21, 21, 21, 25, 25, 14, 14, 14, 9, 14, 14, 26, 4, 17, 34, 1, 1, 14, 9, 9, 9, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 5, 34, 14, 14, 17, 17, 10, 10, 36, 37, 5, 7, 4, 22, 22, 21, 21, 21, 21, 7, 7, 36, 21, 25, 4, 8, 15, 15, 15, 37, 37, 9, 1, 1, 10, 10, 3, 3, 3, 29, 15, 36, 36, 37, 1, 36, 34, 20, 20, 8, 15, 9, 4, 5, 4, 20, 16, 16, 16, 16, 16, 38, 7, 7, 34, 38, 36, 8, 27, 8, 8, 8, 10, 10, 13, 13, 6, 26, 10, 1, 36, 0, 13, 16, 16, 22, 6, 5, 4, 0, 28, 28, 4, 2, 13, 13, 21, 21, 17, 17, 14, 36, 8, 40, 35, 15, 23, 0, 0, 7, 10, 37, 27, 35, 35, 0, 0, 19, 19, 36, 14, 37, 24, 17, 13, 36, 4, 4, 13, 13, 10, 4, 38, 32, 32, 4, 1, 0, 0, 0, 7, 7, 4, 15, 16, 40, 15, 15, 15, 15, 0, 21, 21, 21, 21, 5, 4, 4, 4, 4, 4, 4, 4, 5, 5, 4, 4, 22, 19, 19, 22, 34, 14, 0, 1, 17, 37, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 10, 23, 0, 4, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 10, 14, 14, 1, 14, 7, 13, 20, 31, 40, 6, 4, 0, 8, 9, 9, 10, 0, 10, 14, 14, 14, 14, 39, 17, 4, 28, 17, 17, 17, 4, 4, 0, 0, 23, 4, 21, 36, 36, 0, 22, 21, 15, 37, 0, 4, 4, 4, 14, 4, 7, 7, 1, 15, 15, 38, 26, 20, 20, 20, 21, 9, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 10, 19, 7, 7, 17, 16, 14, 9, 9, 9, 8, 8, 13, 39, 14, 10, 17, 17, 13, 13, 13, 13, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 16, 27, 8, 8, 14, 14, 14, 10, 14, 35, 37, 14, 36, 10, 7, 20, 10, 16, 36, 36, 14, 8, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 4, 9, 4, 0, 4, 16, 38, 14, 14, 21, 26, 27, 28, 21, 4, 1, 1, 9, 10, 15, 4, 26, 14, 35, 10, 34, 4, 4, 12, 17, 17, 14, 37, 37, 37, 34, 6, 13, 13, 13, 13, 4, 14, 10, 10, 10, 3, 17, 17, 17, 1, 4, 14, 14, 6, 27, 22, 21, 4, 4, 1, 34, 17, 30, 30, 4, 23, 14, 15, 1, 22, 12, 31, 6, 15, 15, 8, 15, 8, 8, 1, 15, 22, 2, 3, 4, 10, 4, 14, 14, 25, 6, 6, 40, 4, 36, 23, 14, 3, 14, 14, 14, 14, 14, 14, 14, 14, 14, 31, 15, 15, 14, 0, 23, 35, 8, 4, 1, 1, 35, 23, 21, 2, 4, 4, 9, 14, 4, 10, 25, 14, 14, 3, 21, 35, 4, 9, 15, 6, 9, 3, 15, 23, 4, 4, 4, 11, 35, 10, 6, 15, 15, 15, 22, 2, 2, 14, 4, 3, 14, 27, 31, 34, 4, 4, 19, 14, 14, 4, 4, 14, 14, 21, 4, 14, 4, 0, 4, 27, 27, 17, 3, 15, 2, 4, 4, 21, 21, 11, 23, 11, 23, 17, 5, 36, 15, 23, 23, 2, 19, 4, 36, 14, 1, 22, 1, 21, 34, 14, 13, 6, 4, 37, 6, 24, 35, 6, 17, 16, 6, 4, 0, 21, 4, 26, 21, 4, 15, 7, 1, 20, 19, 7, 21, 21, 21, 19, 38, 19, 16, 23, 6, 37, 25, 1, 22, 6, 14, 1, 26, 8, 37, 4, 0, 17, 6, 17, 14, 16, 4, 32, 14, 15, 0, 23, 21, 29, 14, 14, 1, 17, 26, 15, 0, 0, 0, 22, 34, 21, 6, 16, 4, 15, 21, 0, 36, 4, 1, 1, 22, 14, 14, 30, 4, 9, 10, 4, 4, 14, 16, 16, 15, 21, 0, 4, 15, 29, 24, 21, 14, 11, 11, 9, 13, 10, 31, 4, 22, 14, 23, 1, 4, 9, 17, 27, 28, 22, 14, 20, 7, 23, 1, 6, 15, 15, 23, 4, 20, 5, 36, 10, 21, 39, 41, 31, 17, 7, 21, 34, 1, 14, 2, 18, 16, 27, 16, 38, 7, 38, 21, 1, 9, 15, 15, 15, 0, 6, 23, 28, 11, 23, 34, 24, 4, 4, 4, 24, 23, 17, 10, 17, 1, 1, 15, 15, 4, 21, 14, 14, 20, 28, 20, 22, 26, 3, 32, 4, 0, 21, 13, 4, 15, 17, 5, 4, 14, 0, 9, 21, 14, 38, 4, 14, 31, 21, 14, 6, 4, 4, 6, 17, 0, 4, 7, 16, 4, 4, 21, 1, 10, 3, 21, 4, 0, 1, 7, 17, 15, 14, 0, 9, 32, 13, 5, 2, 21, 28, 21, 22, 22, 7, 7, 33, 0, 1, 15, 4, 31, 30, 15, 11, 19, 21, 9, 21, 13, 21, 9, 32, 9, 32, 38, 9, 38, 38, 14, 9, 10, 38, 10, 22, 21, 13, 21, 4, 0, 1, 1, 23, 0, 5, 4, 4, 15, 14, 14, 13, 11, 1, 5, 5, 10, 23, 21, 14, 9, 20, 10, 19, 19, 21, 17, 19, 19, 36, 17, 35, 16, 4, 16, 4, 6, 4, 41, 6, 7, 23, 9, 23, 7, 6, 22, 36, 14, 15, 11, 35, 5, 14, 14, 15, 4, 6, 4, 9, 19, 11, 4, 29, 14, 15, 15, 5, 32, 15, 14, 5, 9, 10, 19, 13, 23, 12, 10, 21, 10, 35, 7, 22, 22, 22, 8, 21, 32, 4, 21, 21, 6, 14, 11, 14, 15, 4, 21, 1, 6, 22]
else:
c_attributes = nx.get_node_attributes(G_data, 'value')
c_groups = []
for i, val in enumerate(c_attributes.values()):
c_groups.append(val)
X_gt = np.array(c_groups)
# print(X_ae)
# print(X_gt)
return metrics.fowlkes_mallows_score(X_gt, X_ae)
def KmeansCluster(TrainMalSet, labelcsv, FeatureOption):
Logger.debug("Loading Malware and Goodware Sample Data for training and testing")
TrainMalSamples = getApkList(TrainMalSet, ".data")
Logger.info("Loaded Samples")
FeatureVectorizer = TF(input="filename", tokenizer=lambda x: x.split('\n'), token_pattern=None,
binary=FeatureOption)
X = FeatureVectorizer.fit_transform(TrainMalSamples)
get_family_dict(labelcsv)
y_train = []
for file in TrainMalSamples:
if "amd" in labelcsv:
sha256 = os.path.split(file)[-1][:-5].split('_')[-1]
else:
sha256 = os.path.split(file)[-1][:-5]
if sha256 in family_dict:
y_train.append(family_dict[sha256])
else:
y_train.append(-1)
# test
print(y_train[:20])
kmeans = KMeans(n_clusters=family_count, random_state=10).fit(X)
labels = kmeans.labels_
score = fowlkes_mallows_score(y_train, labels)
print(labels[:20])
print(family_count, score)
def test_compare_example_with_sklearn(self):
# Act
metrics = similarity_metrics(self.large_A, self.large_B)
ar_similarity = metrics.adjusted_rand()
fm_similarity = metrics.fowlkes_mallows()
ar_sklearn = []
fm_sklearn = []
for i in range(9, 1, -1):
fcluster_a = fcluster(self.large_A, i, 'maxclust')
fcluster_b = fcluster(self.large_B, i, 'maxclust')
ar = adjusted_rand_score(fcluster_a, fcluster_b)
fm = fowlkes_mallows_score(fcluster_a, fcluster_b)
ar_sklearn.append(ar)
fm_sklearn.append(fm)
# Assert
assert_almost_equal(ar_similarity, ar_sklearn)
assert_almost_equal(fm_similarity, fm_sklearn)
def eval_2(labels_true, labels_pred, is_show=True):
"""
有监督的评估
评价指标 越接近 1 越好
:param labels_true:
:param labels_pred:
:param is_show: 是否显示结果
:return:
"""
if labels_true == []:
info = f"cluster: img_sum:{len(labels_pred)}, id_sum:{len(set(labels_pred))}"
return [], info
nmi = 0 # metrics.normalized_mutual_info_score(labels_true, labels_pred) # 归一化互信息
ari = metrics.adjusted_rand_score(labels_true, labels_pred) # 调整兰德指数
# 纯度,散度, v_measure
homogeneity, completeness, v_measure_score = metrics.homogeneity_completeness_v_measure(
labels_true, labels_pred)
fmi = metrics.fowlkes_mallows_score(labels_true, labels_pred) # 几何平均数
avg_pre, avg_rec, fscore = fowlkes_mallows_score(
labels_true, labels_pred) # 调和平均数 *****
k = 0.5
fscore_2 = 2. * avg_pre * k * avg_rec / (avg_pre * k + avg_rec)
s_1 = f"gt: img_sum:{len(labels_true)}, id_sum:{len(set(labels_true))}"
s_2 = f"cluster: img_sum:{len(labels_pred)}, id_sum:{len(set(labels_pred))}"
s_3 = "有监督: 纯度, 散度, nmi, v_measure, ari:" + f"{r(homogeneity)}, {r(completeness)}, {r(nmi)}, {r(v_measure_score)}, {r(ari)}"
s_4 = 'avg_pre, avg_rec, fscore, fmi:' + f"{r(avg_pre)}, {r(avg_rec)}, {r(fscore)}, {r(fmi)}"
info = f"{s_1}\n{s_2}\n{s_3}\n{s_4}"
if is_show:
print(info)
metric = [avg_pre, avg_rec, fscore, fmi]
return metric, info
def _compute_coref_metrics_threshold(gold_clustering, mst, threshold):
# get the true_labels
true_labels = [(i, x) for i, l in enumerate(gold_clustering) for x in l]
true_labels = sorted(true_labels, key=lambda x: x[1])
true_labels, true_midxs = zip(*true_labels)
# prune mst (make sure to deepcopy!!!!)
pruned_mst = deepcopy(mst)
pruned_mask = pruned_mst.data > threshold
pruned_mst.row = pruned_mst.row[pruned_mask]
pruned_mst.col = pruned_mst.col[pruned_mask]
pruned_mst.data = pruned_mst.data[pruned_mask]
# get connected components to get clusters of `pruned_mst`
n_components, labels = connected_components(csgraph=pruned_mst,
directed=False,
return_labels=True)
pred_midxs = np.arange(labels.size)
label_mask = np.isin(pred_midxs, true_midxs)
pred_labels = zip(labels[label_mask], pred_midxs[label_mask])
pred_labels = sorted(pred_labels, key=lambda x: x[1])
pred_labels, _ = zip(*pred_labels)
return {
'fmi': fowlkes_mallows_score(true_labels, pred_labels),
'rand_index': adjusted_rand_score(true_labels, pred_labels),
'pred_labels': pred_labels,
'true_labels': true_labels,
'midxs': true_midxs
}
def kmeans():
from sklearn.datasets import load_digits
data, target = load_digits(return_X_y=True)
clustering = KMeans(n_clusters=10).fit(data)
results = [[0 for _ in range(10)] for __ in range(10)]
for i, val in enumerate(clustering.labels_):
results[val - 1][target[i]] += 1
print(
'Kmeans Clustering Confusion Matrix \n Cluster on X axis, Target on Y axis',
end='\n ')
[print("{:3d}".format(i), end=' ') for i in range(10)]
print('', end='\n ')
[print("_ _ ", end='') for i in range(10)]
print('')
for x in range(10):
print(x, end=' | ')
for y in range(10):
print("{:3d}".format(results[x][y]), end=' ')
print('')
# Calculating the Fowlkes-Mallows scores for Kmeans
print('For K-Means, Fowlkes-Mallows Score is: ' + str(
metrics.fowlkes_mallows_score(target[:500], clustering.labels_[:500]))
+ '\n' + '\n')
def evaluate(communities1, communities2, number_of_algorithms):
if not communities2:
return None
c1 = communities1.copy()
c2 = communities2.copy()
# Making sure they have the same size
c1_keys = list(c1.keys())
c2_keys = list(c2.keys())
for key in c1_keys:
if not key in c2:
del c1[key]
for key in c2_keys:
if not key in c1:
del c2[key]
od1 = collections.OrderedDict(sorted(c1.items()))
od2 = collections.OrderedDict(sorted(c2.items()))
evaluations = []
for i in range(0,number_of_algorithms):
cluster1_list = list(map(lambda x: x[i], od1.values()))
cluster2_list = list(map(lambda x: x[i], od2.values()))
ari = metrics.adjusted_rand_score(cluster1_list, cluster2_list)
ami = metrics.adjusted_mutual_info_score(cluster1_list, cluster2_list)
v_score = metrics.v_measure_score(cluster1_list, cluster2_list)
fmc = metrics.fowlkes_mallows_score(cluster1_list, cluster2_list)
evaluations.append((ari,ami,v_score,fmc))
return evaluations
def test_given_disjoint_graphs_embeddings_cluster(self):
"""
Embedding disjoint subgraphs should cluster correctly
"""
n_clusters = 5
graph_size = 150
G = nx.complete_graph(graph_size)
for i in range(1, n_clusters):
G = nx.disjoint_union(G, nx.complete_graph(graph_size))
labels = []
for i in range(n_clusters):
labels.append([i] * graph_size)
labels = sum(labels, [])
# Embed Graph and cluster around it
G = cg.csrgraph(G)
v = G.ggvec(n_components=4,
tol=0.1,
max_epoch=550,
learning_rate=0.05,
max_loss=1.,
verbose=False)
cluster_hat = cluster.AgglomerativeClustering(
n_clusters=n_clusters, affinity='cosine',
linkage='average').fit(v).labels_
r1 = metrics.adjusted_mutual_info_score(cluster_hat, labels)
r2 = metrics.adjusted_rand_score(cluster_hat, labels)
r3 = metrics.fowlkes_mallows_score(cluster_hat, labels)
self.assertGreaterEqual(r1, 0.65)
self.assertGreaterEqual(r2, 0.65)
self.assertGreaterEqual(r3, 0.65)
def test_mlops_fowlkes_mallows_score_apis():
mlops.init(ctx=None, mlops_mode=MLOpsMode.STAND_ALONE)
labels_pred = [1, 0, 1, 2, 3, 0]
labels_actual = [0, 1, 0, 1, 3, 1]
fms = metrics.fowlkes_mallows_score(labels_actual, labels_pred)
# first way
mlops.set_stat(ClusteringMetrics.FOWLKES_MALLOWS_SCORE, fms)
# second way
mlops.metrics.fowlkes_mallows_score(labels_true=labels_actual,
labels_pred=labels_pred)
# should throw error if not numeric number is provided
with pytest.raises(MLOpsStatisticsException):
mlops.set_stat(ClusteringMetrics.FOWLKES_MALLOWS_SCORE, [1, 2, 3])
# should throw error if labels predicted is different length than actuals
with pytest.raises(ValueError):
labels_pred_missing_values = [0, 0, 0, 1]
mlops.metrics.fowlkes_mallows_score(
labels_true=labels_actual, labels_pred=labels_pred_missing_values)
mlops.done()
def generate_cluster_labels(classes, K):
#def generate_cluster_labels(classes,K):
#Each cluster is defined by the majority represented cluster
class_labels = []
confusion_mtx = []
i = 0
for c in classes:
class_labels.append([])
confusion_mtx.append(np.zeros(K))
confusion_mtx[i][i] = -1
i += 1
#tally the votes for the highest number number of votes for a class.
for c in classes:
votes = np.zeros(K)
for i in range(0, len(c)):
votes[c[i].label] += 1
dc = np.where(votes == max(votes))
confusion_mtx[dc[0][0]] += votes
print(
'Cluster: %d has %d votes. This cluster has been labeled: C-%d ' %
(i, votes[dc[0]], dc[0]))
print('Cluster Confusion Matrix:')
confusion_mtx = np.array(confusion_mtx)
print(confusion_mtx)
#Generate the accuracy matrix to get scores
print('Fowlkes-Mallows Scores')
labels_true = (np.identity(10) * confusion_mtx).flatten()
labels_pred = confusion_mtx.flatten()
score = metrics.fowlkes_mallows_score(labels_true, labels_pred)
print('Accuracy Score:%f' % (score * 100))
def dimReducedClusters(x_data, y_data, x, n):
kmeans = KMeans(n_clusters=2)
kmeans.fit(x_data)
kmeans.predict(x_data)
kLabels = kmeans.labels_
if x == 0:
nData = pd.DataFrame(x_data)
nData['cluster'] = kLabels
nNetwork(nData, y_data, n + ': After K-Means')
em = GaussianMixture(n_components=2)
em.fit(x_data)
eLabels = em.predict(x_data)
if x == 0:
nData = pd.DataFrame(x_data)
nData['cluster'] = eLabels
nNetwork(nData, y_data, n + ': After EM')
a = silhouette_score(x_data, kLabels)
b = adjusted_rand_score(y_data, kLabels)
c = adjusted_mutual_info_score(y_data, kLabels)
d = homogeneity_score(y_data, kLabels)
e = completeness_score(y_data, kLabels)
f = fowlkes_mallows_score(y_data, kLabels)
g = em.bic(x_data)
h = em.aic(x_data)
return a, b, c, d, e, f, g, h
def evaluate(label, pred):
nmi = metrics.normalized_mutual_info_score(label, pred)
ari = metrics.adjusted_rand_score(label, pred)
f = metrics.fowlkes_mallows_score(label, pred)
pred_adjusted = get_y_preds(label, pred, len(set(label)))
acc = metrics.accuracy_score(pred_adjusted, label)
return nmi, ari, f, acc
def get_performance_metrics(y_pred_num, y_actual):
from sklearn import metrics
adjusted_rand_score = metrics.adjusted_rand_score(y_actual, y_pred_num)
adjusted_mutual_info_score = metrics.adjusted_mutual_info_score(
y_actual, y_pred_num)
normalized_mutual_info_score = metrics.normalized_mutual_info_score(
y_actual, y_pred_num)
homogeneity_score = metrics.homogeneity_score(y_actual, y_pred_num)
completeness_score = metrics.completeness_score(y_actual, y_pred_num)
v_measure_score = metrics.v_measure_score(y_actual, y_pred_num)
fowlkes_mallows_score = metrics.fowlkes_mallows_score(y_actual, y_pred_num)
metric_names = [
"adjusted_rand_score", "adjusted_mutual_info_score",
"normalized_mutual_info_score", "homogeneity_score",
"completeness_score", "v_measure_score", "fowlkes_mallows_score"
]
metric_values = [
adjusted_rand_score, adjusted_mutual_info_score,
normalized_mutual_info_score, homogeneity_score, completeness_score,
v_measure_score, fowlkes_mallows_score
]
print "Metrics"
print "\t".join(metric_names)
print('\t'.join([str(x) for x in metric_values]))
return metric_names, metric_values
def kmeans_clustering(X_train, y_train, X_test, y_test, genre_list):
scalar = StandardScaler()
scalar.fit(X_train, y_train)
new_data = scalar.transform(X_train)
kmeans = KMeans(init='k-means++', n_init=10, n_clusters=4, max_iter=300)
rVal = kmeans.fit(X_train, y_train)
kmeans_predictions = kmeans.predict(X_test)
print("the randomized score is : ",
metrics.adjusted_rand_score(y_test, kmeans_predictions))
print("the normalized mutual info score is : ",
metrics.normalized_mutual_info_score(y_test, kmeans_predictions))
print("the mutual info score is : ",
metrics.mutual_info_score(y_test, kmeans_predictions))
print(
"the homogenity, completeness and v measure score is : ",
metrics.homogeneity_completeness_v_measure(y_test, kmeans_predictions))
print("the fowlkes mallows score is : ",
metrics.fowlkes_mallows_score(y_test, kmeans_predictions))
labels = kmeans.labels_
print(
"the silhouette score is :",
metrics.silhouette_score(X_test,
kmeans_predictions,
metric='euclidean'))
print(kmeans_predictions)
print(y_test)
centers = rVal.cluster_centers_
distances = pairwise_distances(new_data, centers, metric='euclidean')
clusters = np.argmin(distances, axis=1)
print(len(clusters))
plotSamples = PCA(n_components=2).fit_transform(new_data)
plotClusters(plotSamples, clusters, kmeans)
joblib.dump(kmeans, 'saved_models/model_kmeans.pkl')
def evaluate(self, X, y=None):
"""
Depending on the clustering task, evaluate the predictor and
store its performance.
Args:
X (array-like): Test samples
y (array-like, optional): True labels
Returns:
eval_dict: Dictionary containing evaluations
"""
# Start by predicting clusters
y_pred = self.predict(X)
# Result dict
eval_dict = dict()
# Check if ground truth is available:
if y is not None:
eval_dict['rand_index'] = m.adjusted_rand_score(y, y_pred)
eval_dict['mutual_information'] = m.adjusted_mutual_info_score(
y, y_pred)
eval_dict['fowlkes_mallows_score'] = m.fowlkes_mallows_score(
y, y_pred)
# Compute label-agnostic metrics:
eval_dict['silhouette_score'] = m.silhouette_score(X, y_pred)
eval_dict['calinski_harabaz_score'] = m.calinski_harabaz_score(
X, y_pred)
return eval_dict
def agglo_clustering(ctx):
"""
Agglomerative clustering function to be run on dataset.
"""
clusters = 10
linkage = 'ward'
print("loading data...")
x_train, _, y_train = load_data(ctx.obj["data_folder"],
shuffle_seed=ctx.obj["seed"])
print(
"Running agglomerative clustering where linkage = {} and n_clusters = {}"
.format(linkage, clusters))
model = AgglomerativeClustering(linkage=linkage, n_clusters=clusters)
labels_predicted = model.fit_predict(x_train)
y_train = column_or_1d(y_train)
score = metrics.adjusted_rand_score(y_train, labels_predicted)
print(f"Accuracy: {score}.")
score = metrics.homogeneity_score(y_train, labels_predicted)
print(f"Homogeneity Score: {score}.")
score = metrics.completeness_score(y_train, labels_predicted)
print(f"Completeness Score: {score}.")
score = metrics.v_measure_score(y_train, labels_predicted)
print(f"V Measure Score: {score}.")
score = metrics.fowlkes_mallows_score(y_train, labels_predicted)
print(f"Fowlkes Mallows Score: {score}.")
def compareAB(A, B, X):
#measures the similarity of the two assignments, ignoring permutations and with chance normalization
ars = metrics.adjusted_rand_score(A, B)
ars_str = '%17.3f' % ars
# each cluster contains only members of a single class
hs = homogeneity_score(A, B)
hs_str = '%17.3f' % hs
#all members of a given class are assigned to the same cluster
cs = completeness_score(A, B)
cs_str = '%17.3f' % cs
vms = metrics.v_measure_score(A, B)
vms_str = '%17.3f' % vms
# geometric mean of the pairwise precision and recall
fowlkes_mallows_score = metrics.fowlkes_mallows_score(A, B)
fms_str = '%17.3f' % fowlkes_mallows_score
sc = metrics.silhouette_score(X, B, metric='euclidean')
sc_str = '%17.3f' % sc
my_str = ars_str + "&" + hs_str + "&" + cs_str + "&" + vms_str + "&" + fms_str + "&" + sc_str
return my_str
def em_clustering(ctx):
"""
Gaussian Mixture function to be run on dataset.
"""
covariance_type = 'spherical'
n_components = 10
print("loading data...")
x_train, _, y_train = load_data(ctx.obj["data_folder"],
shuffle_seed=ctx.obj["seed"])
print("Running Gaussian Mixture...")
model = GaussianMixture(n_components=n_components,
covariance_type=covariance_type,
verbose=2)
labels_predicted = model.fit_predict(x_train)
y_train = column_or_1d(y_train)
score = metrics.adjusted_rand_score(y_train, labels_predicted)
print(f"Accuracy: {score}.")
score = metrics.homogeneity_score(y_train, labels_predicted)
print(f"Homogeneity Score: {score}.")
score = metrics.completeness_score(y_train, labels_predicted)
print(f"Completeness Score: {score}.")
score = metrics.v_measure_score(y_train, labels_predicted)
print(f"V Measure Score: {score}.")
score = metrics.fowlkes_mallows_score(y_train, labels_predicted)
print(f"Fowlkes Mallows Score: {score}.")
def bench_kmeans(estimator, name, k, x_train, y_train, x_test, y_test):
bench_kmeans_format = '% 9s %.2i %.3f %.2fs %.3f %.2fs %i %.3f %.3f %.3f %.3f %.3f %.3f' #%.3f %.3f'
# Train.
start = time.time()
estimator.fit(x_train)
train_time = time.time() - start
# Test.
start = time.time()
predicted = estimator.predict(x_test)
test_time = time.time() - start
#print('% 9s' % 'init k'
# ' time acc inertia h**o compl v-meas ARI AMI FMI')
results = (name, k, train_time,
metrics.accuracy_score(y_train, estimator.labels_), test_time,
metrics.accuracy_score(y_test, predicted), estimator.inertia_,
metrics.homogeneity_score(y_train, estimator.labels_),
metrics.completeness_score(y_train, estimator.labels_),
metrics.v_measure_score(y_train, estimator.labels_),
metrics.adjusted_rand_score(y_train, estimator.labels_),
metrics.adjusted_mutual_info_score(y_train, estimator.labels_),
metrics.fowlkes_mallows_score(y_train, estimator.labels_))
#metrics.calinski_harabaz_score(x_train, estimator.labels_),
#metrics.silhouette_score(x_train, estimator.labels_, metric='euclidean')))
#print(bench_kmeans_format % (results))
# TODO why does silhouette score throw an error about label size?
# TODO all x_train, estimator.labels_, and y_train all have same size (and its not 1)
# TODO should we pass explicit sample size to silhouette score? len(x_train)
return list(results)
def evaluate_clustering_performance(clusters, labels):
set_of_dimensionality = set()
for cluster in clusters:
set_of_dimensionality.add(frozenset(cluster.dimensions))
# Evaluating performance in all dimensionality
for dim in set_of_dimensionality:
print("\nEvaluating clusters in dimension: ", list(dim))
# Finding clusters with same dimensions
clusters_in_dim = []
for c in clusters:
if c.dimensions == dim:
clusters_in_dim.append(c)
clustering_labels = np.zeros(np.shape(labels))
for i, c in enumerate(clusters_in_dim):
clustering_labels[list(c.data_point_ids)] = i + 1
print("Number of clusters: ", len(clusters_in_dim))
print("Adjusted Rand index: ",
metrics.adjusted_rand_score(labels, clustering_labels))
print("Mutual Information: ",
metrics.adjusted_mutual_info_score(labels, clustering_labels))
print(
"Homogeneity, completeness, V-measure: ",
metrics.homogeneity_completeness_v_measure(labels,
clustering_labels))
print("Fowlkes-Mallows: ",
metrics.fowlkes_mallows_score(labels, clustering_labels))
def fowlkes_mallows_index(df: pd.DataFrame,
column_true: str = "GroundTruth",
column_pred: str = "label") -> float:
"""
Measure the similarity of two clusterings of a set of points.
The Fowlkes-Mallows index (FMI) is defined as the geometric mean between of the precision and recall:
FMI = TP / sqrt((TP + FP) * (TP + FN))
TThe score ranges from 0 to 1. A high value indicates a good similarity between two clusters.
Parameters
----------
df : DataFrame
Input DataFrame with at least two columns: True and Predicted ones
column_true : str, default if 'GroundTruth'
The title for column with True labels
column_pred : str, default is 'label'
The title for column with Predicted labels
Returns
-------
The FMI returns a value of from 0.0 to 1.0: Perfect labeling is scored 1.0,
while Bad (e.g. independent labelings) have zero scores:
"""
str_labels = list(df[column_true].unique())
real_labels = dict(zip(str_labels, range(len(str_labels))))
return fowlkes_mallows_score(
df[column_true].apply(lambda x: real_labels[x]).values,
df[column_pred].values)
def record_sampling_clustermultimeasure_vs_opnum(wn, samplemth):
cora_true = models.CoraPerformanceLog.objects.filter(
explorationMethod='groundtruth').order_by('cora_id')
aa_true = [item.clusterid for item in cora_true]
cluster_membership = models.CoraPerformanceLog.objects.filter(
explorationMethod=samplemth, workerOperationNum=wn).order_by('cora_id')
bb_pred = [item.clusterid for item in cluster_membership]
nmi = metrics.normalized_mutual_info_score(aa_true, bb_pred)
ari = metrics.adjusted_rand_score(aa_true, bb_pred)
ami = metrics.adjusted_mutual_info_score(aa_true, bb_pred)
homogeneity = metrics.homogeneity_score(aa_true, bb_pred)
completeness = metrics.completeness_score(aa_true, bb_pred)
v_measure = metrics.v_measure_score(aa_true, bb_pred)
fmi = metrics.fowlkes_mallows_score(aa_true, bb_pred)
dict = {
'RecordSamplingMethod': samplemth.name,
'workerOperationNum': wn,
'nmi': nmi,
'ari': ari,
'ami': ami,
'homogeneity': homogeneity,
'completeness': completeness,
'v_measure': v_measure,
'fmi': fmi
}
for k, v in dict.items():
print(k, ' ', v)
return dict
def clusterMeasureSet(aa_true, bb_pred, RecordSamplingMethod,
workerOperationNum):
nmi = metrics.normalized_mutual_info_score(aa_true, bb_pred)
ari = metrics.adjusted_rand_score(aa_true, bb_pred)
ami = metrics.adjusted_mutual_info_score(aa_true, bb_pred)
homogeneity = metrics.homogeneity_score(aa_true, bb_pred)
completeness = metrics.completeness_score(aa_true, bb_pred)
v_measure = metrics.v_measure_score(aa_true, bb_pred)
fmi = metrics.fowlkes_mallows_score(aa_true, bb_pred)
dict = {
'RecordSamplingMethod': RecordSamplingMethod,
'workerOperationNum': workerOperationNum,
'nmi': nmi,
'ari': ari,
'ami': ami,
'homogeneity': homogeneity,
'completeness': completeness,
'v_measure': v_measure,
'fmi': fmi
}
for k, v in dict.items():
print(k, ' ', v)
for k, v in dict.items():
print(v)
return dict
def prin_clustering(test_rep, test_label, NUM_OF_CLASS):
# 聚类
km = KMeans(n_clusters=NUM_OF_CLASS)
#km.fit_transform(test_rep)
cls_rs = km.fit_predict(test_rep)
# ARI
ari = metrics.adjusted_rand_score(test_label, cls_rs)
# AMI
ami = metrics.adjusted_mutual_info_score(test_label, cls_rs)
# H,C,V
H, C, V = metrics.homogeneity_completeness_v_measure(test_label, cls_rs)
# FMI
fmi = metrics.fowlkes_mallows_score(test_label, cls_rs)
# s
# s = metrics.silhouette_score(test_label, cls_rs)
# DBI
# dbi = metrics.davies_bouldin_score(test_label, cls_rs)
# nmi
nmi = metrics.normalized_mutual_info_score(test_label, cls_rs)
d = dict()
d['ari'] = ari
d['ami'] = ami
d['nmi'] = nmi
d['fmi'] = fmi
d['H'] = H
d['C'] = C
d['V'] = V
print('ARI:%.4f,AMI:%.4f,HCV:%.4f %.4f %.4f FMI:%.4f NMI:%.4f' %
(ari, ami, H, C, V, fmi, nmi))
return d
def get_fowlkes_mallows_score(standard_file, prediction_file, isjson=False, isint=False):
if isjson:
standard_data = AbstractionUtility.read_json(standard_file)
standard_labels = standard_data.values()
else:
standard_labels = ExternalEvaluation.get_evaluated(standard_file)
prediction_labels = ExternalEvaluation.get_evaluated(prediction_file, isint=isint)
fowlkes_mallows_score = metrics.fowlkes_mallows_score(standard_labels, prediction_labels)
return fowlkes_mallows_score
def compute_stability_score(estimator, X, y=None, n_draws=10, p_samples=0.8, random_state=None):
if estimator.n_clusters == 1:
return 0.0
# Generate data
rng = np.random.RandomState(random_state)
n_samples, _ = X.shape
data = rng.uniform(size=(n_draws, 2 * n_samples)) < p_samples
score = np.empty(n_draws)
for k, d in enumerate(data):
p1, p2 = np.split(d, 2)
labels1 = estimator.fit_predict(X[p1])
labels2 = estimator.fit_predict(X[p2])
score[k] = fowlkes_mallows_score(labels1[p2[p1]], labels2[p1[p2]])
return score.mean()
def get_metrics():
dict_lable = {'eco': 0, 'env': 1, 'sports': 2, 'other': 3}
data = pd.read_table('id2class.txt', header=None, delim_whitespace=True)
data2 = pd.read_table('result.txt', header=None, delim_whitespace=True)
list_true = []
list_pred = []
for i in range(len(data)):
data.iloc[i, 1] = dict_lable[data.iloc[i, 1]]
list_true.append(data.iloc[i, 1])
list_pred.append(data2.iloc[i, 1])
# 文档链接 http://scikit-learn.org/stable/modules/clustering.html#clustering-performance-evaluation
# 2.3.9.1 Adjusted Rand index （RI）
# 2.3.9.2. Mutual Information based scores（NMI）
# 2.3.9.4. Fowlkes-Mallows scores（FMI）
# 章节号为文档里面的章节号
print (metrics.adjusted_rand_score(list_true, list_pred)) # RI指数，越接近1越好
print (metrics.adjusted_mutual_info_score(list_true, list_pred)) # NMI指数，越接近1越好
print (metrics.fowlkes_mallows_score(list_true, list_pred)) # FMI指数，越接近1越好
def compare_clusters(prog, argv):
parser = argparse.ArgumentParser(prog=prog,
description='Compare Equivalence Classes')
parser.add_argument('classes', metavar='eqclass', type=str, nargs=2,
help='Ground Truth / Prediction')
parser.add_argument('-ar', action='store_true', default=False,
help='Adjusted rand score')
parser.add_argument('-mi', action='store_true', default=False,
help='Mutual info score')
parser.add_argument('-ami', action='store_true', default=False,
help='Adjusted mutual info score')
parser.add_argument('-nmi', action='store_true', default=False,
help='Normalised mutual info score')
parser.add_argument('-pur', action='store_true', default=False,
help='Purity')
parser.add_argument('-pr', action='store_true', default=False,
help='Classic Precision/Recall')
parser.add_argument('-fm', action='store_true', default=False,
help='Fowlkes-Mallow score')
parser.add_argument('-remove-identical', action='store_true', default=False,
help='Remove identical clusters before comparing')
parser.add_argument('-f', type=str, help='Write results to filename')
parser.add_argument('-test', action='store_true', default=False,
help='run tests')
args = parser.parse_args(argv)
# These are the converted example from:
# https://nlp.stanford.edu/IR-book/html/htmledition/evaluation-of-clustering-1.html
if args.test:
ground_truth = Cluster()
prediction = Cluster()
prediction.insert(0,1,2,3,4,5)
prediction.insert(6,7,8,9,10,11)
prediction.insert(12, 13, 14, 15, 16)
ground_truth.insert(0,2,3,4,5,6,12,14)
ground_truth.insert(1, 7, 8, 9, 11)
ground_truth.insert(10,13,15,16)
else:
ground_truth = Cluster.from_file(args.classes[0], must_exist=True)
prediction = Cluster.from_file(args.classes[1], must_exist=True)
# remove identical clusters, if desired
if args.remove_identical:
identical = list()
for cluster in ground_truth:
cand = prediction.get_cluster(list(cluster)[0])
if not cand:
continue
elif cand == cluster:
identical.append(cand)
log.info('Removing %d identical clusters (%d elements)...' %
(len(identical), sum([len(x) for x in identical])))
for cluster in identical:
for element in cluster:
ground_truth.remove_key(element)
prediction.remove_key(element)
ground_truth.optimize()
prediction.optimize()
if (args.pr):
prec_rec(ground_truth, prediction)
# intermix all keys
ground_truth_keys = ground_truth.get_keys()
prediction_keys = prediction.get_keys()
missing = ground_truth_keys - prediction_keys
log.info('%d keys missing in prediction' % len(missing))
for key in missing:
prediction.insert_single(key)
missing = prediction_keys - ground_truth_keys
log.info('%d keys missing in ground truth' % len(missing))
for key in missing:
ground_truth.insert_single(key)
gt = list(sorted(ground_truth.lookup.items()))
t = list(sorted(prediction.lookup.items()))
gt = [x[1] for x in gt]
t = [x[1] for x in t]
log.info('Number of equiv classes: %d' % len(ground_truth))
h**o, comp, vm = metrics.homogeneity_completeness_v_measure(gt, t)
log.info("Homogeneity: %0.3f" % h**o)
log.info("Completeness: %0.3f" % comp)
log.info("V-measure: %0.3f" % vm)
if args.ar:
ar = metrics.adjusted_rand_score(gt, t)
log.info("Adjusted rand score: %0.3f" % ar)
if args.mi:
mi = metrics.mutual_info_score(gt, t)
log.info("Mutual info score: %0.3f" % mi)
if args.ami:
ami = metrics.adjusted_mutual_info_score(gt, t)
log.info("Adjusted mutual info score: %0.3f" % ami)
if args.nmi:
nmi = metrics.normalized_mutual_info_score(gt, t)
log.info("Normalised mutual info score: %0.3f" % nmi)
if args.pur:
elements = len(gt)
hits = 0
for w in ground_truth:
this = 0
for element in w:
tmp = prediction[element]
foo = len(w & tmp)
if foo > this:
this = foo
hits += this
purity = hits / elements
log.info('Purity: %0.3f' % purity)
if args.fm:
fm = metrics.fowlkes_mallows_score(gt, t)
log.info("Fowlkes-Mallows score: %0.3f" % fm)
if args.f:
with open(args.f, 'w') as f:
f.write("h**o: %0.3f\n" % h**o)
f.write("comp: %0.3f\n" % comp)
f.write("vm: %0.3f\n" % vm)
if args.ar:
f.write("ar: %0.3f\n" % ar)
if args.mi:
f.write("mi: %0.3f\n" % mi)
if args.nmi:
f.write("nmi: %0.3f\n" % nmi)
if args.ami:
f.write("ami: %0.3f\n" % ami)
if args.pur:
f.write("pur: %0.3f\n" % purity)
if args.fm:
f.write("fm: %0.3f\n" % fm)
return 0
