def cluster_changepoints_level1(self):
print "Level1 : Clustering changepoints in Z(t)"
if constants.REMOTE == 1:
if self.fit_DPGMM:
print "DPGMM L1 - start"
# Previously, when L0 was GMM, alpha = 0.4
print "L1 ", str(
len(self.list_of_cp) /
constants.DPGMM_DIVISOR_L1), " ALPHA ", 10
dpgmm = mixture.DPGMM(n_components=int(
len(self.list_of_cp) / 6),
covariance_type='diag',
n_iter=1000,
alpha=10,
thresh=1e-7)
print "DPGMM L1 - end"
if self.fit_GMM:
gmm = mixture.GMM(n_components=self.n_components_L1,
covariance_type='full',
n_iter=1000,
thresh=5e-5)
print "GMM L1 - end"
elif constants.REMOTE == 2:
gmm = mixture.GMM(n_components=self.n_components_L1,
covariance_type='full')
else:
gmm = mixture.GMM(n_components=self.n_components_L1,
covariance_type='full')
if self.fit_GMM:
gmm.fit(self.change_pts_Z)
Y_gmm = gmm.predict(self.change_pts_Z)
Y = Y_gmm
if self.fit_DPGMM:
Y = []
i = 0
while True:
print "In DPGMM Fit loop"
dpgmm.fit(self.change_pts_Z)
Y = dpgmm.predict(self.change_pts_Z)
if len(set(Y)) > 1:
break
i += 1
if i > 100:
break
self.save_cluster_metrics(self.change_pts_Z, Y, 'level1')
for i in range(len(Y)):
label = constants.alphabet_map[Y[i] + 1]
self.map_cp2level1[i] = label
utils.dict_insert_list(label, i, self.map_level12cp)
self.generate_l2_cluster_matrices()
def cluster_changepoints(self):
"""
Clusters changepoints specified in self.list_of_cp.
"""
print "Clustering changepoints..."
print "L1 ", str(len(self.list_of_cp) / constants.DPGMM_DIVISOR_L1), " ALPHA: ", self.ALPHA_L1
if constants.REMOTE == 1:
if self.fit_DPGMM:
dpgmm = mixture.DPGMM(
n_components=int(len(self.list_of_cp) / constants.DPGMM_DIVISOR_L1),
covariance_type="diag",
n_iter=10000,
alpha=self.ALPHA_L1,
thresh=1e-4,
)
if self.fit_GMM:
gmm = mixture.GMM(n_components=self.n_components_L1, covariance_type="full", n_iter=5000, thresh=0.01)
elif constants.REMOTE == 2:
gmm = mixture.GMM(n_components=self.n_components_L1, covariance_type="full", thresh=0.01)
else:
gmm = mixture.GMM(n_components=self.n_components_L1, covariance_type="full")
if self.fit_GMM:
gmm.fit(self.changepoints)
predictions_gmm = gmm.predict(self.changepoints)
print "L1: Clusters in GMM", len(set(predictions_gmm))
predictions = predictions_gmm
if self.fit_DPGMM:
predictions = []
while True:
print "Inside loop"
dpgmm.fit(self.changepoints)
predictions = dpgmm.predict(self.changepoints)
if len(set(predictions)) > 1:
break
print "L1: Clusters in DP-GMM", len(set(predictions))
for i in range(len(predictions)):
label = constants.alphabet_map[predictions[i] + 1]
self.map_cp2cluster[i] = label
utils.dict_insert_list(label, i, self.map_level1_cp)
demonstration = self.map_cp2demonstrations[i]
frm = self.map_cp2frm[i]
try:
surgeme = self.map_frm2surgeme[demonstration][frm]
except KeyError as e:
print e
sys.exit()
utils.print_and_write(("%3d %s %s %3d %3d\n" % (i, label, demonstration, frm, surgeme)), self.log)
def cluster_changepoints_level1(self):
print "Level1 : Clustering changepoints in Z(t)"
if constants.REMOTE == 1:
if self.fit_DPGMM:
print "DPGMM L1 - start"
# Previously, when L0 was GMM, alpha = 0.4
print "L1 ", str(len(self.list_of_cp) / constants.DPGMM_DIVISOR_L1), " ALPHA ", 10
dpgmm = mixture.DPGMM(
n_components=int(len(self.list_of_cp) / 6),
covariance_type="diag",
n_iter=1000,
alpha=10,
thresh=1e-7,
)
print "DPGMM L1 - end"
if self.fit_GMM:
gmm = mixture.GMM(n_components=self.n_components_L1, covariance_type="full", n_iter=1000, thresh=5e-5)
print "GMM L1 - end"
elif constants.REMOTE == 2:
gmm = mixture.GMM(n_components=self.n_components_L1, covariance_type="full")
else:
gmm = mixture.GMM(n_components=self.n_components_L1, covariance_type="full")
if self.fit_GMM:
gmm.fit(self.change_pts_Z)
Y_gmm = gmm.predict(self.change_pts_Z)
Y = Y_gmm
if self.fit_DPGMM:
Y = []
i = 0
while True:
print "In DPGMM Fit loop"
dpgmm.fit(self.change_pts_Z)
Y = dpgmm.predict(self.change_pts_Z)
if len(set(Y)) > 1:
break
i += 1
if i > 100:
break
self.save_cluster_metrics(self.change_pts_Z, Y, "level1")
for i in range(len(Y)):
label = constants.alphabet_map[Y[i] + 1]
self.map_cp2level1[i] = label
utils.dict_insert_list(label, i, self.map_level12cp)
self.generate_l2_cluster_matrices()
def cluster_changepoints(self):
"""
Clusters changepoints specified in self.list_of_cp.
"""
print "Clustering changepoints..."
print "L1 ", str(len(self.list_of_cp)/constants.DPGMM_DIVISOR_L1)," ALPHA: ", self.ALPHA_L1
if constants.REMOTE == 1:
if self.fit_DPGMM:
dpgmm = mixture.DPGMM(n_components = int(len(self.list_of_cp)/constants.DPGMM_DIVISOR_L1), covariance_type='diag', n_iter = 10000, alpha = self.ALPHA_L1, thresh= 1e-4)
if self.fit_GMM:
gmm = mixture.GMM(n_components = self.n_components_L1, covariance_type='full', n_iter=5000, thresh = 0.01)
elif constants.REMOTE == 2:
gmm = mixture.GMM(n_components = self.n_components_L1, covariance_type='full', thresh = 0.01)
else:
gmm = mixture.GMM(n_components = self.n_components_L1, covariance_type='full')
if self.fit_GMM:
gmm.fit(self.changepoints)
predictions_gmm = gmm.predict(self.changepoints)
print "L1: Clusters in GMM",len(set(predictions_gmm))
predictions = predictions_gmm
if self.fit_DPGMM:
predictions = []
while True:
print "Inside loop"
dpgmm.fit(self.changepoints)
predictions = dpgmm.predict(self.changepoints)
if len(set(predictions)) > 1:
break
print "L1: Clusters in DP-GMM", len(set(predictions))
for i in range(len(predictions)):
label = constants.alphabet_map[predictions[i] + 1]
self.map_cp2cluster[i] = label
utils.dict_insert_list(label, i, self.map_level1_cp)
demonstration = self.map_cp2demonstrations[i]
frm = self.map_cp2frm[i]
try:
surgeme = self.map_frm2surgeme[demonstration][frm]
except KeyError as e:
print e
sys.exit()
utils.print_and_write(("%3d %s %s %3d %3d\n" % (i, label, demonstration, frm, surgeme)), self.log)
def cluster_changepoints_level1(self): # print "Level1 : Clustering changepoints in Z(t)" gmm = mixture.GMM(n_components = 10, covariance_type='full') gmm.fit(self.change_pts_Z) Y = gmm.predict(self.change_pts_Z) self.save_cluster_metrics(self.change_pts_Z, Y, gmm.means_, 'level1', gmm) for i in range(len(Y)): label = constants.alphabet_map[Y[i] + 1] self.map_cp_level1[i] = label utils.dict_insert_list(label, i, self.map_level1_cp) self.generate_l2_cluster_matrices()
def cluster_changepoints_level1(self): # print "Level1 : Clustering changepoints in Z(t)" gmm = mixture.GMM(n_components = 10, covariance_type='full') gmm.fit(self.change_pts_Z) Y = gmm.predict(self.change_pts_Z) self.save_cluster_metrics(self.change_pts_Z, Y, gmm.means_, 'level1', gmm) for i in range(len(Y)): label = constants.alphabet_map[Y[i] + 1] self.map_cp_level1[i] = label utils.dict_insert_list(label, i, self.map_level1_cp) self.generate_l2_cluster_matrices()
def get_time_clusters(data, T_COMPONENTS):
# k-fold validation (Leave one out)
numDemos = len(data.keys()) + 1
sizeTestSet = numDemos - 1
list_of_frms = []
all_frms = []
for key in data.keys():
elem = data[key]
list_of_frms.append(elem)
all_frms += elem
N_COMPONENTS = min(T_COMPONENTS, len(all_frms))
time_cluster = mixture.GMM(n_components=N_COMPONENTS,
covariance_type='full',
n_iter=50000,
thresh=5e-7)
X = np.array(all_frms)
X = X.reshape(len(all_frms), 1)
time_cluster.fit(X)
Y = time_cluster.predict(X)
means = time_cluster.means_
covars = time_cluster.covars_
#dpgmm = mixture.DPGMM(n_components = numDemos * 3, covariance_type='diag', n_iter = 10000, alpha = 1000, thresh= 1e-10)
#dpgmm.fit(X)
#Y = dpgmm.predict(X)
#means = dpgmm.means_
list_of_elem = []
list_of_means = []
for i in range(len(Y)):
list_of_elem.append(
(Y[i], X[i], means[Y[i]][0], np.sqrt(covars[Y[i]][0][0])))
list_of_elem = sorted(list_of_elem, key=lambda x: x[1][0])
dict_time_clusters = {}
for elem in list_of_elem:
utils.dict_insert_list(elem[0], elem[1], dict_time_clusters)
list_time_clusters = []
list_time_clusters_noPrune = []
for cluster in dict_time_clusters.keys():
# get all frames in this cluster
cluster_frames = dict_time_clusters[cluster]
setClusterFrames = set([elem[0] for elem in cluster_frames])
# test if frames in cluster are representative of the test set
rep = []
for id in range(sizeTestSet):
elemSet = set(list_of_frms[id])
commonElem = elemSet.intersection(setClusterFrames)
id_in_cluster = 1. if len(commonElem) > 0 else 0.
rep.append(id_in_cluster)
pruneCluster = True if sum(
rep) / sizeTestSet < constants.PRUNING_FACTOR_T else False
min_frm = min(cluster_frames)
max_frm = max(cluster_frames)
mean = means[cluster][0]
list_of_means.append(int(mean))
std = np.sqrt(covars[cluster][0][0])
leftFrame = max(min_frm[0], mean - std)
rightFrame = min(max_frm[0], mean + std)
list_time_clusters_noPrune.append((leftFrame, rightFrame))
# keep for plotting is pruneFlag = 0
if not (pruneCluster):
# list_time_clusters.append((min_frm[0], max_frm[0]))
list_time_clusters.append((leftFrame, rightFrame))
print "Number of Clusters pruned in Time Clustering: ", len(
list_time_clusters_noPrune) - len(list_time_clusters)
labels_automatic, colors_automatic = setup_automatic_labels_2(
list_time_clusters, "k")
labels_automatic_0, colors_automatic_0 = setup_automatic_labels(
list_of_frms[0], "k")
return labels_automatic, colors_automatic, labels_automatic_0, colors_automatic_0, list_of_means, list_of_frms
def post_evaluation_multimodal(metrics, file, filename, list_of_demonstrations,
feat_fname):
mutual_information_1 = []
normalized_mutual_information_1 = []
adjusted_mutual_information_1 = []
homogeneity_1 = []
precision_1_micro = []
recall_1_micro = []
precision_1_macro = []
recall_1_macro = []
precision_1_weighted = []
recall_1_weighted = []
mutual_information_2 = []
normalized_mutual_information_2 = []
adjusted_mutual_information_2 = []
homogeneity_2 = []
precision_2_micro = []
recall_2_micro = []
precision_2_macro = []
recall_2_macro = []
precision_2_weighted = []
recall_2_weighted = []
silhoutte_level_1_global = []
silhoutte_level_1_weighted = []
dunn1_level_1 = []
dunn2_level_1 = []
dunn3_level_1 = []
silhoutte_level_2_global = []
silhoutte_level_2_weighted = []
dunn1_level_2 = []
dunn2_level_2 = []
dunn3_level_2 = []
list_of_frms = {}
for elem in metrics:
precision_1_micro.append(elem[0]["precision_micro"])
precision_2_micro.append(elem[1]["precision_micro"])
precision_1_macro.append(elem[0]["precision_macro"])
precision_2_macro.append(elem[1]["precision_macro"])
precision_1_weighted.append(elem[0]["precision_weighted"])
precision_2_weighted.append(elem[1]["precision_weighted"])
recall_1_micro.append(elem[0]["recall_micro"])
recall_2_micro.append(elem[1]["recall_micro"])
recall_1_macro.append(elem[0]["recall_macro"])
recall_2_macro.append(elem[1]["recall_macro"])
recall_1_weighted.append(elem[0]["recall_weighted"])
recall_2_weighted.append(elem[1]["recall_weighted"])
mutual_information_1.append(elem[0]["mutual_info_score"])
mutual_information_2.append(elem[1]["mutual_info_score"])
normalized_mutual_information_1.append(
elem[0]["normalized_mutual_info_score"])
normalized_mutual_information_2.append(
elem[1]["normalized_mutual_info_score"])
adjusted_mutual_information_1.append(
elem[0]["adjusted_mutual_info_score"])
adjusted_mutual_information_2.append(
elem[1]["adjusted_mutual_info_score"])
homogeneity_1.append(elem[0]["homogeneity_score"])
homogeneity_2.append(elem[1]["homogeneity_score"])
silhoutte_level_1_global.append(elem[2]["level1"])
dunn1_level_1.append(elem[3]["level1"])
dunn2_level_1.append(elem[4]["level1"])
dunn3_level_1.append(elem[5]["level1"])
silhoutte_level_2_global += elem[6]
dunn1_level_2 += elem[7]
dunn2_level_2 += elem[8]
dunn3_level_2 += elem[9]
viz = elem[10]
for demonstration in viz.keys():
utils.dict_insert_list(demonstration, viz[demonstration],
list_of_frms)
silhoutte_level_1_weighted.append(elem[11]["level1"])
silhoutte_level_2_weighted += elem[12]
utils.print_and_write_2("precision_1_micro", np.mean(precision_1_micro),
np.std(precision_1_micro), file)
utils.print_and_write_2("precision_2_micro", np.mean(precision_2_micro),
np.std(precision_2_micro), file)
utils.print_and_write_2("precision_1_macro", np.mean(precision_1_macro),
np.std(precision_1_macro), file)
utils.print_and_write_2("precision_2_macro", np.mean(precision_2_macro),
np.std(precision_2_macro), file)
utils.print_and_write_2("precision_1_weighted",
np.mean(precision_1_weighted),
np.std(precision_1_weighted), file)
utils.print_and_write_2("precision_2_weighted",
np.mean(precision_2_weighted),
np.std(precision_2_weighted), file)
utils.print_and_write_2("recall_1_micro", np.mean(recall_1_micro),
np.std(recall_1_micro), file)
utils.print_and_write_2("recall_2_micro", np.mean(recall_2_micro),
np.std(recall_2_micro), file)
utils.print_and_write_2("recall_1_macro", np.mean(recall_1_macro),
np.std(recall_1_macro), file)
utils.print_and_write_2("recall_2_macro", np.mean(recall_2_macro),
np.std(recall_2_macro), file)
utils.print_and_write_2("recall_1_weighted", np.mean(recall_1_weighted),
np.std(recall_1_weighted), file)
utils.print_and_write_2("recall_2_weighted", np.mean(recall_2_weighted),
np.std(recall_2_weighted), file)
utils.print_and_write_2("mutual_info_1", np.mean(mutual_information_1),
np.std(mutual_information_1), file)
utils.print_and_write_2("mutual_info_2", np.mean(mutual_information_2),
np.std(mutual_information_2), file)
utils.print_and_write_2("normalized_mutual_info_1",
np.mean(normalized_mutual_information_1),
np.std(normalized_mutual_information_1), file)
utils.print_and_write_2("normalized_mutual_info_2",
np.mean(normalized_mutual_information_2),
np.std(normalized_mutual_information_2), file)
utils.print_and_write_2("adjusted_mutual_info_1",
np.mean(adjusted_mutual_information_1),
np.std(adjusted_mutual_information_1), file)
utils.print_and_write_2("adjusted_mutual_info_2",
np.mean(adjusted_mutual_information_2),
np.std(adjusted_mutual_information_2), file)
utils.print_and_write_2("homogeneity_1", np.mean(homogeneity_1),
np.std(homogeneity_1), file)
utils.print_and_write_2("homogeneity_2", np.mean(homogeneity_2),
np.std(homogeneity_2), file)
utils.print_and_write_2("silhoutte_level_1_global",
np.mean(silhoutte_level_1_global),
np.std(silhoutte_level_1_global), file)
utils.print_and_write_2("silhoutte_level_2_global",
np.mean(silhoutte_level_2_global),
np.std(silhoutte_level_2_global), file)
utils.print_and_write_2("silhoutte_level_1_weighted",
np.mean(silhoutte_level_1_weighted),
np.std(silhoutte_level_1_weighted), file)
utils.print_and_write_2("silhoutte_level_2_weighted",
np.mean(silhoutte_level_2_weighted),
np.std(silhoutte_level_2_weighted), file)
utils.print_and_write_2("dunn1_level1", np.mean(dunn1_level_1),
np.std(dunn1_level_1), file)
utils.print_and_write_2("dunn2_level1", np.mean(dunn2_level_1),
np.std(dunn2_level_1), file)
utils.print_and_write_2("dunn3_level1", np.mean(dunn3_level_1),
np.std(dunn3_level_1), file)
utils.print_and_write_2("dunn1_level2", np.mean(dunn1_level_2),
np.std(dunn1_level_2), file)
utils.print_and_write_2("dunn2_level2", np.mean(dunn2_level_2),
np.std(dunn2_level_2), file)
utils.print_and_write_2("dunn3_level2", np.mean(dunn3_level_2),
np.std(dunn3_level_2), file)
list_of_dtw_values = []
list_of_norm_dtw_values = []
list_of_lengths = []
data_cache = {}
for demonstration in list_of_demonstrations:
try:
list_of_frms_demonstration = list_of_frms[demonstration]
data = {}
for i in range(len(list_of_frms_demonstration)):
data[i] = [elem[0] for elem in list_of_frms_demonstration[i]]
dtw_score, normalized_dtw_score, length, labels_manual_d, colors_manual_d, labels_automatic_d, colors_automatic_d = broken_barh.plot_broken_barh(
demonstration, data, constants.PATH_TO_CLUSTERING_RESULTS +
demonstration + "_" + filename + ".jpg",
constants.N_COMPONENTS_TIME_ZW)
list_of_dtw_values.append(dtw_score)
list_of_norm_dtw_values.append(normalized_dtw_score)
list_of_lengths.append(length)
# Inserting manual and annotations labels into data struct before dumping as pickle file
cache_entry = {}
cache_entry['changepoints'] = list_of_frms_demonstration
cache_entry['plot_labels_manual'] = labels_manual_d
cache_entry['plot_colors_manual'] = colors_manual_d
cache_entry['plot_labels_automatic'] = labels_automatic_d
cache_entry['plot_colors_automatic'] = colors_automatic_d
data_cache[demonstration] = cache_entry
except:
print demonstration
continue
utils.print_and_write_2("dtw_score", np.mean(list_of_dtw_values),
np.std(list_of_dtw_values), file)
utils.print_and_write_2("dtw_score_normalized",
np.mean(list_of_norm_dtw_values),
np.std(list_of_norm_dtw_values), file)
utils.print_and_write(str(list_of_lengths), file)
pickle.dump(
data_cache,
open(constants.PATH_TO_CLUSTERING_RESULTS + filename + "_.p", "wb"))
def cluster_metrics(self):
labels_true, labels_pred_1, labels_pred_2 = self.prepare_labels()
metric_funcs = [
adjusted_rand_score, adjusted_mutual_info_score,
normalized_mutual_info_score, mutual_info_score, homogeneity_score,
completeness_score
]
# ------ Label-based metrics ------
self.file.write(
"\n\nPred = L2 Milestones Pred= L1 Labels Metric\n\n")
for metric in metric_funcs:
try:
score_1 = utils.nsf(metric(labels_true, labels_pred_1))
score_2 = utils.nsf(metric(labels_true, labels_pred_2))
except ValueError as e:
print e
print "Pruning error"
sys.exit()
key = repr(metric).split()[1]
self.label_based_scores_1[key] = score_1
self.label_based_scores_2[key] = score_2
self.file.write("%3.3f %3.3f %s\n" %
(score_1, score_2, key))
# ------ Precision & Recall ------
try:
for ave in ["micro", "macro", "weighted"]:
key = "precision_" + ave
score_1 = utils.nsf(
precision_score(labels_true, labels_pred_1, average=ave))
score_2 = utils.nsf(
precision_score(labels_true, labels_pred_2, average=ave))
self.label_based_scores_1[key] = score_1
self.label_based_scores_2[key] = score_2
self.file.write("%3.3f %3.3f %s\n" %
(score_1, score_2, key))
key = "recall_" + ave
score_1 = utils.nsf(
recall_score(labels_true, labels_pred_1, average=ave))
score_2 = utils.nsf(
recall_score(labels_true, labels_pred_2, average=ave))
self.label_based_scores_1[key] = score_1
self.label_based_scores_2[key] = score_2
self.file.write("%3.3f %3.3f %s\n" %
(score_1, score_2, key))
except ValueError as e:
print e
print "Pruning error"
sys.exit()
self.file.write("\nSilhouette Scores Global\n")
# ------ Silhouette Scores ------
for layer in sorted(self.silhouette_scores_global):
score = self.silhouette_scores_global[layer]
self.file.write("%3.3f %s\n" %
(round(Decimal(score), 2), layer))
self.file.write("\nSilhouette Scores Weighted\n")
# ------ Silhouette Scores ------
for layer in sorted(self.silhouette_scores_weighted):
score = self.silhouette_scores_weighted[layer]
self.file.write("%3.3f %s\n" %
(round(Decimal(score), 2), layer))
self.file.write("\nDunn Scores1\n")
# ------ Dunn Scores ------
for layer in sorted(self.dunn_scores_1):
score = self.dunn_scores_1[layer]
self.file.write("%3.3f %s\n" %
(round(Decimal(score), 2), layer))
self.file.write("\nDunn Scores2\n")
# ------ Dunn Scores ------
for layer in sorted(self.dunn_scores_2):
score = self.dunn_scores_2[layer]
self.file.write("%3.3f %s\n" %
(round(Decimal(score), 2), layer))
self.file.write("\nDunn Scores3\n")
# ------ Dunn Scores ------
for layer in sorted(self.dunn_scores_3):
score = self.dunn_scores_3[layer]
self.file.write("%3.3f %s\n" %
(round(Decimal(score), 2), layer))
# ------ Visualizing changepoints on broken barh ------
viz = {}
for cp in self.list_of_cp:
cp_all_data = (self.map_cp2frm[cp], self.map_cp2milestones[cp],
self.map_cp2level1[cp], self.map_cp2surgemes[cp],
self.map_cp2surgemetransitions[cp])
utils.dict_insert_list(self.map_cp2demonstrations[cp], cp_all_data,
viz)
data = [
self.label_based_scores_1, self.label_based_scores_2,
self.silhouette_scores_global, self.dunn_scores_1,
self.dunn_scores_2, self.dunn_scores_3,
self.level2_silhoutte_global, self.level2_dunn_1,
self.level2_dunn_2, self.level2_dunn_3, viz,
self.silhouette_scores_weighted, self.level2_silhoutte_weighted
]
pickle.dump(data, open(self.metrics_picklefile, "wb"))
return data
def get_time_clusters(data, T_COMPONENTS):
# k-fold validation (Leave one out)
numDemos = len(data.keys()) + 1
sizeTestSet = numDemos - 1
list_of_frms = []
all_frms = []
for key in data.keys():
elem = data[key]
list_of_frms.append(elem)
all_frms += elem
N_COMPONENTS = min(T_COMPONENTS, len(all_frms))
time_cluster = mixture.GMM(n_components=N_COMPONENTS, covariance_type="full", n_iter=50000, thresh=5e-7)
X = np.array(all_frms)
X = X.reshape(len(all_frms), 1)
time_cluster.fit(X)
Y = time_cluster.predict(X)
means = time_cluster.means_
covars = time_cluster.covars_
# dpgmm = mixture.DPGMM(n_components = numDemos * 3, covariance_type='diag', n_iter = 10000, alpha = 1000, thresh= 1e-10)
# dpgmm.fit(X)
# Y = dpgmm.predict(X)
# means = dpgmm.means_
list_of_elem = []
list_of_means = []
for i in range(len(Y)):
list_of_elem.append((Y[i], X[i], means[Y[i]][0], np.sqrt(covars[Y[i]][0][0])))
list_of_elem = sorted(list_of_elem, key=lambda x: x[1][0])
dict_time_clusters = {}
for elem in list_of_elem:
utils.dict_insert_list(elem[0], elem[1], dict_time_clusters)
list_time_clusters = []
list_time_clusters_noPrune = []
for cluster in dict_time_clusters.keys():
# get all frames in this cluster
cluster_frames = dict_time_clusters[cluster]
setClusterFrames = set([elem[0] for elem in cluster_frames])
# test if frames in cluster are representative of the test set
rep = []
for id in range(sizeTestSet):
elemSet = set(list_of_frms[id])
commonElem = elemSet.intersection(setClusterFrames)
id_in_cluster = 1.0 if len(commonElem) > 0 else 0.0
rep.append(id_in_cluster)
pruneCluster = True if sum(rep) / sizeTestSet < constants.PRUNING_FACTOR_T else False
min_frm = min(cluster_frames)
max_frm = max(cluster_frames)
mean = means[cluster][0]
list_of_means.append(int(mean))
std = np.sqrt(covars[cluster][0][0])
leftFrame = max(min_frm[0], mean - std)
rightFrame = min(max_frm[0], mean + std)
list_time_clusters_noPrune.append((leftFrame, rightFrame))
# keep for plotting is pruneFlag = 0
if not (pruneCluster):
# list_time_clusters.append((min_frm[0], max_frm[0]))
list_time_clusters.append((leftFrame, rightFrame))
print "Number of Clusters pruned in Time Clustering: ", len(list_time_clusters_noPrune) - len(list_time_clusters)
labels_automatic, colors_automatic = setup_automatic_labels_2(list_time_clusters, "k")
labels_automatic_0, colors_automatic_0 = setup_automatic_labels(list_of_frms[0], "k")
return labels_automatic, colors_automatic, labels_automatic_0, colors_automatic_0, list_of_means, list_of_frms
def cluster_metrics(self):
"""
Computes metrics and returns as single dictionary.
"""
labels_true, labels_pred = self.prepare_labels()
metric_funcs = [adjusted_rand_score, adjusted_mutual_info_score, normalized_mutual_info_score,
mutual_info_score, homogeneity_score, completeness_score]
# ------ Label-based metrics ------
utils.print_and_write("\n\nPred= L1 Labels Metric\n\n", self.log)
for metric in metric_funcs:
score_1 = round(Decimal(metric(labels_true, labels_pred)), 2)
key = repr(metric).split()[1]
self.label_based_scores_1[key] = score_1
utils.print_and_write(("%3.3f %s\n" % (score_1, key)), self.log)
# ------ Precision & Recall ------
utils.print_and_write("\n Precision & Recall scores \n", self.log)
for ave in ["micro", "macro", "weighted"]:
key = "precision_" + ave
score_1 = utils.nsf(precision_score(labels_true, labels_pred, average = ave))
self.label_based_scores_1[key] = score_1
utils.print_and_write("%3.3f %s\n" % (round(Decimal(score_1), 2), key), self.log)
key = "recall_" + ave
score_1 = utils.nsf(recall_score(labels_true, labels_pred, average = ave))
self.label_based_scores_1[key] = score_1
utils.print_and_write("%3.3f %s\n" % (round(Decimal(score_1), 2), key), self.log)
utils.print_and_write("\nDunn Scores1\n", self.log)
# ------ Dunn Scores # 1------
for layer in sorted(self.dunn_scores_1):
score = self.dunn_scores_1[layer]
utils.print_and_write("%3.3f %s\n" % (round(Decimal(score), 2), layer), self.log)
utils.print_and_write("\nDunn Scores2\n", self.log)
# ------ Dunn Scores # 2------
for layer in sorted(self.dunn_scores_2):
score = self.dunn_scores_2[layer]
utils.print_and_write("%3.3f %s\n" % (round(Decimal(score), 2), layer), self.log)
utils.print_and_write("\nDunn Scores3\n", self.log)
# ------ Dunn Scores #3 ------
for layer in sorted(self.dunn_scores_3):
score = self.dunn_scores_3[layer]
utils.print_and_write("%3.3f %s\n" % (round(Decimal(score), 2), layer), self.log)
# ------ Visualizing changepoints on broken barh ------
viz = {}
for cp in self.list_of_cp:
cp_all_data = (self.map_cp2frm[cp], self.map_cp2cluster[cp], self.map_cp2surgemetransitions[cp], self.map_cp2surgemes[cp])
utils.dict_insert_list(self.map_cp2demonstrations[cp], cp_all_data, viz)
data = [self.label_based_scores_1, self.silhouette_score_global, self.dunn_scores_1,
self.dunn_scores_2, self.dunn_scores_3, viz, self.silhouette_score_weighted]
return data
def evaluate_multi_modal(metrics, file):
mutual_information_1 = []
normalized_mutual_information_1 = []
adjusted_mutual_information_1 = []
homogeneity_1 = []
precision_1_micro = []
recall_1_micro = []
precision_1_macro = []
recall_1_macro = []
precision_1_weighted = []
recall_1_weighted = []
mutual_information_2 = []
normalized_mutual_information_2 = []
adjusted_mutual_information_2 = []
homogeneity_2 = []
precision_2_micro = []
recall_2_micro = []
precision_2_macro = []
recall_2_macro = []
precision_2_weighted = []
recall_2_weighted = []
silhoutte_level_1 = []
dunn1_level_1 = []
dunn2_level_1 = []
dunn3_level_1 = []
silhoutte_level_2 = []
dunn1_level_2 = []
dunn2_level_2 = []
dunn3_level_2 = []
list_of_frms = {}
for elem in metrics:
precision_1_micro.append(elem[0]["precision_micro"])
precision_2_micro.append(elem[1]["precision_micro"])
precision_1_macro.append(elem[0]["precision_macro"])
precision_2_macro.append(elem[1]["precision_macro"])
precision_1_weighted.append(elem[0]["precision_weighted"])
precision_2_weighted.append(elem[1]["precision_weighted"])
recall_1_micro.append(elem[0]["recall_micro"])
recall_2_micro.append(elem[1]["recall_micro"])
recall_1_macro.append(elem[0]["recall_macro"])
recall_2_macro.append(elem[1]["recall_macro"])
recall_1_weighted.append(elem[0]["recall_weighted"])
recall_2_weighted.append(elem[1]["recall_weighted"])
mutual_information_1.append(elem[0]["mutual_info_score"])
mutual_information_2.append(elem[1]["mutual_info_score"])
normalized_mutual_information_1.append(elem[0]["normalized_mutual_info_score"])
normalized_mutual_information_2.append(elem[1]["normalized_mutual_info_score"])
adjusted_mutual_information_1.append(elem[0]["adjusted_mutual_info_score"])
adjusted_mutual_information_2.append(elem[1]["adjusted_mutual_info_score"])
homogeneity_1.append(elem[0]["homogeneity_score"])
homogeneity_2.append(elem[1]["homogeneity_score"])
silhoutte_level_1.append(elem[2]["level1"])
dunn1_level_1.append(elem[3]["level1"])
dunn2_level_1.append(elem[4]["level1"])
dunn3_level_1.append(elem[5]["level1"])
silhoutte_level_2 += elem[6]
dunn1_level_2 += elem[7]
dunn2_level_2 += elem[8]
dunn3_level_2 += elem[9]
viz = elem[10]
for demonstration in viz.keys():
utils.dict_insert_list(demonstration, viz[demonstration], list_of_frms)
utils.print_and_write_2("precision_1_micro", np.mean(precision_1_micro), np.std(precision_1_micro), file)
utils.print_and_write_2("precision_2_micro", np.mean(precision_2_micro), np.std(precision_2_micro), file)
utils.print_and_write_2("precision_1_macro", np.mean(precision_1_macro), np.std(precision_1_macro), file)
utils.print_and_write_2("precision_2_macro", np.mean(precision_2_macro), np.std(precision_2_macro), file)
utils.print_and_write_2("precision_1_weighted", np.mean(precision_1_weighted), np.std(precision_1_weighted), file)
utils.print_and_write_2("precision_2_weighted", np.mean(precision_2_weighted), np.std(precision_2_weighted), file)
utils.print_and_write_2("recall_1_micro", np.mean(recall_1_micro), np.std(recall_1_micro), file)
utils.print_and_write_2("recall_2_micro", np.mean(recall_2_micro), np.std(recall_2_micro), file)
utils.print_and_write_2("recall_1_macro", np.mean(recall_1_macro), np.std(recall_1_macro), file)
utils.print_and_write_2("recall_2_macro", np.mean(recall_2_macro), np.std(recall_2_macro), file)
utils.print_and_write_2("recall_1_weighted", np.mean(recall_1_weighted), np.std(recall_1_weighted), file)
utils.print_and_write_2("recall_2_weighted", np.mean(recall_2_weighted), np.std(recall_2_weighted), file)
utils.print_and_write_2("mutual_info_1", np.mean(mutual_information_1), np.std(mutual_information_1), file)
utils.print_and_write_2("mutual_info_2", np.mean(mutual_information_2), np.std(mutual_information_2), file)
utils.print_and_write_2("normalized_mutual_info_1", np.mean(normalized_mutual_information_1), np.std(normalized_mutual_information_1), file)
utils.print_and_write_2("normalized_mutual_info_2", np.mean(normalized_mutual_information_2), np.std(normalized_mutual_information_2), file)
utils.print_and_write_2("adjusted_mutual_info_1", np.mean(adjusted_mutual_information_1), np.std(adjusted_mutual_information_1), file)
utils.print_and_write_2("adjusted_mutual_info_2", np.mean(adjusted_mutual_information_2), np.std(adjusted_mutual_information_2), file)
utils.print_and_write_2("homogeneity_1", np.mean(homogeneity_1), np.std(homogeneity_1), file)
utils.print_and_write_2("homogeneity_2", np.mean(homogeneity_2), np.std(homogeneity_2), file)
utils.print_and_write_2("silhoutte_level_1", np.mean(silhoutte_level_1), np.std(silhoutte_level_1), file)
utils.print_and_write_2("silhoutte_level_2", np.mean(silhoutte_level_2), np.std(silhoutte_level_2), file)
utils.print_and_write_2("dunn1_level1", np.mean(dunn1_level_1), np.std(dunn1_level_1), file)
utils.print_and_write_2("dunn2_level1", np.mean(dunn2_level_1), np.std(dunn2_level_1), file)
utils.print_and_write_2("dunn3_level1", np.mean(dunn3_level_1), np.std(dunn3_level_1), file)
utils.print_and_write_2("dunn1_level2", np.mean(dunn1_level_2), np.std(dunn1_level_2), file)
utils.print_and_write_2("dunn2_level2", np.mean(dunn2_level_2), np.std(dunn2_level_2), file)
utils.print_and_write_2("dunn3_level2", np.mean(dunn3_level_2), np.std(dunn3_level_2), file)
return list_of_frms
def post_evaluation(metrics, file, fname, vision_mode):
mutual_information_1 = []
normalized_mutual_information_1 = []
adjusted_mutual_information_1 = []
homogeneity_1 = []
precision_1_micro = []
recall_1_micro = []
precision_1_macro = []
recall_1_macro = []
precision_1_weighted = []
recall_1_weighted = []
silhouette_scores_global = []
silhouette_scores_weighted = []
dunn1_level_1 = []
dunn2_level_1 = []
dunn3_level_1 = []
list_of_frms = {}
for elem in metrics:
mutual_information_1.append(elem[0]["mutual_info_score"])
normalized_mutual_information_1.append(elem[0]["normalized_mutual_info_score"])
adjusted_mutual_information_1.append(elem[0]["adjusted_mutual_info_score"])
homogeneity_1.append(elem[0]["homogeneity_score"])
silhouette_scores_global.append(elem[1])
silhouette_scores_weighted.append(elem[6])
dunn1_level_1.append(elem[2]["level1"])
dunn2_level_1.append(elem[3]["level1"])
dunn3_level_1.append(elem[4]["level1"])
precision_1_micro.append(elem[0]["precision_micro"])
precision_1_macro.append(elem[0]["precision_macro"])
precision_1_weighted.append(elem[0]["precision_weighted"])
recall_1_micro.append(elem[0]["recall_micro"])
recall_1_macro.append(elem[0]["recall_macro"])
recall_1_weighted.append(elem[0]["recall_weighted"])
viz = elem[5]
for demonstration in viz.keys():
utils.dict_insert_list(demonstration, viz[demonstration], list_of_frms)
utils.print_and_write_2("precision_1_micro", np.mean(precision_1_micro), np.std(precision_1_micro), file)
utils.print_and_write_2("precision_1_macro", np.mean(precision_1_macro), np.std(precision_1_macro), file)
utils.print_and_write_2("precision_1_weighted", np.mean(precision_1_weighted), np.std(precision_1_weighted), file)
utils.print_and_write_2("recall_1_micro", np.mean(recall_1_micro), np.std(recall_1_micro), file)
utils.print_and_write_2("recall_1_macro", np.mean(recall_1_macro), np.std(recall_1_macro), file)
utils.print_and_write_2("recall_1_weighted", np.mean(recall_1_weighted), np.std(recall_1_weighted), file)
utils.print_and_write_2("mutual_info", np.mean(mutual_information_1), np.std(mutual_information_1), file)
utils.print_and_write_2("normalized_mutual_info", np.mean(normalized_mutual_information_1), np.std(normalized_mutual_information_1), file)
utils.print_and_write_2("adjusted_mutual_info", np.mean(adjusted_mutual_information_1), np.std(adjusted_mutual_information_1), file)
utils.print_and_write_2("silhouette_scores_global", np.mean(silhouette_scores_global), np.std(silhouette_scores_global), file)
utils.print_and_write_2("silhouette_scores_weighted", np.mean(silhouette_scores_weighted), np.std(silhouette_scores_weighted), file)
utils.print_and_write_2("homogeneity", np.mean(homogeneity_1), np.std(homogeneity_1), file)
utils.print_and_write_2("dunn1", np.mean(dunn1_level_1), np.std(dunn1_level_1), file)
utils.print_and_write_2("dunn2", np.mean(dunn2_level_1), np.std(dunn2_level_1), file)
utils.print_and_write_2("dunn3", np.mean(dunn3_level_1), np.std(dunn3_level_1), file)
list_of_dtw_values = []
list_of_norm_dtw_values = []
list_of_lengths = []
if vision_mode:
T = constants.N_COMPONENTS_TIME_Z
else:
T = constants.N_COMPONENTS_TIME_W
pickle.dump(list_of_frms, open(constants.PATH_TO_CLUSTERING_RESULTS + fname + "_.p", "wb"))
for demonstration in list_of_demonstrations:
try:
list_of_frms_demonstration = list_of_frms[demonstration]
data = {}
for i in range(len(list_of_frms_demonstration)):
data[i] = [elem[0] for elem in list_of_frms_demonstration[i]]
dtw_score, normalized_dtw_score, length = broken_barh.plot_broken_barh(demonstration, data,
constants.PATH_TO_CLUSTERING_RESULTS + demonstration +"_" + fname + ".jpg", T)
list_of_dtw_values.append(dtw_score)
list_of_norm_dtw_values.append(normalized_dtw_score)
list_of_lengths.append(length)
except:
print demonstration
pass
utils.print_and_write_2("dtw_score", np.mean(list_of_dtw_values), np.std(list_of_dtw_values), file)
utils.print_and_write_2("dtw_score_normalized", np.mean(list_of_norm_dtw_values), np.std(list_of_norm_dtw_values), file)
utils.print_and_write(str(list_of_lengths), file)
def cluster_metrics(self):
"""
Computes metrics and returns as single dictionary.
"""
labels_true, labels_pred = self.prepare_labels()
metric_funcs = [adjusted_rand_score, adjusted_mutual_info_score, normalized_mutual_info_score,
mutual_info_score, homogeneity_score, completeness_score]
# ------ Label-based metrics ------
utils.print_and_write("\n\nPred= L1 Labels Metric\n\n", self.log)
for metric in metric_funcs:
score_1 = round(Decimal(metric(labels_true, labels_pred)), 2)
key = repr(metric).split()[1]
self.label_based_scores_1[key] = score_1
utils.print_and_write(("%3.3f %s\n" % (score_1, key)), self.log)
# ------ Precision & Recall ------
utils.print_and_write("\n Precision & Recall scores \n", self.log)
for ave in ["micro", "macro", "weighted"]:
key = "precision_" + ave
score_1 = utils.nsf(precision_score(labels_true, labels_pred, average = ave))
self.label_based_scores_1[key] = score_1
utils.print_and_write("%3.3f %s\n" % (round(Decimal(score_1), 2), key), self.log)
key = "recall_" + ave
score_1 = utils.nsf(recall_score(labels_true, labels_pred, average = ave))
self.label_based_scores_1[key] = score_1
utils.print_and_write("%3.3f %s\n" % (round(Decimal(score_1), 2), key), self.log)
utils.print_and_write("\nDunn Scores1\n", self.log)
# ------ Dunn Scores # 1------
for layer in sorted(self.dunn_scores_1):
score = self.dunn_scores_1[layer]
utils.print_and_write("%3.3f %s\n" % (round(Decimal(score), 2), layer), self.log)
utils.print_and_write("\nDunn Scores2\n", self.log)
# ------ Dunn Scores # 2------
for layer in sorted(self.dunn_scores_2):
score = self.dunn_scores_2[layer]
utils.print_and_write("%3.3f %s\n" % (round(Decimal(score), 2), layer), self.log)
utils.print_and_write("\nDunn Scores3\n", self.log)
# ------ Dunn Scores #3 ------
for layer in sorted(self.dunn_scores_3):
score = self.dunn_scores_3[layer]
utils.print_and_write("%3.3f %s\n" % (round(Decimal(score), 2), layer), self.log)
# ------ Visualizing changepoints on broken barh ------
viz = {}
for cp in self.list_of_cp:
cp_all_data = (self.map_cp2frm[cp], self.map_cp2cluster[cp], self.map_cp2surgemetransitions[cp], self.map_cp2surgemes[cp])
utils.dict_insert_list(self.map_cp2demonstrations[cp], cp_all_data, viz)
data = [self.label_based_scores_1, self.silhouette_score_global, self.dunn_scores_1,
self.dunn_scores_2, self.dunn_scores_3, viz, self.silhouette_score_weighted]
return data
def cluster_metrics(self):
labels_true, labels_pred_1, labels_pred_2 = self.prepare_labels()
metric_funcs = [adjusted_rand_score, adjusted_mutual_info_score, normalized_mutual_info_score,
mutual_info_score, homogeneity_score, completeness_score]
# ------ Label-based metrics ------
self.file.write("\n\nPred = L2 Milestones Pred= L1 Labels Metric\n\n")
for metric in metric_funcs:
try:
score_1 = utils.nsf(metric(labels_true, labels_pred_1))
score_2 = utils.nsf(metric(labels_true, labels_pred_2))
except ValueError as e:
print e
print "Pruning error"
sys.exit()
key = repr(metric).split()[1]
self.label_based_scores_1[key] = score_1
self.label_based_scores_2[key] = score_2
self.file.write("%3.3f %3.3f %s\n" % (score_1, score_2, key))
# ------ Precision & Recall ------
try:
for ave in ["micro", "macro", "weighted"]:
key = "precision_" + ave
score_1 = utils.nsf(precision_score(labels_true, labels_pred_1, average = ave))
score_2 = utils.nsf(precision_score(labels_true, labels_pred_2, average = ave))
self.label_based_scores_1[key] = score_1
self.label_based_scores_2[key] = score_2
self.file.write("%3.3f %3.3f %s\n" % (score_1, score_2, key))
key = "recall_" + ave
score_1 = utils.nsf(recall_score(labels_true, labels_pred_1, average = ave))
score_2 = utils.nsf(recall_score(labels_true, labels_pred_2, average = ave))
self.label_based_scores_1[key] = score_1
self.label_based_scores_2[key] = score_2
self.file.write("%3.3f %3.3f %s\n" % (score_1, score_2, key))
except ValueError as e:
print e
print "Pruning error"
sys.exit()
self.file.write("\nSilhouette Scores Global\n")
# ------ Silhouette Scores ------
for layer in sorted(self.silhouette_scores_global):
score = self.silhouette_scores_global[layer]
self.file.write("%3.3f %s\n" % (round(Decimal(score), 2), layer))
self.file.write("\nSilhouette Scores Weighted\n")
# ------ Silhouette Scores ------
for layer in sorted(self.silhouette_scores_weighted):
score = self.silhouette_scores_weighted[layer]
self.file.write("%3.3f %s\n" % (round(Decimal(score), 2), layer))
self.file.write("\nDunn Scores1\n")
# ------ Dunn Scores ------
for layer in sorted(self.dunn_scores_1):
score = self.dunn_scores_1[layer]
self.file.write("%3.3f %s\n" % (round(Decimal(score), 2), layer))
self.file.write("\nDunn Scores2\n")
# ------ Dunn Scores ------
for layer in sorted(self.dunn_scores_2):
score = self.dunn_scores_2[layer]
self.file.write("%3.3f %s\n" % (round(Decimal(score), 2), layer))
self.file.write("\nDunn Scores3\n")
# ------ Dunn Scores ------
for layer in sorted(self.dunn_scores_3):
score = self.dunn_scores_3[layer]
self.file.write("%3.3f %s\n" % (round(Decimal(score), 2), layer))
# ------ Visualizing changepoints on broken barh ------
viz = {}
for cp in self.list_of_cp:
cp_all_data = (self.map_cp2frm[cp], self.map_cp2milestones[cp], self.map_cp2level1[cp],
self.map_cp2surgemes[cp], self.map_cp2surgemetransitions[cp])
utils.dict_insert_list(self.map_cp2demonstrations[cp], cp_all_data, viz)
data = [self.label_based_scores_1, self.label_based_scores_2,
self.silhouette_scores_global, self.dunn_scores_1, self.dunn_scores_2, self.dunn_scores_3,
self.level2_silhoutte_global, self.level2_dunn_1, self.level2_dunn_2, self.level2_dunn_3, viz,
self.silhouette_scores_weighted, self.level2_silhoutte_weighted]
pickle.dump(data, open(self.metrics_picklefile, "wb"))
return data
def post_evaluation_multimodal(metrics, file, filename, list_of_demonstrations, feat_fname):
mutual_information_1 = []
normalized_mutual_information_1 = []
adjusted_mutual_information_1 = []
homogeneity_1 = []
precision_1_micro = []
recall_1_micro = []
precision_1_macro = []
recall_1_macro = []
precision_1_weighted = []
recall_1_weighted = []
mutual_information_2 = []
normalized_mutual_information_2 = []
adjusted_mutual_information_2 = []
homogeneity_2 = []
precision_2_micro = []
recall_2_micro = []
precision_2_macro = []
recall_2_macro = []
precision_2_weighted = []
recall_2_weighted = []
silhoutte_level_1_global = []
silhoutte_level_1_weighted = []
dunn1_level_1 = []
dunn2_level_1 = []
dunn3_level_1 = []
silhoutte_level_2_global = []
silhoutte_level_2_weighted = []
dunn1_level_2 = []
dunn2_level_2 = []
dunn3_level_2 = []
list_of_frms = {}
for elem in metrics:
precision_1_micro.append(elem[0]["precision_micro"])
precision_2_micro.append(elem[1]["precision_micro"])
precision_1_macro.append(elem[0]["precision_macro"])
precision_2_macro.append(elem[1]["precision_macro"])
precision_1_weighted.append(elem[0]["precision_weighted"])
precision_2_weighted.append(elem[1]["precision_weighted"])
recall_1_micro.append(elem[0]["recall_micro"])
recall_2_micro.append(elem[1]["recall_micro"])
recall_1_macro.append(elem[0]["recall_macro"])
recall_2_macro.append(elem[1]["recall_macro"])
recall_1_weighted.append(elem[0]["recall_weighted"])
recall_2_weighted.append(elem[1]["recall_weighted"])
mutual_information_1.append(elem[0]["mutual_info_score"])
mutual_information_2.append(elem[1]["mutual_info_score"])
normalized_mutual_information_1.append(elem[0]["normalized_mutual_info_score"])
normalized_mutual_information_2.append(elem[1]["normalized_mutual_info_score"])
adjusted_mutual_information_1.append(elem[0]["adjusted_mutual_info_score"])
adjusted_mutual_information_2.append(elem[1]["adjusted_mutual_info_score"])
homogeneity_1.append(elem[0]["homogeneity_score"])
homogeneity_2.append(elem[1]["homogeneity_score"])
silhoutte_level_1_global.append(elem[2]["level1"])
dunn1_level_1.append(elem[3]["level1"])
dunn2_level_1.append(elem[4]["level1"])
dunn3_level_1.append(elem[5]["level1"])
silhoutte_level_2_global += elem[6]
dunn1_level_2 += elem[7]
dunn2_level_2 += elem[8]
dunn3_level_2 += elem[9]
viz = elem[10]
for demonstration in viz.keys():
utils.dict_insert_list(demonstration, viz[demonstration], list_of_frms)
silhoutte_level_1_weighted.append(elem[11]["level1"])
silhoutte_level_2_weighted += elem[12]
utils.print_and_write_2("precision_1_micro", np.mean(precision_1_micro), np.std(precision_1_micro), file)
utils.print_and_write_2("precision_2_micro", np.mean(precision_2_micro), np.std(precision_2_micro), file)
utils.print_and_write_2("precision_1_macro", np.mean(precision_1_macro), np.std(precision_1_macro), file)
utils.print_and_write_2("precision_2_macro", np.mean(precision_2_macro), np.std(precision_2_macro), file)
utils.print_and_write_2("precision_1_weighted", np.mean(precision_1_weighted), np.std(precision_1_weighted), file)
utils.print_and_write_2("precision_2_weighted", np.mean(precision_2_weighted), np.std(precision_2_weighted), file)
utils.print_and_write_2("recall_1_micro", np.mean(recall_1_micro), np.std(recall_1_micro), file)
utils.print_and_write_2("recall_2_micro", np.mean(recall_2_micro), np.std(recall_2_micro), file)
utils.print_and_write_2("recall_1_macro", np.mean(recall_1_macro), np.std(recall_1_macro), file)
utils.print_and_write_2("recall_2_macro", np.mean(recall_2_macro), np.std(recall_2_macro), file)
utils.print_and_write_2("recall_1_weighted", np.mean(recall_1_weighted), np.std(recall_1_weighted), file)
utils.print_and_write_2("recall_2_weighted", np.mean(recall_2_weighted), np.std(recall_2_weighted), file)
utils.print_and_write_2("mutual_info_1", np.mean(mutual_information_1), np.std(mutual_information_1), file)
utils.print_and_write_2("mutual_info_2", np.mean(mutual_information_2), np.std(mutual_information_2), file)
utils.print_and_write_2("normalized_mutual_info_1", np.mean(normalized_mutual_information_1), np.std(normalized_mutual_information_1), file)
utils.print_and_write_2("normalized_mutual_info_2", np.mean(normalized_mutual_information_2), np.std(normalized_mutual_information_2), file)
utils.print_and_write_2("adjusted_mutual_info_1", np.mean(adjusted_mutual_information_1), np.std(adjusted_mutual_information_1), file)
utils.print_and_write_2("adjusted_mutual_info_2", np.mean(adjusted_mutual_information_2), np.std(adjusted_mutual_information_2), file)
utils.print_and_write_2("homogeneity_1", np.mean(homogeneity_1), np.std(homogeneity_1), file)
utils.print_and_write_2("homogeneity_2", np.mean(homogeneity_2), np.std(homogeneity_2), file)
utils.print_and_write_2("silhoutte_level_1_global", np.mean(silhoutte_level_1_global), np.std(silhoutte_level_1_global), file)
utils.print_and_write_2("silhoutte_level_2_global", np.mean(silhoutte_level_2_global), np.std(silhoutte_level_2_global), file)
utils.print_and_write_2("silhoutte_level_1_weighted", np.mean(silhoutte_level_1_weighted), np.std(silhoutte_level_1_weighted), file)
utils.print_and_write_2("silhoutte_level_2_weighted", np.mean(silhoutte_level_2_weighted), np.std(silhoutte_level_2_weighted), file)
utils.print_and_write_2("dunn1_level1", np.mean(dunn1_level_1), np.std(dunn1_level_1), file)
utils.print_and_write_2("dunn2_level1", np.mean(dunn2_level_1), np.std(dunn2_level_1), file)
utils.print_and_write_2("dunn3_level1", np.mean(dunn3_level_1), np.std(dunn3_level_1), file)
utils.print_and_write_2("dunn1_level2", np.mean(dunn1_level_2), np.std(dunn1_level_2), file)
utils.print_and_write_2("dunn2_level2", np.mean(dunn2_level_2), np.std(dunn2_level_2), file)
utils.print_and_write_2("dunn3_level2", np.mean(dunn3_level_2), np.std(dunn3_level_2), file)
list_of_dtw_values = []
list_of_norm_dtw_values = []
list_of_lengths = []
data_cache = {}
for demonstration in list_of_demonstrations:
try:
list_of_frms_demonstration = list_of_frms[demonstration]
data = {}
for i in range(len(list_of_frms_demonstration)):
data[i] = [elem[0] for elem in list_of_frms_demonstration[i]]
dtw_score, normalized_dtw_score, length, labels_manual_d, colors_manual_d, labels_automatic_d, colors_automatic_d = broken_barh.plot_broken_barh(demonstration, data,
constants.PATH_TO_CLUSTERING_RESULTS + demonstration +"_" + filename + ".jpg", constants.N_COMPONENTS_TIME_ZW)
list_of_dtw_values.append(dtw_score)
list_of_norm_dtw_values.append(normalized_dtw_score)
list_of_lengths.append(length)
# Inserting manual and annotations labels into data struct before dumping as pickle file
cache_entry = {}
cache_entry['changepoints'] = list_of_frms_demonstration
cache_entry['plot_labels_manual'] = labels_manual_d
cache_entry['plot_colors_manual'] = colors_manual_d
cache_entry['plot_labels_automatic'] = labels_automatic_d
cache_entry['plot_colors_automatic'] = colors_automatic_d
data_cache[demonstration] = cache_entry
except:
print demonstration
continue
utils.print_and_write_2("dtw_score", np.mean(list_of_dtw_values), np.std(list_of_dtw_values), file)
utils.print_and_write_2("dtw_score_normalized", np.mean(list_of_norm_dtw_values), np.std(list_of_norm_dtw_values), file)
utils.print_and_write(str(list_of_lengths), file)
pickle.dump(data_cache, open(constants.PATH_TO_CLUSTERING_RESULTS + filename + "_.p", "wb"))
def evaluate_multi_modal(metrics, file):
mutual_information_1 = []
normalized_mutual_information_1 = []
adjusted_mutual_information_1 = []
homogeneity_1 = []
precision_1_micro = []
recall_1_micro = []
precision_1_macro = []
recall_1_macro = []
precision_1_weighted = []
recall_1_weighted = []
mutual_information_2 = []
normalized_mutual_information_2 = []
adjusted_mutual_information_2 = []
homogeneity_2 = []
precision_2_micro = []
recall_2_micro = []
precision_2_macro = []
recall_2_macro = []
precision_2_weighted = []
recall_2_weighted = []
silhoutte_level_1 = []
dunn1_level_1 = []
dunn2_level_1 = []
dunn3_level_1 = []
silhoutte_level_2 = []
dunn1_level_2 = []
dunn2_level_2 = []
dunn3_level_2 = []
list_of_frms = {}
for elem in metrics:
precision_1_micro.append(elem[0]["precision_micro"])
precision_2_micro.append(elem[1]["precision_micro"])
precision_1_macro.append(elem[0]["precision_macro"])
precision_2_macro.append(elem[1]["precision_macro"])
precision_1_weighted.append(elem[0]["precision_weighted"])
precision_2_weighted.append(elem[1]["precision_weighted"])
recall_1_micro.append(elem[0]["recall_micro"])
recall_2_micro.append(elem[1]["recall_micro"])
recall_1_macro.append(elem[0]["recall_macro"])
recall_2_macro.append(elem[1]["recall_macro"])
recall_1_weighted.append(elem[0]["recall_weighted"])
recall_2_weighted.append(elem[1]["recall_weighted"])
mutual_information_1.append(elem[0]["mutual_info_score"])
mutual_information_2.append(elem[1]["mutual_info_score"])
normalized_mutual_information_1.append(
elem[0]["normalized_mutual_info_score"])
normalized_mutual_information_2.append(
elem[1]["normalized_mutual_info_score"])
adjusted_mutual_information_1.append(
elem[0]["adjusted_mutual_info_score"])
adjusted_mutual_information_2.append(
elem[1]["adjusted_mutual_info_score"])
homogeneity_1.append(elem[0]["homogeneity_score"])
homogeneity_2.append(elem[1]["homogeneity_score"])
silhoutte_level_1.append(elem[2]["level1"])
dunn1_level_1.append(elem[3]["level1"])
dunn2_level_1.append(elem[4]["level1"])
dunn3_level_1.append(elem[5]["level1"])
silhoutte_level_2 += elem[6]
dunn1_level_2 += elem[7]
dunn2_level_2 += elem[8]
dunn3_level_2 += elem[9]
viz = elem[10]
for demonstration in viz.keys():
utils.dict_insert_list(demonstration, viz[demonstration],
list_of_frms)
utils.print_and_write_2("precision_1_micro", np.mean(precision_1_micro),
np.std(precision_1_micro), file)
utils.print_and_write_2("precision_2_micro", np.mean(precision_2_micro),
np.std(precision_2_micro), file)
utils.print_and_write_2("precision_1_macro", np.mean(precision_1_macro),
np.std(precision_1_macro), file)
utils.print_and_write_2("precision_2_macro", np.mean(precision_2_macro),
np.std(precision_2_macro), file)
utils.print_and_write_2("precision_1_weighted",
np.mean(precision_1_weighted),
np.std(precision_1_weighted), file)
utils.print_and_write_2("precision_2_weighted",
np.mean(precision_2_weighted),
np.std(precision_2_weighted), file)
utils.print_and_write_2("recall_1_micro", np.mean(recall_1_micro),
np.std(recall_1_micro), file)
utils.print_and_write_2("recall_2_micro", np.mean(recall_2_micro),
np.std(recall_2_micro), file)
utils.print_and_write_2("recall_1_macro", np.mean(recall_1_macro),
np.std(recall_1_macro), file)
utils.print_and_write_2("recall_2_macro", np.mean(recall_2_macro),
np.std(recall_2_macro), file)
utils.print_and_write_2("recall_1_weighted", np.mean(recall_1_weighted),
np.std(recall_1_weighted), file)
utils.print_and_write_2("recall_2_weighted", np.mean(recall_2_weighted),
np.std(recall_2_weighted), file)
utils.print_and_write_2("mutual_info_1", np.mean(mutual_information_1),
np.std(mutual_information_1), file)
utils.print_and_write_2("mutual_info_2", np.mean(mutual_information_2),
np.std(mutual_information_2), file)
utils.print_and_write_2("normalized_mutual_info_1",
np.mean(normalized_mutual_information_1),
np.std(normalized_mutual_information_1), file)
utils.print_and_write_2("normalized_mutual_info_2",
np.mean(normalized_mutual_information_2),
np.std(normalized_mutual_information_2), file)
utils.print_and_write_2("adjusted_mutual_info_1",
np.mean(adjusted_mutual_information_1),
np.std(adjusted_mutual_information_1), file)
utils.print_and_write_2("adjusted_mutual_info_2",
np.mean(adjusted_mutual_information_2),
np.std(adjusted_mutual_information_2), file)
utils.print_and_write_2("homogeneity_1", np.mean(homogeneity_1),
np.std(homogeneity_1), file)
utils.print_and_write_2("homogeneity_2", np.mean(homogeneity_2),
np.std(homogeneity_2), file)
utils.print_and_write_2("silhoutte_level_1", np.mean(silhoutte_level_1),
np.std(silhoutte_level_1), file)
utils.print_and_write_2("silhoutte_level_2", np.mean(silhoutte_level_2),
np.std(silhoutte_level_2), file)
utils.print_and_write_2("dunn1_level1", np.mean(dunn1_level_1),
np.std(dunn1_level_1), file)
utils.print_and_write_2("dunn2_level1", np.mean(dunn2_level_1),
np.std(dunn2_level_1), file)
utils.print_and_write_2("dunn3_level1", np.mean(dunn3_level_1),
np.std(dunn3_level_1), file)
utils.print_and_write_2("dunn1_level2", np.mean(dunn1_level_2),
np.std(dunn1_level_2), file)
utils.print_and_write_2("dunn2_level2", np.mean(dunn2_level_2),
np.std(dunn2_level_2), file)
utils.print_and_write_2("dunn3_level2", np.mean(dunn3_level_2),
np.std(dunn3_level_2), file)
return list_of_frms
