def generate_change_points_1(self): """ Generates changespoints by clustering within demonstration. """ cp_index = 0 for demonstration in self.list_of_demonstrations: print "Changepoints for " + demonstration N = self.data_N[demonstration] gmm = mixture.GMM(n_components = self.n_components_cp, n_iter=5000, thresh = 5e-5, covariance_type='full') gmm.fit(N) Y = gmm.predict(N) start, end = parser.get_start_end_annotations(constants.PATH_TO_DATA + constants.ANNOTATIONS_FOLDER + demonstration + "_" + constants.CAMERA + ".p") self.save_cluster_metrics(N, Y, 'cpts_' + demonstration) for i in range(len(Y) - 1): if Y[i] != Y[i + 1]: change_pt = N[i][self.X_dimension:] self.append_cp_array(utils.reshape(change_pt)) self.map_cp2frm[cp_index] = start + i * self.sr self.map_cp2demonstrations[cp_index] = demonstration self.list_of_cp.append(cp_index) cp_index += 1
def preprocess(list_of_demonstrations): camera = constants.CAMERA for demonstration in list_of_demonstrations: PATH_TO_ANNOTATION = constants.PATH_TO_DATA + constants.ANNOTATIONS_FOLDER + demonstration + "_" + camera + ".p" start, end = parser.get_start_end_annotations(PATH_TO_ANNOTATION) OLD_FRM_PATH = constants.PATH_TO_DATA + "frames_unprocessed/" + demonstration + "_" + camera + "/" NEW_FRM_PATH = constants.PATH_TO_DATA + constants.NEW_FRAMES_FOLDER + demonstration + "_" + camera + "/" command = "mkdir " + NEW_FRM_PATH print command os.mkdir(NEW_FRM_PATH) for frm in range(start, end + 1): OLD_FRM_NAME = utils.get_full_image_path(OLD_FRM_PATH, frm) NEW_FRM_NAME = utils.get_full_image_path(NEW_FRM_PATH, frm) NEW_FRM_NAME_UNSCALED = utils.get_full_image_path(NEW_FRM_PATH + "unscaled_", frm) command = "ffmpeg -i " + OLD_FRM_NAME + " -filter:v " + constants.CROP_PARAMS[camera] + " " + NEW_FRM_NAME_UNSCALED print command os.system(command) command = "ffmpeg -i " + NEW_FRM_NAME_UNSCALED + " -vf scale=640:480 " + NEW_FRM_NAME print command os.system(command) command = "rm " + NEW_FRM_NAME_UNSCALED print command os.system(command)
def generate_change_points(self): cp_index = 0 for demonstration in self.list_of_demonstrations: # print "Changepoints for " + demonstration N = self.data_N[demonstration] # print N[0].shape for demonstration in self.list_of_demonstrations: # print "Changepoints for " + demonstration N = self.data_N[demonstration] gmm = mixture.GMM(n_components = 10, covariance_type='full') gmm.fit(N) Y = gmm.predict(N) self.save_cluster_metrics(N, Y, gmm.means_, 'cpts_' + demonstration, gmm) start, end = parser.get_start_end_annotations(constants.PATH_TO_DATA + constants.ANNOTATIONS_FOLDER + demonstration + "_capture2.p") size_of_X = self.data_X_size[demonstration] for i in range(len(Y) - 1): if Y[i] != Y[i + 1]: change_pt = N[i][size_of_X:] # print N.shape, change_pt.shape self.append_cp_array(change_pt) self.map_cp2frm[cp_index] = start + i * self.sr self.map_cp2demonstrations[cp_index] = demonstration self.list_of_cp.append(cp_index) cp_index += 1
def plot_broken_barh(demonstration, data, save_fname=None, T=10):
"""
Parameters:
-----------
demonstration: String name of demonstration without camera specification , e.g. "Suturing_E001"
list_of_frms_[1,2,3,4]: List of changepoint frames for each of 4 different clustering experiments.
Use this to compare manually vs. automatically generated transition points.
* For now, the list_of_frms are constrained to 4 for visualization sanity sake.
"""
numDemos = min(5, len(data.keys()) + 1)
sizeTestSet = numDemos - 1
PATH_TO_ANNOTATION = (
constants.PATH_TO_DATA + constants.ANNOTATIONS_FOLDER + demonstration + "_" + constants.CAMERA + ".p"
)
start, end = parser.get_start_end_annotations(
constants.PATH_TO_DATA + constants.ANNOTATIONS_FOLDER + demonstration + "_" + constants.CAMERA + ".p"
)
length = end - start
segments = pickle.load(open(PATH_TO_ANNOTATION, "rb"))
fig, ax = plt.subplots()
# Generate labels for 1) Manual 2) Time clusters
labels_automatic_0, colors_automatic_0, labels_automatic_1, colors_automatic_1, means, list_of_frms = get_time_clusters(
data, T
)
labels_manual, colors_manual = setup_manual_labels(segments)
# Plot 1) Manual 2) Time clusters
ax.broken_barh(labels_manual, (25, 2), facecolors=colors_manual)
ax.broken_barh(labels_automatic_0, (21, 2), facecolors=colors_automatic_0)
list_of_plot_ranges = [(17, 2), (13, 2), (9, 2), (5, 2)]
for i in range(min(sizeTestSet, 4)):
labels_automatic, colors_automatic = setup_automatic_labels(list_of_frms[i], "k")
ax.broken_barh(labels_automatic, list_of_plot_ranges[i], facecolors=colors_automatic)
TASK = constants.TASK_NAME
if TASK in ["lego", "plane"]:
end = end + 20
elif TASK in ["000", "010", "011", "100"]:
end = end + 10
else:
end = end + 50
ticks = get_ticks(labels_manual)
ax.set_ylim(3, 29)
ax.set_xlim(0, end)
ax.set_xlabel("Frame number")
# ax.set_xticks(ticks)
ax.set_yticks([6, 10, 14, 18, 22, 26])
ax.set_yticklabels(["Automatic4", "Automatic3", "Automatic2", "Automatic1", "Time Clustering", "Manual"])
if save_fname:
plt.savefig(save_fname)
else:
plt.show()
pass
time_sequence_1 = [elem[0] + elem[1] for elem in labels_manual]
time_sequence_2 = means
dtw_score = compute_dtw(time_sequence_1, time_sequence_2)
normalized_dtw_score = dtw_score / float(length) * 100
return dtw_score, normalized_dtw_score, length
def plot_broken_barh_all(demonstration, data_W, data_Z, data_ZW, save_fname=None, save_fname2=None):
"""
Plots time-clusters for W, K, KW.
"""
PATH_TO_ANNOTATION = (
constants.PATH_TO_DATA + constants.ANNOTATIONS_FOLDER + demonstration + "_" + constants.CAMERA + ".p"
)
start, end = parser.get_start_end_annotations(
constants.PATH_TO_DATA + constants.ANNOTATIONS_FOLDER + demonstration + "_" + constants.CAMERA + ".p"
)
length = end - start
segments = pickle.load(open(PATH_TO_ANNOTATION, "rb"))
TASK = constants.TASK_NAME
if TASK in ["lego", "plane"]:
end = end + 20
elif TASK in ["000", "010", "011", "100"]:
end = end + 2
else:
end = end + 50
labels_manual, colors_manual = setup_manual_labels(segments)
labels_automatic_W, colors_automatic_W, labels_automatic_W_0, colors_automatic_W_0, means_W, list_of_frms_W = get_time_clusters(
data_W, constants.N_COMPONENTS_TIME_W
)
labels_automatic_Z, colors_automatic_Z, labels_automatic_Z_0, colors_automatic_Z_0, means_Z, list_of_frms_Z = get_time_clusters(
data_Z, constants.N_COMPONENTS_TIME_Z
)
labels_automatic_ZW, colors_automatic_ZW, labels_automatic_ZW_0, colors_automatic_ZW_0, means_ZW, list_of_frms_ZW = get_time_clusters(
data_ZW, constants.N_COMPONENTS_TIME_ZW
)
fig, ax = plt.subplots()
ax.broken_barh(labels_manual, (17, 2), facecolors=colors_manual)
ax.broken_barh(labels_automatic_W, (13, 2), facecolors=colors_automatic_W)
ax.broken_barh(labels_automatic_Z, (9, 2), facecolors=colors_automatic_Z)
ax.broken_barh(labels_automatic_ZW, (5, 2), facecolors=colors_automatic_ZW)
ax.set_ylim(3, 21)
ax.set_xlim(0, end)
ax.set_xlabel("Frame number")
ax.set_yticks([6, 10, 14, 18])
ax.set_yticklabels(["ZW", "Z", "W", "Manual"])
if save_fname:
plt.savefig(save_fname)
else:
plt.show()
pass
fig, ax = plt.subplots()
ax.broken_barh(labels_manual, (17, 2), facecolors=colors_manual)
ax.broken_barh(labels_automatic_W_0, (13, 2), facecolors=colors_automatic_W_0)
ax.broken_barh(labels_automatic_Z_0, (9, 2), facecolors=colors_automatic_Z_0)
ax.broken_barh(labels_automatic_ZW_0, (5, 2), facecolors=colors_automatic_ZW_0)
ax.set_ylim(3, 21)
ax.set_xlim(0, end)
ax.set_xlabel("Frame number")
ax.set_yticks([6, 10, 14, 18])
ax.set_yticklabels(["ZW_0", "Z_0", "W_0", "Manual"])
if save_fname2:
plt.savefig(save_fname2)
else:
plt.show()
pass
time_sequence_1 = [elem[0] + elem[1] for elem in labels_manual]
time_sequence_2 = means_W
dtw_score_W = compute_dtw(time_sequence_1, time_sequence_2)
time_sequence_2 = means_Z
dtw_score_Z = compute_dtw(time_sequence_1, time_sequence_2)
time_sequence_2 = means_ZW
dtw_score_ZW = compute_dtw(time_sequence_1, time_sequence_2)
dtw_score_W_normalized = dtw_score_W / float(length) * 100
dtw_score_Z_normalized = dtw_score_Z / float(length) * 100
dtw_score_ZW_normalized = dtw_score_ZW / float(length) * 100
return (
dtw_score_W,
dtw_score_Z,
dtw_score_ZW,
dtw_score_W_normalized,
dtw_score_Z_normalized,
dtw_score_ZW_normalized,
length,
)
def generate_change_points_2(self):
"""
Generates changespoints by clustering across demonstrations.
"""
cp_index = 0
i = 0
big_N = None
map_index2demonstration = {}
map_index2frm = {}
for demonstration in self.list_of_demonstrations:
print demonstration
N = self.data_N[demonstration]
start, end = parser.get_start_end_annotations(constants.PATH_TO_DATA + constants.ANNOTATIONS_FOLDER
+ demonstration + "_" + constants.CAMERA + ".p")
for j in range(N.shape[0]):
map_index2demonstration[i] = demonstration
map_index2frm[i] = start + j * self.sr
i += 1
big_N = utils.safe_concatenate(big_N, N)
print "Generating Changepoints. Fitting GMM/DP-GMM ..."
if constants.REMOTE == 1:
if self.fit_DPGMM:
print "Init DPGMM"
avg_len = int(big_N.shape[0]/len(self.list_of_demonstrations))
DP_GMM_COMPONENTS = int(avg_len/constants.DPGMM_DIVISOR)
print "L0", DP_GMM_COMPONENTS, "ALPHA: ", self.ALPHA_CP
dpgmm = mixture.DPGMM(n_components = DP_GMM_COMPONENTS, covariance_type='diag', n_iter = 10000, alpha = self.ALPHA_CP, thresh= 1e-7)
if self.fit_GMM:
print "Init GMM"
gmm = mixture.GMM(n_components = self.n_components_cp, covariance_type='full', n_iter=5000, thresh = 5e-5)
if constants.REMOTE == 2:
gmm = mixture.GMM(n_components = self.n_components_cp, covariance_type='full', thresh = 0.01)
else:
gmm = mixture.GMM(n_components = self.n_components_cp, covariance_type='full')
if self.fit_GMM:
print "Fitting GMM"
start = time.time()
gmm.fit(big_N)
end = time.time()
print "GMM Time:", end - start
Y_gmm = gmm.predict(big_N)
print "L0: Clusters in GMM", len(set(Y_gmm))
Y = Y_gmm
if self.fit_DPGMM:
print "Fitting DPGMM"
start = time.time()
dpgmm.fit(big_N)
end = time.time()
print "DPGMM Time:", end - start
Y_dpgmm = dpgmm.predict(big_N)
print "L0: Clusters in DP-GMM", len(set(Y_dpgmm))
Y = Y_dpgmm
for w in range(len(Y) - 1):
if Y[w] != Y[w + 1]:
change_pt = big_N[w][self.X_dimension:]
self.append_cp_array(utils.reshape(change_pt))
self.map_cp2frm[cp_index] = map_index2frm[w]
self.map_cp2demonstrations[cp_index] = map_index2demonstration[w]
self.list_of_cp.append(cp_index)
cp_index += 1
print "Done with generating change points, " + str(cp_index)
def generate_change_points_2(self):
"""
Generates changespoints by clustering across demonstrations.
"""
cp_index = 0
i = 0
big_N = None
map_index2demonstration = {}
map_index2frm = {}
size_of_X = self.data_X_size[self.list_of_demonstrations[0]]
for demonstration in self.list_of_demonstrations:
print demonstration
N = self.data_N[demonstration]
start, end = parser.get_start_end_annotations(
constants.PATH_TO_DATA + constants.ANNOTATIONS_FOLDER + demonstration + "_" + constants.CAMERA + ".p"
)
for j in range(N.shape[0]):
map_index2demonstration[i] = demonstration
map_index2frm[i] = start + j * self.sr
i += 1
big_N = utils.safe_concatenate(big_N, N)
print "Generated big_N"
if constants.REMOTE == 1:
if self.fit_GMM:
gmm = mixture.GMM(n_components=self.n_components_cp, covariance_type="full", thresh=0.01)
if self.fit_DPGMM:
# dpgmm = mixture.DPGMM(n_components = 100, covariance_type='diag', n_iter = 10000, alpha = 100, thresh= 2e-4)
# DO NOT FIDDLE WITH PARAMS WITHOUT CONSENT :)
avg_len = int(big_N.shape[0] / len(self.list_of_demonstrations))
DP_GMM_COMPONENTS = int(
avg_len / constants.DPGMM_DIVISOR
) # tuned with suturing experts only for kinematics
print "L0 ", DP_GMM_COMPONENTS, "ALPHA: ", constants.ALPHA_ZW_CP
dpgmm = mixture.DPGMM(
n_components=DP_GMM_COMPONENTS,
covariance_type="diag",
n_iter=1000,
alpha=constants.ALPHA_ZW_CP,
thresh=1e-7,
)
elif constants.REMOTE == 2:
gmm = mixture.GMM(n_components=self.n_components_cp, covariance_type="full")
else:
gmm = mixture.GMM(n_components=self.n_components_cp, covariance_type="full")
if self.fit_GMM:
start = time.time()
gmm.fit(big_N)
end = time.time()
"GMM time taken: ", str(end - start)
Y_gmm = gmm.predict(big_N)
print "L0: Clusters in GMM", len(set(Y_gmm))
Y = Y_gmm
if self.fit_DPGMM:
start = time.time()
dpgmm.fit(big_N)
end = time.time()
"DP-GMM time taken: ", str(end - start)
Y_dpgmm = dpgmm.predict(big_N)
Y = Y_dpgmm
print "L0: Clusters in DP-GMM", len(set(Y_dpgmm))
for w in range(len(Y) - 1):
if Y[w] != Y[w + 1]:
change_pt = big_N[w][size_of_X:]
self.append_cp_array(change_pt)
self.map_cp2frm[cp_index] = map_index2frm[w]
self.map_cp2demonstrations[cp_index] = map_index2demonstration[w]
self.list_of_cp.append(cp_index)
cp_index += 1
print "Done with generating change points", len(self.list_of_cp)