def generate_change_points_1(self): """ Generates changespoints by clustering within demonstration. """ cp_index = 0 for demonstration in self.list_of_demonstrations: print "Changepoints for " + demonstration N = self.data_N[demonstration] gmm = mixture.GMM(n_components = self.n_components_cp, n_iter=5000, thresh = 5e-5, covariance_type='full') gmm.fit(N) Y = gmm.predict(N) start, end = parser.get_start_end_annotations(constants.PATH_TO_DATA + constants.ANNOTATIONS_FOLDER + demonstration + "_" + constants.CAMERA + ".p") self.save_cluster_metrics(N, Y, 'cpts_' + demonstration) for i in range(len(Y) - 1): if Y[i] != Y[i + 1]: change_pt = N[i][self.X_dimension:] self.append_cp_array(utils.reshape(change_pt)) self.map_cp2frm[cp_index] = start + i * self.sr self.map_cp2demonstrations[cp_index] = demonstration self.list_of_cp.append(cp_index) cp_index += 1
def preprocess(list_of_demonstrations): camera = constants.CAMERA for demonstration in list_of_demonstrations: PATH_TO_ANNOTATION = constants.PATH_TO_DATA + constants.ANNOTATIONS_FOLDER + demonstration + "_" + camera + ".p" start, end = parser.get_start_end_annotations(PATH_TO_ANNOTATION) OLD_FRM_PATH = constants.PATH_TO_DATA + "frames_unprocessed/" + demonstration + "_" + camera + "/" NEW_FRM_PATH = constants.PATH_TO_DATA + constants.NEW_FRAMES_FOLDER + demonstration + "_" + camera + "/" command = "mkdir " + NEW_FRM_PATH print command os.mkdir(NEW_FRM_PATH) for frm in range(start, end + 1): OLD_FRM_NAME = utils.get_full_image_path(OLD_FRM_PATH, frm) NEW_FRM_NAME = utils.get_full_image_path(NEW_FRM_PATH, frm) NEW_FRM_NAME_UNSCALED = utils.get_full_image_path(NEW_FRM_PATH + "unscaled_", frm) command = "ffmpeg -i " + OLD_FRM_NAME + " -filter:v " + constants.CROP_PARAMS[camera] + " " + NEW_FRM_NAME_UNSCALED print command os.system(command) command = "ffmpeg -i " + NEW_FRM_NAME_UNSCALED + " -vf scale=640:480 " + NEW_FRM_NAME print command os.system(command) command = "rm " + NEW_FRM_NAME_UNSCALED print command os.system(command)
def generate_change_points(self): cp_index = 0 for demonstration in self.list_of_demonstrations: # print "Changepoints for " + demonstration N = self.data_N[demonstration] # print N[0].shape for demonstration in self.list_of_demonstrations: # print "Changepoints for " + demonstration N = self.data_N[demonstration] gmm = mixture.GMM(n_components = 10, covariance_type='full') gmm.fit(N) Y = gmm.predict(N) self.save_cluster_metrics(N, Y, gmm.means_, 'cpts_' + demonstration, gmm) start, end = parser.get_start_end_annotations(constants.PATH_TO_DATA + constants.ANNOTATIONS_FOLDER + demonstration + "_capture2.p") size_of_X = self.data_X_size[demonstration] for i in range(len(Y) - 1): if Y[i] != Y[i + 1]: change_pt = N[i][size_of_X:] # print N.shape, change_pt.shape self.append_cp_array(change_pt) self.map_cp2frm[cp_index] = start + i * self.sr self.map_cp2demonstrations[cp_index] = demonstration self.list_of_cp.append(cp_index) cp_index += 1
def plot_broken_barh(demonstration, data, save_fname=None, T=10): """ Parameters: ----------- demonstration: String name of demonstration without camera specification , e.g. "Suturing_E001" list_of_frms_[1,2,3,4]: List of changepoint frames for each of 4 different clustering experiments. Use this to compare manually vs. automatically generated transition points. * For now, the list_of_frms are constrained to 4 for visualization sanity sake. """ numDemos = min(5, len(data.keys()) + 1) sizeTestSet = numDemos - 1 PATH_TO_ANNOTATION = ( constants.PATH_TO_DATA + constants.ANNOTATIONS_FOLDER + demonstration + "_" + constants.CAMERA + ".p" ) start, end = parser.get_start_end_annotations( constants.PATH_TO_DATA + constants.ANNOTATIONS_FOLDER + demonstration + "_" + constants.CAMERA + ".p" ) length = end - start segments = pickle.load(open(PATH_TO_ANNOTATION, "rb")) fig, ax = plt.subplots() # Generate labels for 1) Manual 2) Time clusters labels_automatic_0, colors_automatic_0, labels_automatic_1, colors_automatic_1, means, list_of_frms = get_time_clusters( data, T ) labels_manual, colors_manual = setup_manual_labels(segments) # Plot 1) Manual 2) Time clusters ax.broken_barh(labels_manual, (25, 2), facecolors=colors_manual) ax.broken_barh(labels_automatic_0, (21, 2), facecolors=colors_automatic_0) list_of_plot_ranges = [(17, 2), (13, 2), (9, 2), (5, 2)] for i in range(min(sizeTestSet, 4)): labels_automatic, colors_automatic = setup_automatic_labels(list_of_frms[i], "k") ax.broken_barh(labels_automatic, list_of_plot_ranges[i], facecolors=colors_automatic) TASK = constants.TASK_NAME if TASK in ["lego", "plane"]: end = end + 20 elif TASK in ["000", "010", "011", "100"]: end = end + 10 else: end = end + 50 ticks = get_ticks(labels_manual) ax.set_ylim(3, 29) ax.set_xlim(0, end) ax.set_xlabel("Frame number") # ax.set_xticks(ticks) ax.set_yticks([6, 10, 14, 18, 22, 26]) ax.set_yticklabels(["Automatic4", "Automatic3", "Automatic2", "Automatic1", "Time Clustering", "Manual"]) if save_fname: plt.savefig(save_fname) else: plt.show() pass time_sequence_1 = [elem[0] + elem[1] for elem in labels_manual] time_sequence_2 = means dtw_score = compute_dtw(time_sequence_1, time_sequence_2) normalized_dtw_score = dtw_score / float(length) * 100 return dtw_score, normalized_dtw_score, length
def plot_broken_barh_all(demonstration, data_W, data_Z, data_ZW, save_fname=None, save_fname2=None): """ Plots time-clusters for W, K, KW. """ PATH_TO_ANNOTATION = ( constants.PATH_TO_DATA + constants.ANNOTATIONS_FOLDER + demonstration + "_" + constants.CAMERA + ".p" ) start, end = parser.get_start_end_annotations( constants.PATH_TO_DATA + constants.ANNOTATIONS_FOLDER + demonstration + "_" + constants.CAMERA + ".p" ) length = end - start segments = pickle.load(open(PATH_TO_ANNOTATION, "rb")) TASK = constants.TASK_NAME if TASK in ["lego", "plane"]: end = end + 20 elif TASK in ["000", "010", "011", "100"]: end = end + 2 else: end = end + 50 labels_manual, colors_manual = setup_manual_labels(segments) labels_automatic_W, colors_automatic_W, labels_automatic_W_0, colors_automatic_W_0, means_W, list_of_frms_W = get_time_clusters( data_W, constants.N_COMPONENTS_TIME_W ) labels_automatic_Z, colors_automatic_Z, labels_automatic_Z_0, colors_automatic_Z_0, means_Z, list_of_frms_Z = get_time_clusters( data_Z, constants.N_COMPONENTS_TIME_Z ) labels_automatic_ZW, colors_automatic_ZW, labels_automatic_ZW_0, colors_automatic_ZW_0, means_ZW, list_of_frms_ZW = get_time_clusters( data_ZW, constants.N_COMPONENTS_TIME_ZW ) fig, ax = plt.subplots() ax.broken_barh(labels_manual, (17, 2), facecolors=colors_manual) ax.broken_barh(labels_automatic_W, (13, 2), facecolors=colors_automatic_W) ax.broken_barh(labels_automatic_Z, (9, 2), facecolors=colors_automatic_Z) ax.broken_barh(labels_automatic_ZW, (5, 2), facecolors=colors_automatic_ZW) ax.set_ylim(3, 21) ax.set_xlim(0, end) ax.set_xlabel("Frame number") ax.set_yticks([6, 10, 14, 18]) ax.set_yticklabels(["ZW", "Z", "W", "Manual"]) if save_fname: plt.savefig(save_fname) else: plt.show() pass fig, ax = plt.subplots() ax.broken_barh(labels_manual, (17, 2), facecolors=colors_manual) ax.broken_barh(labels_automatic_W_0, (13, 2), facecolors=colors_automatic_W_0) ax.broken_barh(labels_automatic_Z_0, (9, 2), facecolors=colors_automatic_Z_0) ax.broken_barh(labels_automatic_ZW_0, (5, 2), facecolors=colors_automatic_ZW_0) ax.set_ylim(3, 21) ax.set_xlim(0, end) ax.set_xlabel("Frame number") ax.set_yticks([6, 10, 14, 18]) ax.set_yticklabels(["ZW_0", "Z_0", "W_0", "Manual"]) if save_fname2: plt.savefig(save_fname2) else: plt.show() pass time_sequence_1 = [elem[0] + elem[1] for elem in labels_manual] time_sequence_2 = means_W dtw_score_W = compute_dtw(time_sequence_1, time_sequence_2) time_sequence_2 = means_Z dtw_score_Z = compute_dtw(time_sequence_1, time_sequence_2) time_sequence_2 = means_ZW dtw_score_ZW = compute_dtw(time_sequence_1, time_sequence_2) dtw_score_W_normalized = dtw_score_W / float(length) * 100 dtw_score_Z_normalized = dtw_score_Z / float(length) * 100 dtw_score_ZW_normalized = dtw_score_ZW / float(length) * 100 return ( dtw_score_W, dtw_score_Z, dtw_score_ZW, dtw_score_W_normalized, dtw_score_Z_normalized, dtw_score_ZW_normalized, length, )
def generate_change_points_2(self): """ Generates changespoints by clustering across demonstrations. """ cp_index = 0 i = 0 big_N = None map_index2demonstration = {} map_index2frm = {} for demonstration in self.list_of_demonstrations: print demonstration N = self.data_N[demonstration] start, end = parser.get_start_end_annotations(constants.PATH_TO_DATA + constants.ANNOTATIONS_FOLDER + demonstration + "_" + constants.CAMERA + ".p") for j in range(N.shape[0]): map_index2demonstration[i] = demonstration map_index2frm[i] = start + j * self.sr i += 1 big_N = utils.safe_concatenate(big_N, N) print "Generating Changepoints. Fitting GMM/DP-GMM ..." if constants.REMOTE == 1: if self.fit_DPGMM: print "Init DPGMM" avg_len = int(big_N.shape[0]/len(self.list_of_demonstrations)) DP_GMM_COMPONENTS = int(avg_len/constants.DPGMM_DIVISOR) print "L0", DP_GMM_COMPONENTS, "ALPHA: ", self.ALPHA_CP dpgmm = mixture.DPGMM(n_components = DP_GMM_COMPONENTS, covariance_type='diag', n_iter = 10000, alpha = self.ALPHA_CP, thresh= 1e-7) if self.fit_GMM: print "Init GMM" gmm = mixture.GMM(n_components = self.n_components_cp, covariance_type='full', n_iter=5000, thresh = 5e-5) if constants.REMOTE == 2: gmm = mixture.GMM(n_components = self.n_components_cp, covariance_type='full', thresh = 0.01) else: gmm = mixture.GMM(n_components = self.n_components_cp, covariance_type='full') if self.fit_GMM: print "Fitting GMM" start = time.time() gmm.fit(big_N) end = time.time() print "GMM Time:", end - start Y_gmm = gmm.predict(big_N) print "L0: Clusters in GMM", len(set(Y_gmm)) Y = Y_gmm if self.fit_DPGMM: print "Fitting DPGMM" start = time.time() dpgmm.fit(big_N) end = time.time() print "DPGMM Time:", end - start Y_dpgmm = dpgmm.predict(big_N) print "L0: Clusters in DP-GMM", len(set(Y_dpgmm)) Y = Y_dpgmm for w in range(len(Y) - 1): if Y[w] != Y[w + 1]: change_pt = big_N[w][self.X_dimension:] self.append_cp_array(utils.reshape(change_pt)) self.map_cp2frm[cp_index] = map_index2frm[w] self.map_cp2demonstrations[cp_index] = map_index2demonstration[w] self.list_of_cp.append(cp_index) cp_index += 1 print "Done with generating change points, " + str(cp_index)
def generate_change_points_2(self): """ Generates changespoints by clustering across demonstrations. """ cp_index = 0 i = 0 big_N = None map_index2demonstration = {} map_index2frm = {} size_of_X = self.data_X_size[self.list_of_demonstrations[0]] for demonstration in self.list_of_demonstrations: print demonstration N = self.data_N[demonstration] start, end = parser.get_start_end_annotations( constants.PATH_TO_DATA + constants.ANNOTATIONS_FOLDER + demonstration + "_" + constants.CAMERA + ".p" ) for j in range(N.shape[0]): map_index2demonstration[i] = demonstration map_index2frm[i] = start + j * self.sr i += 1 big_N = utils.safe_concatenate(big_N, N) print "Generated big_N" if constants.REMOTE == 1: if self.fit_GMM: gmm = mixture.GMM(n_components=self.n_components_cp, covariance_type="full", thresh=0.01) if self.fit_DPGMM: # dpgmm = mixture.DPGMM(n_components = 100, covariance_type='diag', n_iter = 10000, alpha = 100, thresh= 2e-4) # DO NOT FIDDLE WITH PARAMS WITHOUT CONSENT :) avg_len = int(big_N.shape[0] / len(self.list_of_demonstrations)) DP_GMM_COMPONENTS = int( avg_len / constants.DPGMM_DIVISOR ) # tuned with suturing experts only for kinematics print "L0 ", DP_GMM_COMPONENTS, "ALPHA: ", constants.ALPHA_ZW_CP dpgmm = mixture.DPGMM( n_components=DP_GMM_COMPONENTS, covariance_type="diag", n_iter=1000, alpha=constants.ALPHA_ZW_CP, thresh=1e-7, ) elif constants.REMOTE == 2: gmm = mixture.GMM(n_components=self.n_components_cp, covariance_type="full") else: gmm = mixture.GMM(n_components=self.n_components_cp, covariance_type="full") if self.fit_GMM: start = time.time() gmm.fit(big_N) end = time.time() "GMM time taken: ", str(end - start) Y_gmm = gmm.predict(big_N) print "L0: Clusters in GMM", len(set(Y_gmm)) Y = Y_gmm if self.fit_DPGMM: start = time.time() dpgmm.fit(big_N) end = time.time() "DP-GMM time taken: ", str(end - start) Y_dpgmm = dpgmm.predict(big_N) Y = Y_dpgmm print "L0: Clusters in DP-GMM", len(set(Y_dpgmm)) for w in range(len(Y) - 1): if Y[w] != Y[w + 1]: change_pt = big_N[w][size_of_X:] self.append_cp_array(change_pt) self.map_cp2frm[cp_index] = map_index2frm[w] self.map_cp2demonstrations[cp_index] = map_index2demonstration[w] self.list_of_cp.append(cp_index) cp_index += 1 print "Done with generating change points", len(self.list_of_cp)