def visualize_pascal(plot_probabilities=False): data = load_pascal('val') ds = PascalSegmentation() for x, y, f, sps in zip(data.X, data.Y, data.file_names, data.superpixels): fig, ax = plt.subplots(2, 3) ax = ax.ravel() image = ds.get_image(f) y_pixel = ds.get_ground_truth(f) x_raw = load_kraehenbuehl(f) boundary_image = mark_boundaries(image, sps) ax[0].imshow(image) ax[1].imshow(y_pixel, cmap=ds.cmap) ax[2].imshow(boundary_image) ax[3].imshow(np.argmax(x_raw, axis=-1), cmap=ds.cmap, vmin=0, vmax=256) ax[4].imshow(y[sps], cmap=ds.cmap, vmin=0, vmax=256) ax[5].imshow(np.argmax(x, axis=-1)[sps], cmap=ds.cmap, vmin=0, vmax=256) for a in ax: a.set_xticks(()) a.set_yticks(()) plt.savefig("figures_pascal_val/%s.png" % f, bbox_inches='tight') plt.close() if plot_probabilities: fig, ax = plt.subplots(3, 7) for k in range(21): ax.ravel()[k].matshow(x[:, :, k], vmin=0, vmax=1) for a in ax.ravel(): a.set_xticks(()) a.set_yticks(()) plt.savefig("figures_pascal_val/%s_prob.png" % f, bbox_inches='tight') plt.close() tracer()
def train_svm(C=0.1, grid=False): pascal = PascalSegmentation() files_train = pascal.get_split("kTrain") superpixels = [slic_n(pascal.get_image(f), n_superpixels=100, compactness=10) for f in files_train] bow = SiftBOW(pascal, n_words=1000, color_sift=True) data_train = bow.fit_transform(files_train, superpixels) data_train = add_global_descriptor(data_train) svm = LinearSVC(C=C, dual=False, class_weight='auto') chi2 = AdditiveChi2Sampler() X, y = np.vstack(data_train.X), np.hstack(data_train.Y) X = chi2.fit_transform(X) svm.fit(X, y) print(svm.score(X, y)) eval_on_sp(pascal, data_train, [svm.predict(chi2.transform(x)) for x in data_train.X], print_results=True) files_val = pascal.get_split("kVal") superpixels_val = [slic_n(pascal.get_image(f), n_superpixels=100, compactness=10) for f in files_val] data_val = bow.transform(files_val, superpixels_val) data_val = add_global_descriptor(data_val) eval_on_sp(pascal, data_val, [svm.predict(chi2.transform(x)) for x in data_val.X], print_results=True) tracer()
def load_pascal(which='train', year="2010", sp_type="slic", n_jobs=-1): pascal = PascalSegmentation() files = pascal.get_split(which=which, year=year) results = Parallel(n_jobs=n_jobs)(delayed(load_pascal_single)( f, which=which, sp_type=sp_type, pascal=pascal) for f in files) X, Y, superpixels, segments = zip(*results) if sp_type == "slic": return DataBunch(X, Y, files, superpixels) else: return HierarchicalDataBunch(X, Y, files, superpixels, segments)
def test_remove_small_segments(): pascal = PascalSegmentation() train_files = pascal.get_split() idx = 10 image = pascal.get_image(train_files[idx]) segments, superpixels = superpixels_segments(train_files[idx]) new_regions, correspondences = merge_small_sp(image, superpixels) new_counts = np.bincount(new_regions.ravel()) if np.any(new_counts < 50): raise ValueError("Stupid thing!")
def visualize_sps(): pascal = PascalSegmentation() train_files = pascal.get_split() for image_file in train_files: print(image_file) image = pascal.get_image(image_file) segments, superpixels = superpixels_segments(image_file) new_regions, correspondences = merge_small_sp(image, superpixels) clean_regions = morphological_clean_sp(image, new_regions, 4) imsave("segment_sp_fixed/%s.png" % image_file, mark_boundaries(image, clean_regions))
def load_pascal_pixelwise(which='train', year="2010"): pascal = PascalSegmentation() if which not in ["train", "val"]: raise ValueError("Expected 'which' to be 'train' or 'val', got %s." % which) split_file = pascal_path + "/ImageSets/Segmentation/%s.txt" % which files = np.loadtxt(split_file, dtype=np.str) files = [f for f in files if f.split("_")[0] <= year] X, Y = [], [] for f in files: X.append(load_kraehenbuehl(f)) Y.append(pascal.get_ground_truth(f)) return DataBunchNoSP(X, Y, files)
def visualize_segments(): pascal = PascalSegmentation() train_files = pascal.get_split() for image_file in train_files: print(image_file) image = pascal.get_image(image_file) segments, superpixels = superpixels_segments(image_file) new_regions, correspondences = merge_small_sp(image, superpixels) clean_regions = morphological_clean_sp(image, new_regions, 4) new_regions, correspondences = merge_small_sp(image, superpixels) clean_regions = morphological_clean_sp(image, new_regions, 4) marked = mark_boundaries(image, clean_regions) edges = create_segment_sp_graph(segments, clean_regions) edges = np.array(edges) n_segments = segments.shape[2] segment_centers = [ regionprops(segments.astype(np.int)[:, :, i], ['Centroid'])[0]['Centroid'] for i in range(n_segments) ] segment_centers = np.vstack(segment_centers)[:, ::-1] superpixel_centers = get_superpixel_centers(clean_regions) grr = min(n_segments, 10) fig, axes = plt.subplots(3, grr // 3, figsize=(30, 30)) for i, ax in enumerate(axes.ravel()): ax.imshow(mark_boundaries(marked, segments[:, :, i], (1, 0, 0))) ax.scatter(segment_centers[:, 0], segment_centers[:, 1], color='red') ax.scatter(superpixel_centers[:, 0], superpixel_centers[:, 1], color='blue') this_edges = edges[edges[:, 1] == i] for edge in this_edges: ax.plot([ superpixel_centers[edge[0]][0], segment_centers[edge[1]][0] ], [ superpixel_centers[edge[0]][1], segment_centers[edge[1]][1] ], c='black') ax.set_xlim(0, superpixels.shape[1]) ax.set_ylim(superpixels.shape[0], 0) ax.set_xticks(()) ax.set_yticks(()) #plt.show() #imsave("segments_test/%s.png" % image_file, marked) plt.savefig("segments_test/%s.png" % image_file) plt.close()
def train_svm(C=0.1, grid=False): ds = PascalSegmentation() svm = LinearSVC(C=C, dual=False, class_weight='auto') if grid: data_train = load_pascal("kTrain") X, y = shuffle(data_train.X, data_train.Y) # prepare leave-one-label-out by assigning labels to images image_indicators = np.hstack([np.repeat(i, len(x)) for i, x in enumerate(X)]) # go down to only 5 "folds" labels = image_indicators % 5 X, y = np.vstack(X), np.hstack(y) cv = LeavePLabelOut(labels=labels, p=1) param_grid = {'C': 10. ** np.arange(-3, 3)} scorer = Scorer(recall_score, average="macro") grid_search = GridSearchCV(svm, param_grid=param_grid, cv=cv, verbose=10, scoring=scorer, n_jobs=-1) grid_search.fit(X, y) else: data_train = load_pascal("train") X, y = np.vstack(data_train.X), np.hstack(data_train.Y) svm.fit(X, y) print(svm.score(X, y)) eval_on_sp(ds, data_train, [svm.predict(x) for x in data_train.X], print_results=True) data_val = load_pascal("val") eval_on_sp(ds, data_val, [svm.predict(x) for x in data_val.X], print_results=True)
def main(): from pascal.pascal_helpers import load_pascal from datasets.pascal import PascalSegmentation from utils import add_edges from scipy.misc import imsave from skimage.segmentation import mark_boundaries ds = PascalSegmentation() data = load_pascal("train1") data = add_edges(data, independent=False) # X, Y, image_names, images, all_superpixels = load_data( # "train", independent=False) for x, name, sps in zip(data.X, data.file_names, data.superpixels): segments = get_km_segments(x, ds.get_image(name), sps, n_segments=25) boundary_image = mark_boundaries(mark_boundaries(ds.get_image(name), sps), segments[sps], color=[1, 0, 0]) imsave("hierarchy_sp_own_25/%s.png" % name, boundary_image)
def eval_segment_best_possible(): ds = PascalSegmentation() print("loading") data = load_pascal('train') print("getting edges") data = add_edges(data) print("computing segments") segments = [get_km_segments(x, ds.get_image(image_name), sps, n_segments=25) for x, image_name, sps in zip(data.X, data.file_names, data.superpixels)] print("combining superpixels") segments = [seg[sp] for seg, sp in zip(segments, data.superpixels)] predictions = [gt_in_sp(ds, f, seg)[seg] for seg, f in zip(segments, data.file_names)] Y_true = [ds.get_ground_truth(f) for f in data.file_names] hamming, jaccard = eval_on_pixels(ds, Y_true, predictions, print_results=True) tracer()
def visualize_segments(): pascal = PascalSegmentation() train_files = pascal.get_split() for image_file in train_files: print(image_file) image = pascal.get_image(image_file) segments, superpixels = superpixels_segments(image_file) new_regions, correspondences = merge_small_sp(image, superpixels) clean_regions = morphological_clean_sp(image, new_regions, 4) new_regions, correspondences = merge_small_sp(image, superpixels) clean_regions = morphological_clean_sp(image, new_regions, 4) marked = mark_boundaries(image, clean_regions) edges = create_segment_sp_graph(segments, clean_regions) edges = np.array(edges) n_segments = segments.shape[2] segment_centers = [regionprops(segments.astype(np.int)[:, :, i], ['Centroid'])[0]['Centroid'] for i in range(n_segments)] segment_centers = np.vstack(segment_centers)[:, ::-1] superpixel_centers = get_superpixel_centers(clean_regions) grr = min(n_segments, 10) fig, axes = plt.subplots(3, grr // 3, figsize=(30, 30)) for i, ax in enumerate(axes.ravel()): ax.imshow(mark_boundaries(marked, segments[:, :, i], (1, 0, 0))) ax.scatter(segment_centers[:, 0], segment_centers[:, 1], color='red') ax.scatter(superpixel_centers[:, 0], superpixel_centers[:, 1], color='blue') this_edges = edges[edges[:, 1] == i] for edge in this_edges: ax.plot([superpixel_centers[edge[0]][0], segment_centers[edge[1]][0]], [superpixel_centers[edge[0]][1], segment_centers[edge[1]][1]], c='black') ax.set_xlim(0, superpixels.shape[1]) ax.set_ylim(superpixels.shape[0], 0) ax.set_xticks(()) ax.set_yticks(()) #plt.show() #imsave("segments_test/%s.png" % image_file, marked) plt.savefig("segments_test/%s.png" % image_file) plt.close()
def main(): from pascal.pascal_helpers import load_pascal from datasets.pascal import PascalSegmentation from utils import add_edges from scipy.misc import imsave from skimage.segmentation import mark_boundaries ds = PascalSegmentation() data = load_pascal("train1") data = add_edges(data, independent=False) #X, Y, image_names, images, all_superpixels = load_data( #"train", independent=False) for x, name, sps in zip(data.X, data.file_names, data.superpixels): segments = get_km_segments(x, ds.get_image(name), sps, n_segments=25) boundary_image = mark_boundaries(mark_boundaries( ds.get_image(name), sps), segments[sps], color=[1, 0, 0]) imsave("hierarchy_sp_own_25/%s.png" % name, boundary_image)
def main(): ds = PascalSegmentation() # load training data edge_type = "pairwise" which = "kTrain" data_train = load_pascal(which=which, sp_type="cpmc") data_train = add_edges(data_train, edge_type) data_train = add_edge_features(ds, data_train) data_train = discard_void(ds, data_train, ds.void_label) X, Y = data_train.X, data_train.Y class_weights = 1. / np.bincount(np.hstack(Y)) class_weights *= 21. / np.sum(class_weights) model = crfs.EdgeFeatureGraphCRF(class_weight=class_weights, symmetric_edge_features=[0, 1], antisymmetric_edge_features=[2], inference_method='qpbo') ssvm = learners.NSlackSSVM(model, C=0.01, n_jobs=-1) ssvm.fit(X, Y)
def train_svm(C=0.1, grid=False): pascal = PascalSegmentation() files_train = pascal.get_split("kTrain") superpixels = [ slic_n(pascal.get_image(f), n_superpixels=100, compactness=10) for f in files_train ] bow = SiftBOW(pascal, n_words=1000, color_sift=True) data_train = bow.fit_transform(files_train, superpixels) data_train = add_global_descriptor(data_train) svm = LinearSVC(C=C, dual=False, class_weight='auto') chi2 = AdditiveChi2Sampler() X, y = np.vstack(data_train.X), np.hstack(data_train.Y) X = chi2.fit_transform(X) svm.fit(X, y) print(svm.score(X, y)) eval_on_sp(pascal, data_train, [svm.predict(chi2.transform(x)) for x in data_train.X], print_results=True) files_val = pascal.get_split("kVal") superpixels_val = [ slic_n(pascal.get_image(f), n_superpixels=100, compactness=10) for f in files_val ] data_val = bow.transform(files_val, superpixels_val) data_val = add_global_descriptor(data_val) eval_on_sp(pascal, data_val, [svm.predict(chi2.transform(x)) for x in data_val.X], print_results=True) tracer()
def main(C=1, test=False): ds = PascalSegmentation() # load training data edge_type = "pairwise" if test: which = "train" else: which = "kTrain" data_train = load_pascal(which=which, sp_type="cpmc") data_train = add_edges(data_train, edge_type) data_train = add_edge_features(ds, data_train) data_train = discard_void(ds, data_train, ds.void_label) print("number of samples: %s" % len(data_train.X)) class_weights = 1. / np.bincount(np.hstack(data_train.Y)) class_weights *= 21. / np.sum(class_weights) print(class_weights) #model = crfs.GraphCRF(n_states=n_states, #n_features=data_train.X[0][0].shape[1], #inference_method='qpbo', class_weight=class_weights) model = crfs.EdgeFeatureGraphCRF(inference_method='qpbo', class_weight=class_weights, symmetric_edge_features=[0, 1], antisymmetric_edge_features=[2]) experiment_name = "cpmc_edge_features_trainval_new_%f" % C #warm_start = True warm_start = False ssvm = learners.OneSlackSSVM(model, verbose=2, C=C, max_iter=100000, n_jobs=-1, tol=0.0001, show_loss_every=50, inference_cache=50, cache_tol='auto', logger=SaveLogger(experiment_name + ".pickle", save_every=100), inactive_threshold=1e-5, break_on_bad=False, inactive_window=50, switch_to=None) #ssvm = learners.SubgradientSSVM( #model, verbose=3, C=C, max_iter=10000, n_jobs=-1, show_loss_every=10, #logger=SaveLogger(experiment_name + ".pickle", save_every=10), #momentum=0, learning_rate=0.1, decay_exponent=1, decay_t0=100) if warm_start: ssvm = SaveLogger(experiment_name + ".pickle").load() ssvm.logger = SaveLogger(file_name=experiment_name + "_refit.pickle", save_every=10) #ssvm.learning_rate = 0.000001 ssvm.model.inference_method = 'ad3bb' #ssvm.n_jobs = 1 ssvm.fit(data_train.X, data_train.Y, warm_start=warm_start) return print("fit finished!") if test: data_val = load_pascal('val') else: data_val = load_pascal('kVal') data_val = add_edges(data_val, edge_type) data_val = add_edge_features(ds, data_val, more_colors=True) eval_on_sp(ds, data_val, ssvm.predict(data_val.X), print_results=True)
def main(): argv = sys.argv print("loading %s ..." % argv[1]) ssvm = SaveLogger(file_name=argv[1]).load() if hasattr(ssvm, 'problem'): ssvm.model = ssvm.problem print(ssvm) if hasattr(ssvm, 'base_ssvm'): ssvm = ssvm.base_ssvm print("Iterations: %d" % len(ssvm.objective_curve_)) print("Objective: %f" % ssvm.objective_curve_[-1]) inference_run = None if hasattr(ssvm, 'cached_constraint_'): inference_run = ~np.array(ssvm.cached_constraint_) print("Gap: %f" % (np.array(ssvm.primal_objective_curve_)[inference_run][-1] - ssvm.objective_curve_[-1])) if len(argv) <= 2: argv.append("acc") if len(argv) <= 3: dataset = 'nyu' else: dataset = argv[3] if argv[2] == 'acc': ssvm.n_jobs = 1 for data_str, title in zip(["train", "val"], ["TRAINING SET", "VALIDATION SET"]): print(title) edge_type = "pairwise" if dataset == 'msrc': ds = MSRC21Dataset() data = msrc_helpers.load_data(data_str, which="piecewise_new") #data = add_kraehenbuehl_features(data, which="train_30px") data = msrc_helpers.add_kraehenbuehl_features(data, which="train") elif dataset == 'pascal': ds = PascalSegmentation() data = pascal_helpers.load_pascal(data_str, sp_type="cpmc") #data = pascal_helpers.load_pascal(data_str) elif dataset == 'nyu': ds = NYUSegmentation() data = nyu_helpers.load_nyu(data_str, n_sp=500, sp='rgbd') else: raise ValueError("Excepted dataset to be 'nyu', 'pascal' or 'msrc'," " got %s." % dataset) if type(ssvm.model).__name__ == "LatentNodeCRF": print("making data hierarchical") data = pascal_helpers.make_cpmc_hierarchy(ds, data) #data = make_hierarchical_data( #ds, data, lateral=True, latent=True, latent_lateral=False, #add_edge_features=False) else: data = add_edges(data, edge_type) if type(ssvm.model).__name__ == 'EdgeFeatureGraphCRF': data = add_edge_features(ds, data, depth_diff=True, normal_angles=True) if type(ssvm.model).__name__ == "EdgeFeatureLatentNodeCRF": data = add_edge_features(ds, data) data = make_hierarchical_data( ds, data, lateral=True, latent=True, latent_lateral=False, add_edge_features=True) #ssvm.model.inference_method = "qpbo" Y_pred = ssvm.predict(data.X) if isinstance(ssvm.model, LatentNodeCRF): Y_pred = [ssvm.model.label_from_latent(h) for h in Y_pred] Y_flat = np.hstack(data.Y) print("superpixel accuracy: %.2f" % (np.mean((np.hstack(Y_pred) == Y_flat)[Y_flat != ds.void_label]) * 100)) if dataset == 'msrc': res = msrc_helpers.eval_on_pixels(data, Y_pred, print_results=True) print("global: %.2f, average: %.2f" % (res['global'] * 100, res['average'] * 100)) #msrc_helpers.plot_confusion_matrix(res['confusion']) else: hamming, jaccard = eval_on_sp(ds, data, Y_pred, print_results=True) print("Jaccard: %.2f, Hamming: %.2f" % (jaccard.mean(), hamming.mean())) plt.show() elif argv[2] == 'plot': data_str = 'val' if len(argv) <= 4: raise ValueError("Need a folder name for plotting.") if dataset == "msrc": ds = MSRC21Dataset() data = msrc_helpers.load_data(data_str, which="piecewise") data = add_edges(data, independent=False) data = msrc_helpers.add_kraehenbuehl_features( data, which="train_30px") data = msrc_helpers.add_kraehenbuehl_features( data, which="train") elif dataset == "pascal": ds = PascalSegmentation() data = pascal_helpers.load_pascal("val") data = add_edges(data) elif dataset == "nyu": ds = NYUSegmentation() data = nyu_helpers.load_nyu("test") data = add_edges(data) if type(ssvm.model).__name__ == 'EdgeFeatureGraphCRF': data = add_edge_features(ds, data, depth_diff=True, normal_angles=True) Y_pred = ssvm.predict(data.X) plot_results(ds, data, Y_pred, argv[4])
def eval_spixel_best_possible(): data = load_pascal('kTrain', sp_type='cpmc') pascal = PascalSegmentation() hamming, jaccard = eval_on_sp(pascal, data, data.Y, print_results=True)
def get_kraehenbuehl_pot_sp(filename, superpixels): probs = load_kraehenbuehl(filename) ds = PascalSegmentation() return probabilities_on_sp(ds, probs, superpixels)
def svm_on_segments(C=.1, learning_rate=.001, subgradient=False): data_file = "data_train_XY.pickle" ds = PascalSegmentation() if os.path.exists(data_file): X_, Y_ = cPickle.load(open(data_file)) else: # load and prepare data data_train = load_pascal("train", sp_type="cpmc") data_train = make_cpmc_hierarchy(ds, data_train) data_train = discard_void(ds, data_train) X_, Y_ = data_train.X, data_train.Y cPickle.dump((X_, Y_), open(data_file, 'wb'), -1) class_weights = 1. / np.bincount(np.hstack(Y_)) class_weights *= 21. / np.sum(class_weights) experiment_name = ("latent_25_cpmc_%f_qpbo_n_slack_blub3" % C) logger = SaveLogger(experiment_name + ".pickle", save_every=10) model = LatentNodeCRF(n_hidden_states=25, inference_method='qpbo', class_weight=class_weights, latent_node_features=False) if subgradient: ssvm = learners.LatentSubgradientSSVM(model, C=C, verbose=1, show_loss_every=10, logger=logger, n_jobs=-1, learning_rate=learning_rate, decay_exponent=1, momentum=0., max_iter=100000, decay_t0=100) else: latent_logger = SaveLogger("lssvm_" + experiment_name + "_%d.pickle", save_every=1) #base_ssvm = learners.OneSlackSSVM( #model, verbose=2, C=C, max_iter=100, n_jobs=-1, tol=0.001, #show_loss_every=200, inference_cache=50, logger=logger, #cache_tol='auto', inactive_threshold=1e-5, break_on_bad=False, #switch_to=('ogm', {'alg': 'dd'})) base_ssvm = learners.NSlackSSVM(model, verbose=4, C=C, n_jobs=-1, tol=0.1, show_loss_every=20, logger=logger, inactive_threshold=1e-8, break_on_bad=False, batch_size=36, inactive_window=10, switch_to=('ad3', { 'branch_and_bound': True })) ssvm = learners.LatentSSVM(base_ssvm, logger=latent_logger, latent_iter=3) #warm_start = True warm_start = False if warm_start: ssvm = logger.load() ssvm.logger = SaveLogger(experiment_name + "_retrain.pickle", save_every=10) ssvm.max_iter = 10000 ssvm.decay_exponent = 1 #ssvm.decay_t0 = 1000 #ssvm.learning_rate = 0.00001 #ssvm.momentum = 0 X_, Y_ = shuffle(X_, Y_) #ssvm.fit(data_train.X, data_train.Y) ssvm.fit(X_, Y_) #H_init = [np.hstack([y, np.random.randint(21, 26)]) for y in Y_] #ssvm.fit(X_, Y_, H_init=H_init) print("fit finished!")