def test_logging(): iris = load_iris() X, y = iris.data, iris.target X_ = [(np.atleast_2d(x), np.empty((0, 2), dtype=np.int)) for x in X] Y = y.reshape(-1, 1) X_train, X_test, y_train, y_test = train_test_split(X_, Y, random_state=1) _, file_name = mkstemp() pbl = GraphCRF(n_features=4, n_states=3, inference_method=inference_method) logger = SaveLogger(file_name) svm = NSlackSSVM(pbl, C=100, n_jobs=1, logger=logger) svm.fit(X_train, y_train) score_current = svm.score(X_test, y_test) score_auto_saved = logger.load().score(X_test, y_test) alt_file_name = file_name + "alt" logger.save(svm, alt_file_name) logger.file_name = alt_file_name logger.load() score_manual_saved = logger.load().score(X_test, y_test) assert_less(.97, score_current) assert_less(.97, score_auto_saved) assert_less(.97, score_manual_saved) assert_almost_equal(score_auto_saved, score_manual_saved)
def test_n_slack_svm_as_crf_pickling(): iris = load_iris() X, y = iris.data, iris.target X_ = [(np.atleast_2d(x), np.empty((0, 2), dtype=np.int)) for x in X] Y = y.reshape(-1, 1) X_train, X_test, y_train, y_test = train_test_split(X_, Y, random_state=1) _, file_name = mkstemp() pbl = GraphCRF(n_features=4, n_states=3, inference_method='lp') logger = SaveLogger(file_name) svm = NSlackSSVM(pbl, C=100, n_jobs=1, logger=logger) svm.fit(X_train, y_train) assert_less(.97, svm.score(X_test, y_test)) assert_less(.97, logger.load().score(X_test, y_test))
svm.fit(X_train_, y_train) # Now, use a latent-variabile CRF model with SVM training. # 5 states per label is enough capacity to encode the 5 digit classes. latent_pbl = LatentGraphCRF(n_states_per_label=5, inference_method='unary') base_ssvm = NSlackSSVM(latent_pbl, C=1, tol=.01, inactive_threshold=1e-3, batch_size=10) latent_svm = LatentSSVM(base_ssvm=base_ssvm, latent_iter=2) latent_svm.fit(X_train_, y_train) print("Score with binary SVM:") print("Train: {:2.2f}".format(svm.score(X_train_, y_train))) print("Test: {:2.2f}".format(svm.score(X_test_, y_test))) print("Score with latent SVM:") print("Train: {:2.2f}".format(latent_svm.score(X_train_, y_train))) print("Test: {:2.2f}".format(latent_svm.score(X_test_, y_test))) h_pred = np.hstack(latent_svm.predict_latent(X_test_)) print("Latent class counts: %s" % repr(np.bincount(h_pred))) # plot first few digits from each latent class plt.figure(figsize=(3, 5)) plt.suptitle("Example digits from each of\nthe ten latent classes.") n_latent_classes = 10 n_examples = 7
# flatten X and Y X_flat = [x.reshape(-1, 1).astype(np.float) for x in X] Y_flat = [y.ravel() for y in Y] # first, use standard graph CRF. Can't do much, high loss. crf = GraphCRF() svm = NSlackSSVM(model=crf, max_iter=200, C=1, n_jobs=1) G = [make_grid_edges(x) for x in X] X_grid_edges = list(zip(X_flat, G)) svm.fit(X_grid_edges, Y_flat) plot_boxes(svm.predict(X_grid_edges), title="Non-latent SSVM predictions") print("Training score binary grid CRF: %f" % svm.score(X_grid_edges, Y_flat)) # using one latent variable for each 2x2 rectangle latent_crf = LatentNodeCRF(n_labels=2, n_features=1, n_hidden_states=2, inference_method='lp') ssvm = OneSlackSSVM(model=latent_crf, max_iter=200, C=100, n_jobs=-1, show_loss_every=10, inference_cache=50) latent_svm = LatentSSVM(ssvm) # make edges for hidden states: edges = [] node_indices = np.arange(4 * 4).reshape(4, 4) for i, (x, y) in enumerate(itertools.product([0, 2], repeat=2)): for j in range(x, x + 2): for k in range(y, y + 2):
# flatten X and Y X_flat = [x.reshape(-1, 1).astype(np.float) for x in X] Y_flat = [y.ravel() for y in Y] # first, use standard graph CRF. Can't do much, high loss. crf = GraphCRF() svm = NSlackSSVM(model=crf, max_iter=200, C=1, n_jobs=1) G = [make_grid_edges(x) for x in X] asdf = zip(X_flat, G) svm.fit(asdf, Y_flat) plot_boxes(svm.predict(asdf), title="Non-latent SSVM predictions") print("Training score binary grid CRF: %f" % svm.score(asdf, Y_flat)) # using one latent variable for each 2x2 rectangle latent_crf = LatentNodeCRF(n_labels=2, n_features=1, n_hidden_states=2, inference_method='lp') ssvm = OneSlackSSVM(model=latent_crf, max_iter=200, C=100, verbose=1, n_jobs=-1, show_loss_every=10, inference_cache=50) latent_svm = LatentSSVM(ssvm) # make edges for hidden states: edges = [] node_indices = np.arange(4 * 4).reshape(4, 4) for i, (x, y) in enumerate(itertools.product([0, 2], repeat=2)): for j in xrange(x, x + 2): for k in xrange(y, y + 2):
X, Y = make_simple_2x2(seed=1) # flatten X and Y X_flat = [x.reshape(-1, 1).astype(np.float) for x in X] Y_flat = [y.ravel() for y in Y] # first, use standard graph CRF. Can't do much, high loss. crf = GraphCRF() svm = NSlackSSVM(model=crf, max_iter=200, C=1, n_jobs=1) G = [make_grid_edges(x) for x in X] asdf = zip(X_flat, G) svm.fit(asdf, Y_flat) plot_boxes(svm.predict(asdf), title="Non-latent SSVM predictions") print("Training score binary grid CRF: %f" % svm.score(asdf, Y_flat)) # using one latent variable for each 2x2 rectangle latent_crf = LatentNodeCRF(n_labels=2, n_features=1, n_hidden_states=2, inference_method='lp') ssvm = OneSlackSSVM(model=latent_crf, max_iter=200, C=100, verbose=1, n_jobs=-1, show_loss_every=10, inference_cache=50) latent_svm = LatentSSVM(ssvm)
pbl = BinaryClf() n_slack_svm = NSlackSSVM(pbl, C=10, batch_size=-1) one_slack_svm = OneSlackSSVM(pbl, C=10, tol=0.1) subgradient_svm = SubgradientSSVM(pbl, C=10, learning_rate=0.1, max_iter=100, batch_size=10) # we add a constant 1 feature for the bias X_train_bias = np.hstack([X_train, np.ones((X_train.shape[0], 1))]) X_test_bias = np.hstack([X_test, np.ones((X_test.shape[0], 1))]) # n-slack cutting plane ssvm start = time() n_slack_svm.fit(X_train_bias, y_train) time_n_slack_svm = time() - start acc_n_slack = n_slack_svm.score(X_test_bias, y_test) print("Score with pystruct n-slack ssvm: %f (took %f seconds)" % (acc_n_slack, time_n_slack_svm)) ## 1-slack cutting plane ssvm start = time() one_slack_svm.fit(X_train_bias, y_train) time_one_slack_svm = time() - start acc_one_slack = one_slack_svm.score(X_test_bias, y_test) print("Score with pystruct 1-slack ssvm: %f (took %f seconds)" % (acc_one_slack, time_one_slack_svm)) # online subgradient ssvm start = time() subgradient_svm.fit(X_train_bias, y_train) time_subgradient_svm = time() - start
one_slack_svm = OneSlackSSVM(pbl, C=10, tol=0.1) subgradient_svm = SubgradientSSVM(pbl, C=10, learning_rate=0.1, max_iter=100, batch_size=10) # we add a constant 1 feature for the bias X_train_bias = np.hstack([X_train, np.ones((X_train.shape[0], 1))]) X_test_bias = np.hstack([X_test, np.ones((X_test.shape[0], 1))]) # n-slack cutting plane ssvm start = time() n_slack_svm.fit(X_train_bias, y_train) time_n_slack_svm = time() - start acc_n_slack = n_slack_svm.score(X_test_bias, y_test) print("Score with pystruct n-slack ssvm: %f (took %f seconds)" % (acc_n_slack, time_n_slack_svm)) ## 1-slack cutting plane ssvm start = time() one_slack_svm.fit(X_train_bias, y_train) time_one_slack_svm = time() - start acc_one_slack = one_slack_svm.score(X_test_bias, y_test) print("Score with pystruct 1-slack ssvm: %f (took %f seconds)" % (acc_one_slack, time_one_slack_svm)) # online subgradient ssvm start = time() subgradient_svm.fit(X_train_bias, y_train) time_subgradient_svm = time() - start
def _start(self, out_dir, calcium_data=None, test_size=.75, feats_unary='feats_pm', feats_pairwise='feats_xcorr_green', which_gt='gt', test_gt=None, dump_ssvm=False, which_ssvm='nslack', ssvm_c=0.3, ssvm_tol=0.001, ssvm_iter=30, verbosity=0, which_solver=('ogm', {'alg': 'gc'}), no_class_weights=False, dummy_data=False, only_nice_volumes=False): self.stats = {'feats_unary' : feats_unary, 'feats_pairwise' : feats_pairwise, 'which_gt' : which_gt, 'test_size' : test_size} print '===== CRF =====' print "* build edge graph" img_x, img_y = calcium_data.subvolumes[0].dims edges = create_2d_edge_graph(img_x, img_y) if only_nice_volumes: print "* choosing only nice volumes" subvolumes = [sub for sub in calcium_data.subvolumes if sub.is_nice()] else: subvolumes = calcium_data.subvolumes print "* load features and labels" X, Y = calcium_data._create_features_and_labels_ssvm(subvolumes, which_gt, feats_unary, feats_pairwise, edges, verbosity) if len(subvolumes) == 2: print '[info] only 2 volumes present in data, using one for testing the other for training' X_train, X_test, Y_train, Y_test = [X[0]], [X[1]], [Y[0]], [Y[1]] else: X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=test_size) if no_class_weights: print "* create graphical model" crf_graph = EdgeFeatureGraphCRF(inference_method=which_solver) self.class_weights_str = 'no' else: print "* calculate class weights: number of instances in X: %s" % len(X) class_weights = compute_class_weights(Y) print "* create graphical model" crf_graph = EdgeFeatureGraphCRF(inference_method=which_solver, class_weight=class_weights) self.class_weights_str = 'yes' print '===== SSVM =====' if dump_ssvm: print '* setting up logger' logger = pystruct_logger(out_dir + 'ssvm_model.logger', save_every=10) else: logger = None if which_ssvm == 'nslack': print "* creating NSlackSSVM" model = NSlackSSVM(crf_graph, verbose=verbosity, C=ssvm_c, max_iter=ssvm_iter, n_jobs=-1, tol=ssvm_tol, show_loss_every=1, inactive_threshold=1e-3, inactive_window=10, batch_size=1000, logger=logger) ssvm_name = 'NSlackSSVM' if which_ssvm == 'oneslack': print "* creating OneSlackSSVM" model = OneSlackSSVM(crf_graph, verbose=verbosity, C=ssvm_c, max_iter=ssvm_iter, n_jobs=-1, tol=ssvm_tol, show_loss_every=1, inactive_threshold=1e-3, inactive_window=10, logger=logger) ssvm_name = 'OneSlackSSVM' if which_ssvm == 'frankwolfe': print "* creating FrankWolfeSSVM" model = FrankWolfeSSVM(crf_graph, verbose=verbosity, C=ssvm_c, max_iter=ssvm_iter, tol=ssvm_tol, line_search=True, check_dual_every=10, do_averaging=True, sample_method='perm', random_state=None, logger=logger) ssvm_name = 'FrankWolfeSSVM' print '* fitting model...' model.fit(X_train, Y_train) print '* scoring model...' self.score = model.score(X_test, Y_test) print '* saving object properties' self.crf_graph = crf_graph self.model = model self.X = X self.Y = Y self.edges = edges self.X_train, self.X_test, self.Y_train, self.Y_test = X_train, X_test, Y_train, Y_test self.img_x, self.img_y = img_x, img_y self.subvolumes = subvolumes self.ssvm_name = ssvm_name print '--> model trained'
svm = NSlackSSVM(pbl, C=1, batch_size=-1) svm.fit(X_train_, y_train) # Now, use a latent-variabile CRF model with SVM training. # 5 states per label is enough capacity to encode the 5 digit classes. latent_pbl = LatentGraphCRF(n_states_per_label=5, inference_method='unary') base_ssvm = NSlackSSVM(latent_pbl, C=1, tol=.01, inactive_threshold=1e-3, batch_size=10) latent_svm = LatentSSVM(base_ssvm=base_ssvm, latent_iter=2) latent_svm.fit(X_train_, y_train) print("Score with binary SVM:") print("Train: {:2.2f}".format(svm.score(X_train_, y_train))) print("Test: {:2.2f}".format(svm.score(X_test_, y_test))) print("Score with latent SVM:") print("Train: {:2.2f}".format(latent_svm.score(X_train_, y_train))) print("Test: {:2.2f}".format(latent_svm.score(X_test_, y_test))) h_pred = np.hstack(latent_svm.predict_latent(X_test_)) print("Latent class counts: %s" % repr(np.bincount(h_pred))) # plot first few digits from each latent class plt.figure(figsize=(3, 5)) plt.suptitle("Example digits from each of\nthe ten latent classes.") n_latent_classes = 10 n_examples = 7
def test_final_segmentations(out_dir): test_size = .75 #ssvm_c = [5, .3, .1, .01, .001] ssvm_c = [50] ssvm_tol = 0.001 ssvm_iter = 50 which_ssvm = 'nslack' which_solver = ('ogm', {'alg': 'gc'}) which_combo = which_ssvm + '_ogm-' + which_solver[1]['alg'] out_dir = join_path(out_dir, 'test_final_segmentations', which_combo) create_out_path(out_dir, except_on_exist=False) cd = JaneliaData(dummy_data=False, only_nice_volumes=True) subs = cd.subvolumes demo_volumes = ['an197522_2013_03_10_13002', 'an197522_2013_03_08_06003', 'an229719_2013_12_05_03004', 'an229719_2013_12_05_07003', 'an229719_2013_12_05_08004'] img_x, img_y = cd.subvolumes[0].dims edges = create_2d_edge_graph(img_x, img_y) feats_unaries = ['feats_pm_++', 'feats_pm_+++', 'feats_pm_+++x'] feats_pairwise = 'feats_xcorr_green' train_labelings = ['gt_active_++', 'gt_active_+++', 'gt_active_+++x'] test_labelings = ['gt_active_++', 'gt_active_+++', 'gt_active_+++x'] failed_combos = [] sp = ShuffleSplit(len(subs), random_state=42, test_size=test_size) train_idxs, test_idxs = next(iter(sp)) for feats_unary in feats_unaries: X = ssvm_construct_X(subs, feats_unary, feats_pairwise, edges) for train_labeling in train_labelings: Y_full_train = ssvm_construct_Y(train_labeling, subs) class_weights = compute_class_weights(Y_full_train) crf_graph = EdgeFeatureGraphCRF(inference_method=which_solver, class_weight=class_weights) for test_labeling in test_labelings: X_train = [X[i] for i in train_idxs] Y_train = [Y_full_train[i] for i in train_idxs] Y_full_test = ssvm_construct_Y(test_labeling, subs) X_test = [X[i] for i in test_idxs] Y_test = [Y_full_test[i] for i in test_idxs] for c in ssvm_c: try: test_name = '%s_u=%s_p=%s_train-gt=%s_test-gt=%s_C=%.1e' % (which_ssvm, feats_unary, feats_pairwise, train_labeling, test_labeling, c) test_out_dir = join_path(out_dir, test_name) create_out_path(test_out_dir, except_on_exist=True) print '\n==================================================================' print '-->' + test_name except IOError as e: print e continue try: logger = pystruct_logger(join_path(test_out_dir, 'ssvm_model.logger'), save_every=10) if which_ssvm == 'nslack': print "* creating NSlackSSVM" model = NSlackSSVM(crf_graph, verbose=1, C=c, max_iter=ssvm_iter, n_jobs=-1, tol=ssvm_tol, show_loss_every=1, inactive_threshold=1e-3, inactive_window=10, batch_size=1000, logger=logger) ssvm_name = 'NSlackSSVM' if which_ssvm == 'oneslack': print "* creating OneSlackSSVM" model = OneSlackSSVM(crf_graph, verbose=1, C=c, max_iter=ssvm_iter, n_jobs=-1, tol=ssvm_tol, show_loss_every=1, inactive_threshold=1e-3, inactive_window=10, logger=logger) ssvm_name = 'OneSlackSSVM' if which_ssvm == 'frankwolfe': print "* creating FrankWolfeSSVM" model = FrankWolfeSSVM(crf_graph, verbose=1, C=c, max_iter=ssvm_iter, tol=ssvm_tol, line_search=True, check_dual_every=10, do_averaging=True, sample_method='perm', random_state=None, logger=logger) ssvm_name = 'FrankWolfeSSVM' print '* fitting model...' model.fit(X_train, Y_train) print '--> model trained' print '* scoring model...' score = model.score(X_test, Y_test) print score print '* saving results for test run' fig, _ = ssvm_plot_learning(model, title=test_name, time=False) fig.savefig(join_path(test_out_dir, 'ssvm_plot_learning.png'), dpi=300) plt.close('all') print '--> saved learning plots' ### save results to JSON ### with open(join_path(test_out_dir, 'ssvm_stats.json'), 'w') as json_out: stats = { 'test_accuracy' : float(score), 'test_size' : float(test_size), 'ssvm_c' : float(c), 'ssvm_iter' : ssvm_iter, 'ssvm_tol' : float(ssvm_tol), 'feats_unary' : feats_unary, 'feats_pairwise' : feats_pairwise, 'which_ssvm' : which_ssvm, 'which_solver' : which_solver[0], 'which_alg' : which_solver[1]['alg'], 'train_labeling' : train_labeling, 'test_labeling' : test_labeling } json_out.write(json.dumps(stats, indent=0, sort_keys=True)) print '--> saved SSVM stats to JSON' except: print 'Combo %s failed, removing its out folder.' % test_out_dir failed_combos.append(test_out_dir) os.rmdir(test_out_dir) create_out_path('FAILED_' + test_out_dir) continue print '--> saving vizzz for demo volumes and computing scores...' demo_out_dir = join_path(test_out_dir, 'demo_vols') create_out_path(demo_out_dir, except_on_exist=True) demo_scores = {} for demo_sub in demo_volumes: demo_sub = cd.get_subvolume_by_name(demo_sub) print ' -', demo_sub.name unary_feats = demo_sub.get(feats_unary) pairwise_feats = demo_sub.get(feats_pairwise) x = (unary_feats, edges, pairwise_feats) pred = model.predict([x]) pred = pred[0].reshape(img_x, img_y) gt_train = demo_sub.get(train_labeling).astype('uint8') gt_test = demo_sub.get(test_labeling).astype('uint8') rgb_vis_train = plot_it_like_ferran(gt_train, pred) rgb_vis_test = plot_it_like_ferran(gt_test, pred) plt.imsave(join_path(demo_out_dir, '%s_ssvm_prediction.png' % demo_sub.name), pred) plt.imsave(join_path(demo_out_dir, '%s_%s.png' % (demo_sub.name, train_labeling)), gt_train) plt.imsave(join_path(demo_out_dir, '%s_%s.png' % (demo_sub.name, test_labeling)), gt_test) plt.imsave(join_path(demo_out_dir, '%s_rgb_vis_train.png' % demo_sub.name), rgb_vis_train) plt.imsave(join_path(demo_out_dir, '%s_rgb_vis_test.png' % demo_sub.name), rgb_vis_test) labeled_pm_ax = demo_sub.plot_activity_labels(image=pred, cmap='gray') plt.savefig(join_path(demo_out_dir, '%s_labeled_pred.png' % demo_sub.name), dpi=300) plt.close('all') score_train = model.score([x], [gt_train.reshape(img_x * img_y)]) score_test = model.score([x], [gt_test.reshape(img_x * img_y)]) scores = { 'accuracy_train' : float(score_train), 'accuracy_test' : float(score_test) } demo_scores[demo_sub.name] = scores with open(join_path(test_out_dir, 'demo_vol_scores.json'), 'w') as json_out: json_out.write(json.dumps(demo_scores, indent=0, sort_keys=True)) print '--> saved demo volume scores to JSON' print '\nDone.' print 'There were issues, the following combos got skipped:' for combo in failed_combos: print combo
# list X_ = [(np.atleast_2d(x), np.empty((0, 2), dtype=np.int)) for x in X] Y = y.reshape(-1, 1) X_train_, X_test_, X_train, X_test, y_train, y_test, y_org_train, y_org_test =\ train_test_split(X_, X, Y, y_org, test_size=.5) # first, do it with a standard CRF / SVM pbl = GraphCRF(n_features=64, n_states=2, inference_method='lp') svm = NSlackSSVM(pbl, verbose=1, check_constraints=True, C=1000, n_jobs=1, batch_size=-1) svm.fit(X_train_, y_train) y_pred = np.vstack(svm.predict(X_test_)) print("Score with pystruct crf svm: %f" % np.mean(y_pred == y_test)) print(svm.score(X_train_, y_train)) print(svm.score(X_test_, y_test)) # now with latent CRF SVM latent_pbl = LatentGraphCRF(n_features=64, n_labels=2, n_states_per_label=5, inference_method='dai') latent_svm = LatentSubgradientSSVM(model=latent_pbl, max_iter=5000, C=1, verbose=2, n_jobs=1, learning_rate=0.1, show_loss_every=10, momentum=0.0, decay_exponent=0.5) #latent_svm = LatentSSVM(latent_pbl, verbose=2, check_constraints=True, C=100, #n_jobs=1, batch_size=-1, tol=.1, latent_iter=2) latent_svm.fit(X_train_, y_train) print(latent_svm.score(X_train_, y_train)) print(latent_svm.score(X_test_, y_test))
X, Y = make_simple_2x2(seed=1) # flatten X and Y X_flat = [x.reshape(-1, 1).astype(np.float) for x in X] Y_flat = [y.ravel() for y in Y] # first, use standard graph CRF. Can't do much, high loss. crf = GraphCRF() svm = NSlackSSVM(model=crf, max_iter=200, C=1, n_jobs=1) G = [make_grid_edges(x) for x in X] X_grid_edges = list(zip(X_flat, G)) svm.fit(X_grid_edges, Y_flat) plot_boxes(svm.predict(X_grid_edges), title="Non-latent SSVM predictions") print("Training score binary grid CRF: %f" % svm.score(X_grid_edges, Y_flat)) # using one latent variable for each 2x2 rectangle latent_crf = LatentNodeCRF(n_labels=2, n_features=1, n_hidden_states=2, inference_method='lp') ssvm = OneSlackSSVM(model=latent_crf, max_iter=200, C=100, n_jobs=-1, show_loss_every=10, inference_cache=50) latent_svm = LatentSSVM(ssvm)
Y_flat = [y.ravel() for y in Y] # first, use standard graph CRF. Can't do much, high loss. crf = GraphCRF(n_states=2, n_features=1, inference_method='lp') svm = NSlackSSVM(model=crf, max_iter=200, C=1, verbose=0, check_constraints=True, break_on_bad=False, n_jobs=1) # make dataset from X and graph without edges #G_ = [np.zeros((0, 2), dtype=np.int) for x in X] G = [make_grid_edges(x) for x in X] asdf = zip(X_flat, G) svm.fit(asdf, Y_flat) plot_boxes(svm.predict(asdf)) print("Training score multiclass svm CRF: %f" % svm.score(asdf, Y_flat)) # using one latent variable for each 2x2 rectangle latent_crf = LatentNodeCRF(n_labels=2, n_features=1, inference_method='lp', n_hidden_states=2) #latent_svm = LatentSSVM(model=latent_crf, max_iter=200, C=10, verbose=10, #check_constraints=True, break_on_bad=True, n_jobs=1, #latent_iter=10, base_svm='subgradient', tol=-1, #inactive_window=0, learning_rate=0.01, momentum=0) latent_svm = LatentSubgradientSSVM(model=latent_crf, max_iter=200, C=100, verbose=1, n_jobs=1, show_loss_every=10, learning_rate=0.01, momentum=0) # make edges for hidden states: edges = [] node_indices = np.arange(4 * 4).reshape(4, 4)