def test_binary_blocks_cutting_plane(): #testing cutting plane ssvm on easy binary dataset # generate graphs explicitly for each example for inference_method in get_installed(["dai", "lp", "qpbo", "ad3", 'ogm']): print("testing %s" % inference_method) X, Y = generate_blocks(n_samples=3) crf = GraphCRF(inference_method=inference_method) clf = NSlackSSVM(model=crf, max_iter=20, C=100, check_constraints=True, break_on_bad=False, n_jobs=1) x1, x2, x3 = X y1, y2, y3 = Y n_states = len(np.unique(Y)) # delete some rows to make it more fun x1, y1 = x1[:, :-1], y1[:, :-1] x2, y2 = x2[:-1], y2[:-1] # generate graphs X_ = [x1, x2, x3] G = [make_grid_edges(x) for x in X_] # reshape / flatten x and y X_ = [x.reshape(-1, n_states) for x in X_] Y = [y.ravel() for y in [y1, y2, y3]] X = zip(X_, G) clf.fit(X, Y) Y_pred = clf.predict(X) for y, y_pred in zip(Y, Y_pred): assert_array_equal(y, y_pred)
def test_binary_blocks_cutting_plane(): #testing cutting plane ssvm on easy binary dataset # generate graphs explicitly for each example for inference_method in get_installed(["lp", "qpbo", "ad3", 'ogm']): X, Y = generate_blocks(n_samples=3) crf = GraphCRF(inference_method=inference_method) clf = NSlackSSVM(model=crf, max_iter=20, C=100, check_constraints=True, break_on_bad=False, n_jobs=1) x1, x2, x3 = X y1, y2, y3 = Y n_states = len(np.unique(Y)) # delete some rows to make it more fun x1, y1 = x1[:, :-1], y1[:, :-1] x2, y2 = x2[:-1], y2[:-1] # generate graphs X_ = [x1, x2, x3] G = [make_grid_edges(x) for x in X_] # reshape / flatten x and y X_ = [x.reshape(-1, n_states) for x in X_] Y = [y.ravel() for y in [y1, y2, y3]] X = list(zip(X_, G)) clf.fit(X, Y) Y_pred = clf.predict(X) for y, y_pred in zip(Y, Y_pred): assert_array_equal(y, y_pred)
def test_binary_blocks_batches_n_slack(): #testing cutting plane ssvm on easy binary dataset X, Y = generate_blocks(n_samples=5) crf = GridCRF(inference_method=inference_method) clf = NSlackSSVM(model=crf, max_iter=20, batch_size=1, C=100) clf.fit(X, Y) Y_pred = clf.predict(X) assert_array_equal(Y, Y_pred)
def test_multinomial_checker_cutting_plane(): X, Y = generate_checker_multinomial(n_samples=10, noise=.1) n_labels = len(np.unique(Y)) crf = GridCRF(n_states=n_labels, inference_method=inference_method) clf = NSlackSSVM(model=crf, max_iter=20, C=100000, check_constraints=True) clf.fit(X, Y) Y_pred = clf.predict(X) assert_array_equal(Y, Y_pred)
def test_binary_blocks_batches_n_slack(): #testing cutting plane ssvm on easy binary dataset X, Y = toy.generate_blocks(n_samples=5) crf = GridCRF() clf = NSlackSSVM(model=crf, max_iter=20, C=100, check_constraints=True, break_on_bad=False, n_jobs=1, batch_size=1) clf.fit(X, Y) Y_pred = clf.predict(X) assert_array_equal(Y, Y_pred)
def test_binary_blocks_cutting_plane(): #testing cutting plane ssvm on easy binary dataset X, Y = generate_blocks(n_samples=5) crf = GridCRF(inference_method=inference_method) clf = NSlackSSVM(model=crf, max_iter=20, C=100, check_constraints=True, break_on_bad=False) clf.fit(X, Y) Y_pred = clf.predict(X) assert_array_equal(Y, Y_pred)
def test_multinomial_blocks_cutting_plane(): #testing cutting plane ssvm on easy multinomial dataset X, Y = generate_blocks_multinomial(n_samples=40, noise=0.5, seed=0) n_labels = len(np.unique(Y)) crf = GridCRF(n_states=n_labels, inference_method=inference_method) clf = NSlackSSVM(model=crf, max_iter=100, C=100, check_constraints=False, batch_size=1) clf.fit(X, Y) Y_pred = clf.predict(X) assert_array_equal(Y, Y_pred)
def test_simple_1d_dataset_cutting_plane(): # 10 1d datapoints between 0 and 1 X = np.random.uniform(size=(30, 1)) Y = (X.ravel() > 0.5).astype(np.int) # we have to add a constant 1 feature by hand :-/ X = np.hstack([X, np.ones((X.shape[0], 1))]) pbl = MultiClassClf(n_features=2) svm = NSlackSSVM(pbl, check_constraints=True, C=10000) svm.fit(X, Y) assert_array_equal(Y, np.hstack(svm.predict(X)))
def test_multinomial_blocks_directional(): # testing cutting plane ssvm with directional CRF on easy multinomial # dataset X, Y = toy.generate_blocks_multinomial(n_samples=10, noise=0.3, seed=0) n_labels = len(np.unique(Y)) crf = DirectionalGridCRF(n_states=n_labels) clf = NSlackSSVM(model=crf, max_iter=100, C=100, verbose=0, check_constraints=True, batch_size=1) clf.fit(X, Y) Y_pred = clf.predict(X) assert_array_equal(Y, Y_pred)
def test_binary_ssvm_attractive_potentials(): # test that submodular SSVM can learn the block dataset X, Y = toy.generate_blocks(n_samples=10) crf = GridCRF() submodular_clf = NSlackSSVM(model=crf, max_iter=200, C=100, check_constraints=True, positive_constraint=[5]) submodular_clf.fit(X, Y) Y_pred = submodular_clf.predict(X) assert_array_equal(Y, Y_pred) assert_true(submodular_clf.w[5] < 0) # don't ask me about signs
def test_binary_ssvm_attractive_potentials(): # test that submodular SSVM can learn the block dataset X, Y = generate_blocks(n_samples=10) crf = GridCRF(inference_method=inference_method) submodular_clf = NSlackSSVM(model=crf, max_iter=200, C=100, check_constraints=True, negativity_constraint=[5]) submodular_clf.fit(X, Y) Y_pred = submodular_clf.predict(X) assert_array_equal(Y, Y_pred) assert_true(submodular_clf.w[5] < 0)
def CRF_oneNode(x_train, x_test, y_train, y_test): pbl = GraphCRF(n_states = 4,n_features=20) svm = NSlackSSVM(pbl,max_iter=200, C=10,n_jobs=2) svm.fit(x_train,y_train) y_pred = svm.predict(x_test) target_names = ['Start','Mid','End','Others'] #eclf = EnsembleClassifier(clfs=[pipe1, pipe2],voting='soft',weights=[0.5,0.2]) #eclf.fit(x_train,y_train) #y_pred = eclf.predict(x_test) print classification_report(y_test, y_pred, target_names=target_names)
def test_simple_1d_dataset_cutting_plane(): # 10 1d datapoints between 0 and 1 X = np.random.uniform(size=(30, 1)) # linearly separable labels Y = 1 - 2 * (X.ravel() < .5) # we have to add a constant 1 feature by hand :-/ X = np.hstack([X, np.ones((X.shape[0], 1))]) pbl = BinaryClf(n_features=2) svm = NSlackSSVM(pbl, check_constraints=True, C=1000) svm.fit(X, Y) assert_array_equal(Y, np.hstack(svm.predict(X)))
def test_multinomial_blocks_directional(): # testing cutting plane ssvm with directional CRF on easy multinomial # dataset X, Y = generate_blocks_multinomial(n_samples=10, noise=0.3, seed=0) n_labels = len(np.unique(Y)) crf = DirectionalGridCRF(n_states=n_labels, inference_method=inference_method) clf = NSlackSSVM(model=crf, max_iter=100, C=100, verbose=0, check_constraints=True, batch_size=1) clf.fit(X, Y) Y_pred = clf.predict(X) assert_array_equal(Y, Y_pred)
def test_binary_ssvm_repellent_potentials(): # test non-submodular problem with and without submodularity constraint # dataset is checkerboard X, Y = generate_checker() crf = GridCRF(inference_method=inference_method) clf = NSlackSSVM(model=crf, max_iter=10, C=100, check_constraints=True) clf.fit(X, Y) Y_pred = clf.predict(X) # standard crf can predict perfectly assert_array_equal(Y, Y_pred) submodular_clf = NSlackSSVM(model=crf, max_iter=10, C=100, check_constraints=True, negativity_constraint=[4, 5, 6]) submodular_clf.fit(X, Y) Y_pred = submodular_clf.predict(X) # submodular crf can not do better than unaries for i, x in enumerate(X): y_pred_unaries = crf.inference(x, np.array([1, 0, 0, 1, 0, 0, 0])) assert_array_equal(y_pred_unaries, Y_pred[i])
def test_multinomial_blocks_cutting_plane(): #testing cutting plane ssvm on easy multinomial dataset X, Y = toy.generate_blocks_multinomial(n_samples=10, noise=0.3, seed=0) n_labels = len(np.unique(Y)) for inference_method in get_installed(['lp', 'qpbo', 'ad3']): crf = GridCRF(n_states=n_labels, inference_method=inference_method) clf = NSlackSSVM(model=crf, max_iter=10, C=100, check_constraints=False) clf.fit(X, Y) Y_pred = clf.predict(X) assert_array_equal(Y, Y_pred)
def test_blobs_2d_cutting_plane(): # make two gaussian blobs X, Y = make_blobs(n_samples=80, centers=2, random_state=1) Y = 2 * Y - 1 # we have to add a constant 1 feature by hand :-/ X = np.hstack([X, np.ones((X.shape[0], 1))]) X_train, X_test, Y_train, Y_test = X[:40], X[40:], Y[:40], Y[40:] pbl = BinaryClf(n_features=3) svm = NSlackSSVM(pbl, check_constraints=True, C=1000) svm.fit(X_train, Y_train) assert_array_equal(Y_test, np.hstack(svm.predict(X_test)))
def test_blobs_2d_cutting_plane(): # make two gaussian blobs X, Y = make_blobs(n_samples=80, centers=3, random_state=42) # we have to add a constant 1 feature by hand :-/ X = np.hstack([X, np.ones((X.shape[0], 1))]) X_train, X_test, Y_train, Y_test = X[:40], X[40:], Y[:40], Y[40:] pbl = MultiClassClf(n_features=3, n_classes=3) svm = NSlackSSVM(pbl, check_constraints=True, C=1000, batch_size=1) svm.fit(X_train, Y_train) assert_array_equal(Y_test, np.hstack(svm.predict(X_test)))
def test_multinomial_blocks_directional_simple(): # testing cutting plane ssvm with directional CRF on easy multinomial # dataset X_, Y_ = generate_blocks_multinomial(n_samples=10, noise=0.3, seed=0) G = [make_grid_edges(x, return_lists=True) for x in X_] edge_features = [edge_list_to_features(edge_list) for edge_list in G] edges = [np.vstack(g) for g in G] X = zip([x.reshape(-1, 3) for x in X_], edges, edge_features) Y = [y.ravel() for y in Y_] crf = EdgeFeatureGraphCRF(n_states=3, n_edge_features=2) clf = NSlackSSVM(model=crf, max_iter=10, C=1, check_constraints=False) clf.fit(X, Y) Y_pred = clf.predict(X) assert_array_equal(Y, Y_pred)
def test_multinomial_blocks_directional_simple(): # testing cutting plane ssvm with directional CRF on easy multinomial # dataset X_, Y_ = generate_blocks_multinomial(n_samples=10, noise=0.3, seed=0) G = [make_grid_edges(x, return_lists=True) for x in X_] edge_features = [edge_list_to_features(edge_list) for edge_list in G] edges = [np.vstack(g) for g in G] X = list(zip([x.reshape(-1, 3) for x in X_], edges, edge_features)) Y = [y.ravel() for y in Y_] crf = EdgeFeatureGraphCRF(n_states=3, n_edge_features=2) clf = NSlackSSVM(model=crf, max_iter=10, C=1, check_constraints=False) clf.fit(X, Y) Y_pred = clf.predict(X) assert_array_equal(Y, Y_pred)
def test_binary_blocks_cutting_plane_latent_node(): #testing cutting plane ssvm on easy binary dataset # we use the LatentNodeCRF without latent nodes and check that it does the # same as GraphCRF X, Y = generate_blocks(n_samples=3) crf = GraphCRF() clf = NSlackSSVM(model=crf, max_iter=20, C=100, check_constraints=True, break_on_bad=False, n_jobs=1) x1, x2, x3 = X y1, y2, y3 = Y n_states = len(np.unique(Y)) # delete some rows to make it more fun x1, y1 = x1[:, :-1], y1[:, :-1] x2, y2 = x2[:-1], y2[:-1] # generate graphs X_ = [x1, x2, x3] G = [make_grid_edges(x) for x in X_] # reshape / flatten x and y X_ = [x.reshape(-1, n_states) for x in X_] Y = [y.ravel() for y in [y1, y2, y3]] X = zip(X_, G) clf.fit(X, Y) Y_pred = clf.predict(X) for y, y_pred in zip(Y, Y_pred): assert_array_equal(y, y_pred) latent_crf = LatentNodeCRF(n_labels=2, n_hidden_states=0) latent_svm = LatentSSVM(NSlackSSVM(model=latent_crf, max_iter=20, C=100, check_constraints=True, break_on_bad=False, n_jobs=1), latent_iter=3) X_latent = zip(X_, G, np.zeros(len(X_))) latent_svm.fit(X_latent, Y, H_init=Y) Y_pred = latent_svm.predict(X_latent) for y, y_pred in zip(Y, Y_pred): assert_array_equal(y, y_pred) assert_array_almost_equal(latent_svm.w, clf.w)
def runIt(train_list): X_org = list2features(train_list) X = np.array(X_org) y = list2labels_sleep(train_list) y_org = np.array(y) Y = y_org.reshape(-1, 1) X_ = [(np.atleast_2d(x), np.empty((0, 2), dtype=np.int)) for x in X] X_train, X_test, y_train, y_test = train_test_split(X_, Y, test_size=.5) pbl = GraphCRF(inference_method='unary') svm = NSlackSSVM(pbl, C=100) start = time() svm.fit(X_train, y_train) time_svm = time() - start y_pred = np.vstack(svm.predict(X_test)) print("Score with pystruct crf svm: %f (took %f seconds)" % (np.mean(y_pred == y_test), time_svm))
def test_multinomial_blocks_directional_anti_symmetric(): # testing cutting plane ssvm with directional CRF on easy multinomial # dataset X_, Y_ = generate_blocks_multinomial(n_samples=10, noise=0.3, seed=0) G = [make_grid_edges(x, return_lists=True) for x in X_] edge_features = [edge_list_to_features(edge_list) for edge_list in G] edges = [np.vstack(g) for g in G] X = list(zip([x.reshape(-1, 3) for x in X_], edges, edge_features)) Y = [y.ravel() for y in Y_] crf = EdgeFeatureGraphCRF(symmetric_edge_features=[0], antisymmetric_edge_features=[1]) clf = NSlackSSVM(model=crf, C=100) clf.fit(X, Y) Y_pred = clf.predict(X) assert_array_equal(Y, Y_pred) pairwise_params = clf.w[-9 * 2 :].reshape(2, 3, 3) sym = pairwise_params[0] antisym = pairwise_params[1] assert_array_equal(sym, sym.T) assert_array_equal(antisym, -antisym.T)
def test_multinomial_blocks_directional_anti_symmetric(): # testing cutting plane ssvm with directional CRF on easy multinomial # dataset X_, Y_ = generate_blocks_multinomial(n_samples=10, noise=0.3, seed=0) G = [make_grid_edges(x, return_lists=True) for x in X_] edge_features = [edge_list_to_features(edge_list) for edge_list in G] edges = [np.vstack(g) for g in G] X = zip([x.reshape(-1, 3) for x in X_], edges, edge_features) Y = [y.ravel() for y in Y_] crf = EdgeFeatureGraphCRF(symmetric_edge_features=[0], antisymmetric_edge_features=[1]) clf = NSlackSSVM(model=crf, C=100) clf.fit(X, Y) Y_pred = clf.predict(X) assert_array_equal(Y, Y_pred) pairwise_params = clf.w[-9 * 2:].reshape(2, 3, 3) sym = pairwise_params[0] antisym = pairwise_params[1] assert_array_equal(sym, sym.T) assert_array_equal(antisym, -antisym.T)
# --- Train model --- # model = EdgeFeatureGraphCRF(n_states, n_features, n_edge_features) ssvm = NSlackSSVM(model=model, C=0.1, tol=0.001, verbose=0, show_loss_every=10) # ssvm = OneSlackSSVM(model=model, C=.1, inference_cache=50, tol=0.1, verbose=0,show_loss_every=10) ssvm.fit(X_train, Y_train) # --- Test with pystruct --- # # print("Test score with graph CRF: %f" % ssvm.score(X_test, Y_test)) # --- Test manually - get contingency tables --- # prediction = ssvm.predict(X_test) contingency = np.array([0, 0, 0, 0]) for i in xrange(len(test)): pred = prediction[i] true = Y_test[i] contingency = contingency + get_contingency(pred, true) TP, FP, TN, FN = contingency[0], contingency[1], contingency[ 2], contingency[3] # sens = float(TP)/(TP+FN) # sens_all.append(sens) # spec = float(TN)/(FP+TN) # spec_all.append(spec) acc = float(TP + TN) / (TP + FP + TN + FN)
X, Y = make_simple_2x2(seed=1) # flatten X and Y X_flat = [x.reshape(-1, 1).astype(np.float) for x in X] Y_flat = [y.ravel() for y in Y] # first, use standard graph CRF. Can't do much, high loss. crf = GraphCRF() svm = NSlackSSVM(model=crf, max_iter=200, C=1, n_jobs=1) G = [make_grid_edges(x) for x in X] X_grid_edges = list(zip(X_flat, G)) svm.fit(X_grid_edges, Y_flat) plot_boxes(svm.predict(X_grid_edges), title="Non-latent SSVM predictions") print("Training score binary grid CRF: %f" % svm.score(X_grid_edges, Y_flat)) # using one latent variable for each 2x2 rectangle latent_crf = LatentNodeCRF(n_labels=2, n_features=1, n_hidden_states=2, inference_method='lp') ssvm = OneSlackSSVM(model=latent_crf, max_iter=200, C=100, n_jobs=-1, show_loss_every=10, inference_cache=50) latent_svm = LatentSSVM(ssvm)
X = zip([x.reshape(-1, 3) for x in X_], edges, edge_features) Y = [y.ravel() for y in Y_] Xtrain = np.array(X[0:int(np.floor(N/2))]) Ytrain = np.array(Y[0:int(np.floor(N/2))]) Xtest = np.array(X[int(np.floor(N/2)):]) Ytest = np.array(Y[int(np.floor(N/2)):]) crf = EdgeFeatureGraphCRF(n_states=3, n_edge_features=2) ## SSVM ssvm_logger = './logExampleSSVM.pickle' ssvm = NSlackSSVM(model=crf, max_iter=10, C=1, check_constraints=False, logger=SaveLogger(ssvm_logger,save_every=100) ) ssvm.fit(Xtrain, Ytrain) Ypred_ssvm = ssvm.predict(Xtest) ## CSSVM M = 5 # number of models in ensemble gamma = 0.001 # diversity weight C = 1 # regularizer weight cssvm = SubgradientMBestSSVM(crf, verbose=1, C=C, max_iter=100, n_jobs=1, M=M, gamma=gamma, initialize_w_by=ssvm_logger, momentum=0.0, learning_rate='auto', break_on_no_constraints=True, batch_size=None, decay_t0=1, decay_exponent=0.5, averaging=None, shuffle=False, sample_assignment_strategy='all', online=False, break_on_no_loss_improvement=2,parallel_inference=True, zero_constraint=None, force_moment=1) cssvm.fit(Xtrain, Ytrain) Ypreds_cssvm = cssvm.predict(Xtest)
def crf_postprocess(X_train, y_train, X_test, train_examples=2000): clf = NSlackSSVM(MultiLabelClf(), verbose=1, n_jobs=-1, show_loss_every=1) clf.fit(X_train, y_train) pred = clf.predict(X_test) pred = np.array(pred) return pred
X, Y = make_simple_2x2(seed=1) # flatten X and Y X_flat = [x.reshape(-1, 1).astype(np.float) for x in X] Y_flat = [y.ravel() for y in Y] # first, use standard graph CRF. Can't do much, high loss. crf = GraphCRF() svm = NSlackSSVM(model=crf, max_iter=200, C=1, n_jobs=1) G = [make_grid_edges(x) for x in X] X_grid_edges = list(zip(X_flat, G)) svm.fit(X_grid_edges, Y_flat) plot_boxes(svm.predict(X_grid_edges), title="Non-latent SSVM predictions") print("Training score binary grid CRF: %f" % svm.score(X_grid_edges, Y_flat)) # using one latent variable for each 2x2 rectangle latent_crf = LatentNodeCRF(n_labels=2, n_features=1, n_hidden_states=2, inference_method='lp') ssvm = OneSlackSSVM(model=latent_crf, max_iter=200, C=100, n_jobs=-1, show_loss_every=10, inference_cache=50) latent_svm = LatentSSVM(ssvm) # make edges for hidden states: edges = [] node_indices = np.arange(4 * 4).reshape(4, 4) for i, (x, y) in enumerate(itertools.product([0, 2], repeat=2)): for j in range(x, x + 2):
def test_final_segmentations(out_dir): test_size = .75 #ssvm_c = [5, .3, .1, .01, .001] ssvm_c = [50] ssvm_tol = 0.001 ssvm_iter = 50 which_ssvm = 'nslack' which_solver = ('ogm', {'alg': 'gc'}) which_combo = which_ssvm + '_ogm-' + which_solver[1]['alg'] out_dir = join_path(out_dir, 'test_final_segmentations', which_combo) create_out_path(out_dir, except_on_exist=False) cd = JaneliaData(dummy_data=False, only_nice_volumes=True) subs = cd.subvolumes demo_volumes = ['an197522_2013_03_10_13002', 'an197522_2013_03_08_06003', 'an229719_2013_12_05_03004', 'an229719_2013_12_05_07003', 'an229719_2013_12_05_08004'] img_x, img_y = cd.subvolumes[0].dims edges = create_2d_edge_graph(img_x, img_y) feats_unaries = ['feats_pm_++', 'feats_pm_+++', 'feats_pm_+++x'] feats_pairwise = 'feats_xcorr_green' train_labelings = ['gt_active_++', 'gt_active_+++', 'gt_active_+++x'] test_labelings = ['gt_active_++', 'gt_active_+++', 'gt_active_+++x'] failed_combos = [] sp = ShuffleSplit(len(subs), random_state=42, test_size=test_size) train_idxs, test_idxs = next(iter(sp)) for feats_unary in feats_unaries: X = ssvm_construct_X(subs, feats_unary, feats_pairwise, edges) for train_labeling in train_labelings: Y_full_train = ssvm_construct_Y(train_labeling, subs) class_weights = compute_class_weights(Y_full_train) crf_graph = EdgeFeatureGraphCRF(inference_method=which_solver, class_weight=class_weights) for test_labeling in test_labelings: X_train = [X[i] for i in train_idxs] Y_train = [Y_full_train[i] for i in train_idxs] Y_full_test = ssvm_construct_Y(test_labeling, subs) X_test = [X[i] for i in test_idxs] Y_test = [Y_full_test[i] for i in test_idxs] for c in ssvm_c: try: test_name = '%s_u=%s_p=%s_train-gt=%s_test-gt=%s_C=%.1e' % (which_ssvm, feats_unary, feats_pairwise, train_labeling, test_labeling, c) test_out_dir = join_path(out_dir, test_name) create_out_path(test_out_dir, except_on_exist=True) print '\n==================================================================' print '-->' + test_name except IOError as e: print e continue try: logger = pystruct_logger(join_path(test_out_dir, 'ssvm_model.logger'), save_every=10) if which_ssvm == 'nslack': print "* creating NSlackSSVM" model = NSlackSSVM(crf_graph, verbose=1, C=c, max_iter=ssvm_iter, n_jobs=-1, tol=ssvm_tol, show_loss_every=1, inactive_threshold=1e-3, inactive_window=10, batch_size=1000, logger=logger) ssvm_name = 'NSlackSSVM' if which_ssvm == 'oneslack': print "* creating OneSlackSSVM" model = OneSlackSSVM(crf_graph, verbose=1, C=c, max_iter=ssvm_iter, n_jobs=-1, tol=ssvm_tol, show_loss_every=1, inactive_threshold=1e-3, inactive_window=10, logger=logger) ssvm_name = 'OneSlackSSVM' if which_ssvm == 'frankwolfe': print "* creating FrankWolfeSSVM" model = FrankWolfeSSVM(crf_graph, verbose=1, C=c, max_iter=ssvm_iter, tol=ssvm_tol, line_search=True, check_dual_every=10, do_averaging=True, sample_method='perm', random_state=None, logger=logger) ssvm_name = 'FrankWolfeSSVM' print '* fitting model...' model.fit(X_train, Y_train) print '--> model trained' print '* scoring model...' score = model.score(X_test, Y_test) print score print '* saving results for test run' fig, _ = ssvm_plot_learning(model, title=test_name, time=False) fig.savefig(join_path(test_out_dir, 'ssvm_plot_learning.png'), dpi=300) plt.close('all') print '--> saved learning plots' ### save results to JSON ### with open(join_path(test_out_dir, 'ssvm_stats.json'), 'w') as json_out: stats = { 'test_accuracy' : float(score), 'test_size' : float(test_size), 'ssvm_c' : float(c), 'ssvm_iter' : ssvm_iter, 'ssvm_tol' : float(ssvm_tol), 'feats_unary' : feats_unary, 'feats_pairwise' : feats_pairwise, 'which_ssvm' : which_ssvm, 'which_solver' : which_solver[0], 'which_alg' : which_solver[1]['alg'], 'train_labeling' : train_labeling, 'test_labeling' : test_labeling } json_out.write(json.dumps(stats, indent=0, sort_keys=True)) print '--> saved SSVM stats to JSON' except: print 'Combo %s failed, removing its out folder.' % test_out_dir failed_combos.append(test_out_dir) os.rmdir(test_out_dir) create_out_path('FAILED_' + test_out_dir) continue print '--> saving vizzz for demo volumes and computing scores...' demo_out_dir = join_path(test_out_dir, 'demo_vols') create_out_path(demo_out_dir, except_on_exist=True) demo_scores = {} for demo_sub in demo_volumes: demo_sub = cd.get_subvolume_by_name(demo_sub) print ' -', demo_sub.name unary_feats = demo_sub.get(feats_unary) pairwise_feats = demo_sub.get(feats_pairwise) x = (unary_feats, edges, pairwise_feats) pred = model.predict([x]) pred = pred[0].reshape(img_x, img_y) gt_train = demo_sub.get(train_labeling).astype('uint8') gt_test = demo_sub.get(test_labeling).astype('uint8') rgb_vis_train = plot_it_like_ferran(gt_train, pred) rgb_vis_test = plot_it_like_ferran(gt_test, pred) plt.imsave(join_path(demo_out_dir, '%s_ssvm_prediction.png' % demo_sub.name), pred) plt.imsave(join_path(demo_out_dir, '%s_%s.png' % (demo_sub.name, train_labeling)), gt_train) plt.imsave(join_path(demo_out_dir, '%s_%s.png' % (demo_sub.name, test_labeling)), gt_test) plt.imsave(join_path(demo_out_dir, '%s_rgb_vis_train.png' % demo_sub.name), rgb_vis_train) plt.imsave(join_path(demo_out_dir, '%s_rgb_vis_test.png' % demo_sub.name), rgb_vis_test) labeled_pm_ax = demo_sub.plot_activity_labels(image=pred, cmap='gray') plt.savefig(join_path(demo_out_dir, '%s_labeled_pred.png' % demo_sub.name), dpi=300) plt.close('all') score_train = model.score([x], [gt_train.reshape(img_x * img_y)]) score_test = model.score([x], [gt_test.reshape(img_x * img_y)]) scores = { 'accuracy_train' : float(score_train), 'accuracy_test' : float(score_test) } demo_scores[demo_sub.name] = scores with open(join_path(test_out_dir, 'demo_vol_scores.json'), 'w') as json_out: json_out.write(json.dumps(demo_scores, indent=0, sort_keys=True)) print '--> saved demo volume scores to JSON' print '\nDone.' print 'There were issues, the following combos got skipped:' for combo in failed_combos: print combo
X, Y = make_simple_2x2(seed=1) # flatten X and Y X_flat = [x.reshape(-1, 1).astype(np.float) for x in X] Y_flat = [y.ravel() for y in Y] # first, use standard graph CRF. Can't do much, high loss. crf = GraphCRF() svm = NSlackSSVM(model=crf, max_iter=200, C=1, n_jobs=1) G = [make_grid_edges(x) for x in X] asdf = zip(X_flat, G) svm.fit(asdf, Y_flat) plot_boxes(svm.predict(asdf), title="Non-latent SSVM predictions") print("Training score binary grid CRF: %f" % svm.score(asdf, Y_flat)) # using one latent variable for each 2x2 rectangle latent_crf = LatentNodeCRF(n_labels=2, n_features=1, n_hidden_states=2, inference_method='lp') ssvm = OneSlackSSVM(model=latent_crf, max_iter=200, C=100, verbose=1, n_jobs=-1, show_loss_every=10, inference_cache=50) latent_svm = LatentSSVM(ssvm) # make edges for hidden states: edges = [] node_indices = np.arange(4 * 4).reshape(4, 4) for i, (x, y) in enumerate(itertools.product([0, 2], repeat=2)): for j in xrange(x, x + 2):
X, Y = make_simple_2x2(seed=1) # flatten X and Y X_flat = [x.reshape(-1, 1).astype(np.float) for x in X] Y_flat = [y.ravel() for y in Y] # first, use standard graph CRF. Can't do much, high loss. crf = GraphCRF() svm = NSlackSSVM(model=crf, max_iter=200, C=1, n_jobs=1) G = [make_grid_edges(x) for x in X] asdf = zip(X_flat, G) svm.fit(asdf, Y_flat) plot_boxes(svm.predict(asdf), title="Non-latent SSVM predictions") print("Training score binary grid CRF: %f" % svm.score(asdf, Y_flat)) # using one latent variable for each 2x2 rectangle latent_crf = LatentNodeCRF(n_labels=2, n_features=1, n_hidden_states=2, inference_method='lp') ssvm = OneSlackSSVM(model=latent_crf, max_iter=200, C=100, verbose=1, n_jobs=-1, show_loss_every=10, inference_cache=50)
asdf2 = zip(X_flat2,G2) #%% folders = os.listdir(folderName) num_models = len(folders) print "Number of models :",num_models modelTestErrors=[] for i in range(num_models): errTest=0 iteration = int(re.findall(r'\d+',folders[i])[0]) with open(folderName+folders[i], "rb") as f: svm = pickle.load(f) predTest = svm.predict(asdf2) for j in range(len(predTest)): errTest += accuracy(predTest[j],Y_flat2[j]) errTest = errTest/float(len(predTest)) modelTestErrors.append((iteration,errTest)) modelTestErrors.sort(key=lambda x:x[0]) for i in range(len(modelTestErrors)): print "Iteration : %d , Dice Score : %f"%(modelTestErrors[i][0],modelTestErrors[i][1]) #%% plt.plot([x[0] for x in modelTestErrors], [x[1] for x in modelTestErrors])
asdf2 = zip(X_flat2, G2) #%% folders = os.listdir(folderName) num_models = len(folders) print "Number of models :", num_models modelTestErrors = [] for i in range(num_models): errTest = 0 iteration = int(re.findall(r'\d+', folders[i])[0]) with open(folderName + folders[i], "rb") as f: svm = pickle.load(f) predTest = svm.predict(asdf2) for j in range(len(predTest)): errTest += accuracy(predTest[j], Y_flat2[j]) errTest = errTest / float(len(predTest)) modelTestErrors.append((iteration, errTest)) modelTestErrors.sort(key=lambda x: x[0]) for i in range(len(modelTestErrors)): print "Iteration : %d , Dice Score : %f" % (modelTestErrors[i][0], modelTestErrors[i][1]) #%%
edgeFeatures2 = [] for i in range(len(X_flat2)): feature = [] for j in range(len(edgeList)): feature.append( np.append(X_flat2[i][edgeList[j][0]], X_flat2[i][edgeList[j][1]])) edgeFeatures2.append(feature) edgeFeatures2 = np.array(edgeFeatures2) #asdf2 = zip(X_flat2,G2,edgeFeatures2) asdf2 = zip(X_flat2, G2) #%% predTrain = svm.predict(asdf) errTrain = 0 for i in range(len(predTrain)): errTrain += accuracy(predTrain[i], Y_flat[i]) errTrain = errTrain / float(len(predTrain)) #%% predTest = svm.predict(asdf2) errTest = 0 for i in range(len(predTest)): errTest += accuracy(predTest[i], Y_flat2[i]) errTest = errTest / float(len(predTest)) print "The train set DICE is %f" % (errTrain) print "The test set DICE is %f" % (errTest)
def test_multivariate_ssvm(): def loss_fn(tn, fp, fn, tp): return tp / (tp + .5 * (fp + fn)) #def loss_fn(tn, fp, fn, tp): # return (fp + fn * 1.) / (tn + fp + fn + tp) n_classes = 2 n_features = 3 n_examples = 600 df, feature_cols, label_col = get_center_dataset(n_classes=n_classes, n_features=n_features, n_examples=n_examples) # Break into training / test df_train, df_test = train_test_split(df, test_size=0.15, random_state=22) print('\nSetup') print("*" * 15) print('loss=f1_score') print('n_train_examples = %s' % df_train.shape[0]) print('n_test_examples = %s' % df_test.shape[0]) print('n_classes = %s' % n_classes) print('n_features = %s' % n_features) # Select out right columns X_train, X_test = df_train[feature_cols].values, df_test[ feature_cols].values y_train, y_test = df_train[label_col].values, df_test[label_col].values # Let's normalize our data mean, std = X_train.mean(axis=0), X_train.std(axis=0) X_train = (X_train - mean) / std X_test = (X_test - mean) / std # Handy for later n_train_examples = X_train.shape[0] n_test_examples = X_test.shape[0] print('\nRandom Algorithm') print("*" * 15) n = len(y_test.ravel()) losses = [] for _ in range(1000): y_pred = np.random.choice([-1, 1], n) tn, fp, fn, tp = confusion_matrix(y_test.ravel(), y_pred).ravel() losses.append(loss_fn(tn, fp, fn, tp)) lower = np.percentile(losses, 5) mid = np.percentile(losses, 50) upper = np.percentile(losses, 95) print('test_loss_mid=%s' % lower) print('test_loss_lower=%s' % mid) print('test_loss_upper=%s' % upper) # Pystruct SSVM print('\n Pystruct SSM Algorithm') print('*' * 15) model = PystructSSVM(X_train, y_train, X_test, y_test, loss_fn) clf = NSlackSSVM(model, C=1.0, batch_size=-1) clf.fit(X_train, y_train) clf.predict(X_test) tn, fp, fn, tp = confusion_matrix(y_test.ravel(), y_pred).ravel() print('test_loss={}' % loss_fn(tn, fp, fn, tp)) # Simple SVM print('\nSVM Algorithm') print("*" * 15) from sklearn import svm C = 1. clf = svm.SVC(C=C) print('Fitting SVM. C={}' % C) clf.fit(X_train, y_train) y_pred = clf.predict(X_test) tn, fp, fn, tp = confusion_matrix(y_test.ravel(), y_pred).ravel() print('test_loss={}' % loss_fn(tn, fp, fn, tp)) """
X_train, X_test = features[fidx == 0], features[fidx == 1] y_train, y_test = labels[fidx == 0], labels[fidx == 1] model = ChainCRF() #model = BinaryClf(X_train.shape[0]) model = MultiLabelClf() ssvm = NSlackSSVM(model=model, max_iter=500) X_train_l = [row for row in X_train] y_train_l = [row for row in y_train] X_test_l = [row for row in X_test] ssvm.fit(X_train, y_train) y_pred_l = ssvm.predict(X_test_l) y_pred = np.array(y_pred_l) #for pred in y_pred: # print(pred) #print(y_pred) assert y_pred.shape == y_test.shape for i in range(y_pred.shape[1]): print("bACC [{0:d}] = {1:0.3f}".format(i, balanced_accuracy_score(y_test[:, i], y_pred[:, i])))
ssvm.fit(X_train, Y_train) train_time = time.time() - start print("Training completed and testing started") print("Time used for training: {}".format(train_time)) pickle.dump( ssvm, open( os.path.join(param_fdp_results, "results_{}_{}_model.p".format(k, pattern)), "wb")) start = time.time() Y_pred = ssvm.predict(X_test) test_time = time.time() - start acc = accuracy_score(np.hstack(Y_test), np.hstack(Y_pred)) cm = confusion_matrix(np.hstack(Y_test), np.hstack(Y_pred)) pickle.dump( cm, open( os.path.join(param_fdp_results, "results_{}_{}_cm.p".format(k, pattern)), "wb")) print("Test completed with an accuracy: %.3f" % acc) print(cm) print("Time used for testing: {}".format(test_time)) results.append([
Y_flat2 = np.array(testLabels[0:n_test]) edgeFeatures2=[] for i in range(len(X_flat2)): feature=[] for j in range(len(edgeList)): feature.append( np.append(X_flat2[i][edgeList[j][0]] , X_flat2[i][edgeList[j][1]]) ) edgeFeatures2.append(feature) edgeFeatures2=np.array(edgeFeatures2) #asdf2 = zip(X_flat2,G2,edgeFeatures2) asdf2 = zip(X_flat2,G2) #%% predTrain = svm.predict(asdf) errTrain = 0 for i in range(len(predTrain)): errTrain += accuracy(predTrain[i],Y_flat[i]) errTrain = errTrain/float(len(predTrain)) #%% predTest = svm.predict(asdf2) errTest = 0 for i in range(len(predTest)): errTest += accuracy(predTest[i],Y_flat2[i]) errTest = errTest/float(len(predTest)) print "The train set DICE is %f" %(errTrain) print "The test set DICE is %f" %(errTest)
Y_test = [labels[j] for j in test] # if verbose: # print np.mean(map(np.mean,Y_train)), 'pm',np.var(map(np.mean,Y_train)) # print np.mean(map(np.mean,Y_test)), 'pm',np.var(map(np.mean,Y_test)) # --- Train model --- # model = EdgeFeatureGraphCRF(n_states,n_features,n_edge_features) ssvm = NSlackSSVM(model=model, C=0.1, tol=0.001, verbose=0,show_loss_every=10) # ssvm = OneSlackSSVM(model=model, C=.1, inference_cache=50, tol=0.1, verbose=0,show_loss_every=10) ssvm.fit(X_train, Y_train) # --- Test with pystruct --- # # print("Test score with graph CRF: %f" % ssvm.score(X_test, Y_test)) # --- Test manually - get contingency tables --- # prediction = ssvm.predict(X_test) contingency = np.array([0,0,0,0]) for i in xrange(len(test)): pred = prediction[i] true = Y_test[i] contingency = contingency+get_contingency(pred,true) TP, FP, TN, FN = contingency[0], contingency[1], contingency[2], contingency[3] # sens = float(TP)/(TP+FN) # sens_all.append(sens) # spec = float(TN)/(FP+TN) # spec_all.append(spec) acc = float(TP+TN)/(TP+FP+TN+FN) acc_all.append(acc)
model = MultiClassClf(n_features=X_train_bias.shape[1], n_classes=10) n_slack_svm = NSlackSSVM(model, verbose=2, check_constraints=False, C=0.1, batch_size=100, tol=1e-2) one_slack_svm = OneSlackSSVM(model, verbose=2, C=.10, tol=.001) subgradient_svm = SubgradientSSVM(model, C=0.1, learning_rate=0.000001, max_iter=1000, verbose=0) fw_bc_svm = FrankWolfeSSVM(model, C=.1, max_iter=50) fw_batch_svm = FrankWolfeSSVM(model, C=.1, max_iter=50, batch_mode=True) # n-slack cutting plane ssvm start = time() n_slack_svm.fit(X_train_bias, y_train) time_n_slack_svm = time() - start y_pred = np.hstack(n_slack_svm.predict(X_test_bias)) print("Score with pystruct n-slack ssvm: %f (took %f seconds)" % (np.mean(y_pred == y_test), time_n_slack_svm)) ## 1-slack cutting plane ssvm start = time() one_slack_svm.fit(X_train_bias, y_train) time_one_slack_svm = time() - start y_pred = np.hstack(one_slack_svm.predict(X_test_bias)) print("Score with pystruct 1-slack ssvm: %f (took %f seconds)" % (np.mean(y_pred == y_test), time_one_slack_svm)) #online subgradient ssvm start = time() subgradient_svm.fit(X_train_bias, y_train) time_subgradient_svm = time() - start
""" from time import time import numpy as np from sklearn.datasets import load_iris from sklearn.cross_validation import train_test_split from pystruct.models import GraphCRF from pystruct.learners import NSlackSSVM iris = load_iris() X, y = iris.data, iris.target # make each example into a tuple of a single feature vector and an empty edge # list X_ = [(np.atleast_2d(x), np.empty((0, 2), dtype=np.int)) for x in X] Y = y.reshape(-1, 1) X_train, X_test, y_train, y_test = train_test_split(X_, Y) pbl = GraphCRF(inference_method='unary') svm = NSlackSSVM(pbl, C=100) start = time() svm.fit(X_train, y_train) time_svm = time() - start y_pred = np.vstack(svm.predict(X_test)) print("Score with pystruct crf svm: %f (took %f seconds)" % (np.mean(y_pred == y_test), time_svm))
from time import time import numpy as np from sklearn.datasets import load_iris from sklearn.cross_validation import train_test_split from pystruct.models import GraphCRF from pystruct.learners import NSlackSSVM iris = load_iris() X, y = iris.data, iris.target # make each example into a tuple of a single feature vector and an empty edge # list X_ = [(np.atleast_2d(x), np.empty((0, 2), dtype=np.int)) for x in X] Y = y.reshape(-1, 1) X_train, X_test, y_train, y_test = train_test_split(X_, Y) pbl = GraphCRF(n_features=4, n_states=3, inference_method='lp') svm = NSlackSSVM(pbl, verbose=1, check_constraints=True, C=100, n_jobs=1) start = time() svm.fit(X_train, y_train) time_svm = time() - start y_pred = np.vstack(svm.predict(X_test)) print("Score with pystruct crf svm: %f (took %f seconds)" % (np.mean(y_pred == y_test), time_svm))
X_flat = [x.reshape(-1, 1).astype(np.float) for x in X] Y_flat = [y.ravel() for y in Y] # first, use standard graph CRF. Can't do much, high loss. crf = GraphCRF(n_states=2, n_features=1, inference_method='lp') svm = NSlackSSVM(model=crf, max_iter=200, C=1, verbose=0, check_constraints=True, break_on_bad=False, n_jobs=1) # make dataset from X and graph without edges #G_ = [np.zeros((0, 2), dtype=np.int) for x in X] G = [make_grid_edges(x) for x in X] asdf = zip(X_flat, G) svm.fit(asdf, Y_flat) plot_boxes(svm.predict(asdf)) print("Training score multiclass svm CRF: %f" % svm.score(asdf, Y_flat)) # using one latent variable for each 2x2 rectangle latent_crf = LatentNodeCRF(n_labels=2, n_features=1, inference_method='lp', n_hidden_states=2) #latent_svm = LatentSSVM(model=latent_crf, max_iter=200, C=10, verbose=10, #check_constraints=True, break_on_bad=True, n_jobs=1, #latent_iter=10, base_svm='subgradient', tol=-1, #inactive_window=0, learning_rate=0.01, momentum=0) latent_svm = LatentSubgradientSSVM(model=latent_crf, max_iter=200, C=100, verbose=1, n_jobs=1, show_loss_every=10, learning_rate=0.01, momentum=0) # make edges for hidden states: edges = []