def test_constraint_removal(): digits = load_digits() X, y = digits.data, digits.target y = 2 * (y % 2) - 1 # even vs odd as +1 vs -1 X = X / 16. pbl = BinaryClf(n_features=X.shape[1]) clf_no_removal = OneSlackSSVM(model=pbl, max_iter=500, C=1, inactive_window=0, tol=0.01) clf_no_removal.fit(X, y) clf = OneSlackSSVM(model=pbl, max_iter=500, C=1, tol=0.01, inactive_threshold=1e-8) clf.fit(X, y) # check that we learned something assert_greater(clf.score(X, y), .92) # results are mostly equal # if we decrease tol, they will get more similar assert_less(np.mean(clf.predict(X) != clf_no_removal.predict(X)), 0.02) # without removal, have as many constraints as iterations assert_equal(len(clf_no_removal.objective_curve_), len(clf_no_removal.constraints_)) # with removal, there are less constraints than iterations assert_less(len(clf.constraints_), len(clf.objective_curve_))
def test_standard_svm_blobs_2d_class_weight(): # no edges, reduce to crammer-singer svm X, Y = make_blobs(n_samples=210, centers=3, random_state=1, cluster_std=3, shuffle=False) X = np.hstack([X, np.ones((X.shape[0], 1))]) X, Y = X[:170], Y[:170] X_graphs = [(x[np.newaxis, :], np.empty((0, 2), dtype=np.int)) for x in X] pbl = GraphCRF(n_features=3, n_states=3, inference_method='unary') svm = OneSlackSSVM(pbl, check_constraints=False, C=1000) svm.fit(X_graphs, Y[:, np.newaxis]) weights = 1. / np.bincount(Y) weights *= len(weights) / np.sum(weights) pbl_class_weight = GraphCRF(n_features=3, n_states=3, class_weight=weights, inference_method='unary') svm_class_weight = OneSlackSSVM(pbl_class_weight, C=10, check_constraints=False, break_on_bad=False) svm_class_weight.fit(X_graphs, Y[:, np.newaxis]) assert_greater(f1_score(Y, np.hstack(svm_class_weight.predict(X_graphs))), f1_score(Y, np.hstack(svm.predict(X_graphs))))
def test_constraint_removal(): digits = load_digits() X, y = digits.data, digits.target y = 2 * (y % 2) - 1 # even vs odd as +1 vs -1 X = X / 16. pbl = BinarySVMModel(n_features=X.shape[1]) clf_no_removal = OneSlackSSVM(model=pbl, max_iter=500, verbose=1, C=10, inactive_window=0, tol=0.01) clf_no_removal.fit(X, y) clf = OneSlackSSVM(model=pbl, max_iter=500, verbose=1, C=10, tol=0.01, inactive_threshold=1e-8) clf.fit(X, y) # results are mostly equal # if we decrease tol, they will get more similar assert_less(np.mean(clf.predict(X) != clf_no_removal.predict(X)), 0.02) # without removal, have as many constraints as iterations # +1 for true y constraint assert_equal(len(clf_no_removal.objective_curve_) + 1, len(clf_no_removal.constraints_)) # with removal, there are less constraints than iterations assert_less(len(clf.constraints_), len(clf.objective_curve_))
def main(): print("Please be patient. Will take 5-20 minutes.") snakes = load_snakes() X_train, Y_train = snakes['X_train'], snakes['Y_train'] X_train = [one_hot_colors(x) for x in X_train] Y_train_flat = [y_.ravel() for y_ in Y_train] X_train_directions, X_train_edge_features = prepare_data(X_train) # first, train on X with directions only: crf = EdgeFeatureGraphCRF(inference_method='qpbo') ssvm = OneSlackSSVM(crf, inference_cache=50, C=.1, tol=.1, switch_to='ad3', n_jobs=-1) ssvm.fit(X_train_directions, Y_train_flat) # Evaluate using confusion matrix. # Clearly the middel of the snake is the hardest part. X_test, Y_test = snakes['X_test'], snakes['Y_test'] X_test = [one_hot_colors(x) for x in X_test] Y_test_flat = [y_.ravel() for y_ in Y_test] X_test_directions, X_test_edge_features = prepare_data(X_test) Y_pred = ssvm.predict(X_test_directions) print("Results using only directional features for edges") print("Test accuracy: %.3f" % accuracy_score(np.hstack(Y_test_flat), np.hstack(Y_pred))) print(confusion_matrix(np.hstack(Y_test_flat), np.hstack(Y_pred))) # now, use more informative edge features: crf = EdgeFeatureGraphCRF(inference_method='qpbo') ssvm = OneSlackSSVM(crf, inference_cache=50, C=.1, tol=.1, switch_to='ad3', n_jobs=-1) ssvm.fit(X_train_edge_features, Y_train_flat) Y_pred2 = ssvm.predict(X_test_edge_features) print("Results using also input features for edges") print("Test accuracy: %.3f" % accuracy_score(np.hstack(Y_test_flat), np.hstack(Y_pred2))) print(confusion_matrix(np.hstack(Y_test_flat), np.hstack(Y_pred2))) # plot stuff fig, axes = plt.subplots(2, 2) axes[0, 0].imshow(snakes['X_test'][0], interpolation='nearest') axes[0, 0].set_title('Input') y = Y_test[0].astype(np.int) bg = 2 * (y != 0) # enhance contrast axes[0, 1].matshow(y + bg, cmap=plt.cm.Greys) axes[0, 1].set_title("Ground Truth") axes[1, 0].matshow(Y_pred[0].reshape(y.shape) + bg, cmap=plt.cm.Greys) axes[1, 0].set_title("Prediction w/o edge features") axes[1, 1].matshow(Y_pred2[0].reshape(y.shape) + bg, cmap=plt.cm.Greys) axes[1, 1].set_title("Prediction with edge features") for a in axes.ravel(): a.set_xticks(()) a.set_yticks(()) plt.show() from IPython.core.debugger import Tracer Tracer()()
def test_one_slack_constraint_caching(): # testing cutting plane ssvm on easy multinomial dataset X, Y = generate_blocks_multinomial(n_samples=10, noise=0.5, seed=0, size_x=9) n_labels = len(np.unique(Y)) exact_inference = get_installed([('ad3', {'branch_and_bound': True}), "lp"])[0] crf = GridCRF(n_states=n_labels, inference_method=exact_inference) clf = OneSlackSSVM(model=crf, max_iter=150, C=1, check_constraints=True, break_on_bad=True, inference_cache=50, inactive_window=0) clf.fit(X, Y) Y_pred = clf.predict(X) assert_array_equal(Y, Y_pred) assert_equal(len(clf.inference_cache_), len(X)) # there should be 13 constraints, which are less than the 94 iterations # that are done # check that we didn't change the behavior of how we construct the cache constraints_per_sample = [len(cache) for cache in clf.inference_cache_] if exact_inference == "lp": assert_equal(len(clf.inference_cache_[0]), 18) assert_equal(np.max(constraints_per_sample), 18) assert_equal(np.min(constraints_per_sample), 18) else: assert_equal(len(clf.inference_cache_[0]), 13) assert_equal(np.max(constraints_per_sample), 20) assert_equal(np.min(constraints_per_sample), 11)
def test_one_slack_constraint_caching(): #testing cutting plane ssvm on easy multinomial dataset X, Y = generate_blocks_multinomial(n_samples=10, noise=0.5, seed=0, size_x=9) n_labels = len(np.unique(Y)) crf = GridCRF(n_states=n_labels, inference_method='lp') clf = OneSlackSSVM(model=crf, max_iter=150, C=1, check_constraints=True, break_on_bad=True, inference_cache=50, inactive_window=0, verbose=10) clf.fit(X, Y) Y_pred = clf.predict(X) assert_array_equal(Y, Y_pred) assert_equal(len(clf.inference_cache_), len(X)) # there should be 11 constraints, which are less than the 94 iterations # that are done assert_equal(len(clf.inference_cache_[0]), 11) # check that we didn't change the behavior of how we construct the cache constraints_per_sample = [len(cache) for cache in clf.inference_cache_] assert_equal(np.max(constraints_per_sample), 19) assert_equal(np.min(constraints_per_sample), 11)
def msrc(): models_basedir = 'models/msrc/' crf = EdgeCRF(n_states=24, n_features=2028, n_edge_features=4, inference_method='gco') clf = OneSlackSSVM(crf, max_iter=10000, C=0.01, verbose=2, tol=0.1, n_jobs=4, inference_cache=100) X, Y = load_msrc('train') Y = remove_areas(Y) start = time() clf.fit(X, Y) stop = time() np.savetxt(models_basedir + 'msrc_full.csv', clf.w) with open(models_basedir + 'msrc_full' + '.pickle', 'w') as f: cPickle.dump(clf, f) X, Y = load_msrc('test') Y = remove_areas(Y) Y_pred = clf.predict(X) print 'Error on test set: %f' % compute_error(Y, Y_pred) print 'Score on test set: %f' % clf.score(X, Y) print 'Norm of weight vector: |w|=%f' % np.linalg.norm(clf.w) print 'Elapsed time: %f s' % (stop - start) return clf
def syntetic_test(): # test model on different train set size & on different train sets results = np.zeros((18, 5)) full_labeled = np.array([2, 4, 10, 25, 100]) train_size = 400 for dataset in xrange(1, 19): X, Y = load_syntetic(dataset) for j, nfull in enumerate(full_labeled): crf = EdgeCRF(n_states=10, n_features=10, n_edge_features=2, inference_method='qpbo') clf = OneSlackSSVM(crf, max_iter=10000, C=0.01, verbose=0, tol=0.1, n_jobs=4, inference_cache=100) x_train = X[:nfull] y_train = Y[:nfull] x_test = X[(train_size + 1):] y_test = Y[(train_size + 1):] try: clf.fit(x_train, y_train) y_pred = clf.predict(x_test) results[dataset - 1, j] = compute_error(y_test, y_pred) print 'dataset=%d, nfull=%d, error=%f' % (dataset, nfull, results[dataset - 1, j]) except ValueError: print 'dataset=%d, nfull=%d: Failed' % (dataset, nfull) np.savetxt('results/syntetic/full_labeled.txt', results)
def test_binary_blocks_one_slack_graph(): #testing cutting plane ssvm on easy binary dataset # generate graphs explicitly for each example X, Y = generate_blocks(n_samples=3) crf = GraphCRF(inference_method=inference_method) clf = OneSlackSSVM(model=crf, max_iter=100, C=1, check_constraints=True, break_on_bad=True, n_jobs=1, tol=.1) x1, x2, x3 = X y1, y2, y3 = Y n_states = len(np.unique(Y)) # delete some rows to make it more fun x1, y1 = x1[:, :-1], y1[:, :-1] x2, y2 = x2[:-1], y2[:-1] # generate graphs X_ = [x1, x2, x3] G = [make_grid_edges(x) for x in X_] # reshape / flatten x and y X_ = [x.reshape(-1, n_states) for x in X_] Y = [y.ravel() for y in [y1, y2, y3]] X = list(zip(X_, G)) clf.fit(X, Y) Y_pred = clf.predict(X) for y, y_pred in zip(Y, Y_pred): assert_array_equal(y, y_pred)
def msrc(): models_basedir = 'models/msrc/' crf = EdgeCRF(n_states=24, n_features=2028, n_edge_features=4, inference_method='gco') clf = OneSlackSSVM(crf, max_iter=10000, C=0.01, verbose=2, tol=0.1, n_jobs=4, inference_cache=100) X, Y = load_msrc('train') Y = remove_areas(Y) start = time() clf.fit(X, Y) stop = time() np.savetxt(models_basedir + 'msrc_full.csv', clf.w) with open(models_basedir + 'msrc_full' + '.pickle', 'w') as f: pickle.dump(clf, f) X, Y = load_msrc('test') Y = remove_areas(Y) Y_pred = clf.predict(X) print('Error on test set: %f' % compute_error(Y, Y_pred)) print('Score on test set: %f' % clf.score(X, Y)) print('Norm of weight vector: |w|=%f' % np.linalg.norm(clf.w)) print('Elapsed time: %f s' % (stop - start)) return clf
def test_binary_blocks_one_slack_graph(): #testing cutting plane ssvm on easy binary dataset # generate graphs explicitly for each example for inference_method in ["dai", "lp"]: print("testing %s" % inference_method) X, Y = toy.generate_blocks(n_samples=3) crf = GraphCRF(inference_method=inference_method) clf = OneSlackSSVM(problem=crf, max_iter=100, C=100, verbose=100, check_constraints=True, break_on_bad=True, n_jobs=1) x1, x2, x3 = X y1, y2, y3 = Y n_states = len(np.unique(Y)) # delete some rows to make it more fun x1, y1 = x1[:, :-1], y1[:, :-1] x2, y2 = x2[:-1], y2[:-1] # generate graphs X_ = [x1, x2, x3] G = [make_grid_edges(x) for x in X_] # reshape / flatten x and y X_ = [x.reshape(-1, n_states) for x in X_] Y = [y.ravel() for y in [y1, y2, y3]] X = zip(X_, G) clf.fit(X, Y) Y_pred = clf.predict(X) for y, y_pred in zip(Y, Y_pred): assert_array_equal(y, y_pred)
def train(x_train, y_train, x_test, y_test): x_train = np.asarray(x_train, dtype=np.float) y_train = np.asarray(y_train, dtype=np.int64) # x_test = np.asarray(x_test, dtype=np.float) # y_test = np.asarray(y_test, dtype=np.int64) x_test = x_train y_test = y_train from pystruct.learners import NSlackSSVM, OneSlackSSVM, SubgradientSSVM, LatentSSVM, SubgradientLatentSSVM, PrimalDSStructuredSVM from pystruct.models import MultiLabelClf, MultiClassClf clf = OneSlackSSVM(MultiLabelClf(), C=1, show_loss_every=1, verbose=1, max_iter=1000) # print(x_train, y_train) # input() clf.fit(x_train, y_train) result = clf.predict(x_test) print('Result: \n', result) print('True label:\n', y_test) clf.score(x_test, y_test) print('\n') count = 0 for i in range(len(result)): # print(np.sum(np.square(y_test[i]-result[i]))) if np.sum(np.square(y_test[i] - result[i])) != 0: print('True label: ', y_test[i], 'Predict: ', result[i]) count += 1 print(count) translate_vector(x_test, y_test)
def syntetic(): # train model on a single set models_basedir = 'models/syntetic/' crf = EdgeCRF(n_states=10, n_features=10, n_edge_features=2, inference_method='gco') clf = OneSlackSSVM(crf, max_iter=10000, C=0.01, verbose=2, tol=0.1, n_jobs=4, inference_cache=100) X, Y = load_syntetic(1) x_train, x_test, y_train, y_test = train_test_split(X, Y, train_size=100, random_state=179) start = time() clf.fit(x_train, y_train) stop = time() np.savetxt(models_basedir + 'syntetic_full.csv', clf.w) with open(models_basedir + 'syntetic_full' + '.pickle', 'w') as f: cPickle.dump(clf, f) y_pred = clf.predict(x_test) print 'Error on test set: %f' % compute_error(y_test, y_pred) print 'Score on test set: %f' % clf.score(x_test, y_test) print 'Score on train set: %f' % clf.score(x_train, y_train) print 'Norm of weight vector: |w|=%f' % np.linalg.norm(clf.w) print 'Elapsed time: %f s' % (stop - start) return clf
def syntetic_test(): # test model on different train set size & on different train sets results = np.zeros((18, 5)) full_labeled = np.array([2, 4, 10, 25, 100]) train_size = 400 for dataset in range(1, 19): X, Y = load_syntetic(dataset) for j, nfull in enumerate(full_labeled): crf = EdgeCRF(n_states=10, n_features=10, n_edge_features=2, inference_method='qpbo') clf = OneSlackSSVM(crf, max_iter=10000, C=0.01, verbose=0, tol=0.1, n_jobs=4, inference_cache=100) x_train = X[:nfull] y_train = Y[:nfull] x_test = X[(train_size + 1):] y_test = Y[(train_size + 1):] try: clf.fit(x_train, y_train) y_pred = clf.predict(x_test) results[dataset - 1, j] = compute_error(y_test, y_pred) print('dataset=%d, nfull=%d, error=%f' % (dataset, nfull, results[dataset - 1, j])) except ValueError: print('dataset=%d, nfull=%d: Failed' % (dataset, nfull)) np.savetxt('results/syntetic/full_labeled.txt', results)
def CRF_pred(eeg1, eeg2, emg, y, eeg1test, eeg2test, emgtest, C=0.9, weight_shift=0, max_iter=1000, fs=128): # For the ith iteration, select as trainin the sub_indices other than those at index i for train_index eeg1_train = eeg1.values eeg2_train = eeg2.values emg_train = emg.values y_train = y.values # The test subject is the one at index i eeg1_test = eeg1test.values eeg2_test = eeg2test.values emg_test = emgtest.values # CRF Model Preprocessing eeg1_ = process_EEG(eeg1_train) eeg2_ = process_EEG(eeg2_train) emg_ = process_EMG(emg_train) xtrain_ = np.concatenate((eeg1_, eeg2_, emg_), axis=1) ytrain_classes = np.reshape(y_train, (y_train.shape[0], )) ytrain_ = y_train eeg1_ = process_EEG(eeg1_test) eeg2_ = process_EEG(eeg2_test) emg_ = process_EMG(emg_test) xtest_ = np.concatenate((eeg1_, eeg2_, emg_), axis=1) xtrain_crf = np.reshape( xtrain_, (3, -1, xtrain_.shape[1])) # Reshape so that it works with CRF ytrain_crf = np.reshape(ytrain_, (3, -1)) - 1 # Reshape so that it works with CRF # CRF Model fitting: classes = np.unique(ytrain_) weights_crf = compute_class_weight("balanced", list(classes), list(ytrain_classes)) weights_crf[0] = weights_crf[0] + (2.5 * weight_shift) weights_crf[1] = weights_crf[1] + (1.5 * weight_shift) model = ChainCRF(class_weight=weights_crf) ssvm = OneSlackSSVM(model=model, C=C, max_iter=max_iter) ssvm.fit(xtrain_crf, ytrain_crf) # Test on the third guy xtest_crf = np.reshape(xtest_, (2, -1, xtest_.shape[1])) y_pred_crf = ssvm.predict(xtest_crf) y_pred_crf = np.asarray(y_pred_crf).reshape(-1) + 1 return y_pred_crf
def losocv_CRF_prepro(xtrain, y, C=0.5, weight_shift=0, max_iter=1000, fs=128): """Leave one subject out cross validation for the CRF model becasuse it requires special datahandling. Input should be a Pandas Dataframe.""" epochs = 21600 num_sub = 3 # Indices of the subjects sub_indices = [ np.arange(0, epochs), np.arange(epochs, epochs * 2), np.arange(epochs * 2, epochs * 3) ] res = [] for i in range(len(sub_indices)): # For the ith iteration, select as trainin the sub_indices other than those at index i for train_index train_index = np.concatenate( [sub_indices[(i + 1) % num_sub], sub_indices[(i + 2) % num_sub]]) xtrain_ = xtrain[train_index] y_train = y.values[train_index] ytrain_ = y_train # The test subject is the one at index i test_index = sub_indices[i] xtest_ = xtrain[test_index] y_test = y.values[test_index] ytest_ = y_test # CRF Model Preprocessing ytrain_classes = np.reshape(y_train, (y_train.shape[0], )) xtrain_crf = np.reshape( xtrain_, (2, -1, xtrain_.shape[1])) # Reshape so that it works with CRF ytrain_crf = np.reshape( ytrain_, (2, -1)) - 1 # Reshape so that it works with CRF # CRF Model fitting: classes = np.unique(ytrain_) weights_crf = compute_class_weight("balanced", list(classes), list(ytrain_classes)) weights_crf[0] = weights_crf[0] + (2.5 * weight_shift) weights_crf[1] = weights_crf[1] + (1.5 * weight_shift) model = ChainCRF(class_weight=weights_crf) ssvm = OneSlackSSVM(model=model, C=C, max_iter=max_iter) ssvm.fit(xtrain_crf, ytrain_crf) # Test on the third guy xtest_crf = np.reshape(xtest_, (1, -1, xtest_.shape[1])) ytest_crf = np.reshape(ytest_, (1, -1)) - 1 y_pred_crf = ssvm.predict(xtest_crf) y_pred_crf = np.asarray(y_pred_crf).reshape(-1) + 1 resy = sklearn.metrics.balanced_accuracy_score(ytest_, y_pred_crf) print("Iteration, result:", i, resy) res.append(resy) return res
def test(nfiles): X = [] Y = [] X_tst = [] Y_tst = [] ntrain = nfiles ntest = 5*nfiles print("Training/testing with %d/%d files." % (ntrain,ntest)) start = time.clock() filename = '../maps/MAPS_AkPnCGdD_2/AkPnCGdD/MUS' #filename = '../maps/MAPS_AkPnCGdD_1/AkPnCGdD/ISOL/NO' files = dirs.get_files_with_extension(filename, '.mid') train_files = files[:ntrain] print("\t" + str(train_files)) #test_files = files[ntrain:ntest+ntrain] # for legit testing test_files = files[-ntest:] map(per_file, train_files, it.repeat(X, ntrain), it.repeat(Y, ntrain)) map(per_file, test_files, it.repeat(X_tst, ntest), it.repeat(Y_tst, ntest)) end = time.clock() print("\tRead time: %f" % (end - start)) print("\tnWindows train: " + str([X[i].shape[0] for i in range(len(X))])) start = time.clock() crf = ChainCRF(n_states=2) clf = OneSlackSSVM(model=crf, C=100, n_jobs=-1, inference_cache=100, tol=.1) clf.fit(np.array(X), np.array(Y)) end = time.clock() print("\tTrain time: %f" % (end - start)) start = time.clock() Y_pred = clf.predict(X_tst) comp = [] for i in range(len(Y_tst)): for j in range(len(Y_tst[i])): comp.append((Y_tst[i][j], Y_pred[i][j])) print Y_tst[i][j], print for j in range(len(Y_tst[i])): print Y_pred[i][j], print print print("\tTrue positives: %d" % comp.count((1,1))) print("\tTrue negatives: %d" % comp.count((0,0))) print("\tFalse positives: %d" % comp.count((0,1))) print("\tFalse negatives: %d" % comp.count((1,0))) end = time.clock() print("\tTest time: %f" % (end - start))
def model_test(k, head, tail): """ CRF训练和预测 """ each_fold_time = time.time() #开始计时 #divide train set and test set train_id = dataId[head:tail] test_id = dataId[:head] + dataId[tail:] X_train = X_arr[train_id, :] Y_train = Y_arr[train_id] X_test = X_arr[test_id, :] Y_test = Y_arr[test_id] campTest = Camp_arr[test_id] #ends divide train set and test set if len(X_train) > 0: #实例化CRF EFGCRF = EdgeFeatureGraphCRF(inference_method='qpbo', class_weight=CLASS_WEIGHT) if LEARNER == "OneSlackSSVM": #利用OneSlackSSVM训练模型参数 ssvm = OneSlackSSVM(EFGCRF, C=.1, tol=.1, max_iter=100, switch_to='ad3') elif LEARNER == "FrankWolfeSSVM": #利用FrankWolfeSSVM训练模型参数 ssvm = FrankWolfeSSVM(EFGCRF, C=.1, tol=.1, max_iter=100) else: #没有选择分类器退出 pass ssvm.fit(X_train, Y_train) Y_pred = ssvm.predict(X_test) df_result = statistic_result(Y_pred, Y_test, campTest) V_precision = precision_score(df_result["label"], df_result["pred"]) V_recall = recall_score(df_result["label"], df_result["pred"]) V_f1 = f1_score(df_result["label"], df_result["pred"]) camps_pred, camps_lbl = statistic_campaign_result(Y_pred, Y_test) C_precision = precision_score(camps_lbl, camps_pred) C_recall = recall_score(camps_lbl, camps_pred) C_f1 = f1_score(camps_lbl, camps_pred) result_Queue.put( [V_precision, V_recall, V_f1, C_precision, C_recall, C_f1]) else: print("TRAIN SET is NULL") print("the {}th fold using time: {:.4f} min".format( k + 1, (time.time() - each_fold_time) / 60)) del X_train, Y_train, X_test, Y_test, Y_pred, campTest
def test_multinomial_blocks_one_slack(): #testing cutting plane ssvm on easy multinomial dataset X, Y = generate_blocks_multinomial(n_samples=10, noise=0.5, seed=0) n_labels = len(np.unique(Y)) crf = GridCRF(n_states=n_labels, inference_method=inference_method) clf = OneSlackSSVM(model=crf, max_iter=150, C=1, check_constraints=True, break_on_bad=True, tol=.1, inference_cache=50) clf.fit(X, Y) Y_pred = clf.predict(X) assert_array_equal(Y, Y_pred)
def test_one_slack_attractive_potentials(): # test that submodular SSVM can learn the block dataset X, Y = generate_blocks(n_samples=10) crf = GridCRF(inference_method=inference_method) submodular_clf = OneSlackSSVM( model=crf, max_iter=200, C=1, check_constraints=True, negativity_constraint=[5], inference_cache=50 ) submodular_clf.fit(X, Y) Y_pred = submodular_clf.predict(X) assert_array_equal(Y, Y_pred) assert_true(submodular_clf.w[5] < 0)
def test_equal_class_weights(): # test that equal class weight is the same as no class weight X, Y = make_blobs(n_samples=80, centers=3, random_state=42) X = np.hstack([X, np.ones((X.shape[0], 1))]) X_train, X_test, Y_train, Y_test = X[:40], X[40:], Y[:40], Y[40:] pbl = MultiClassClf(n_features=3, n_classes=3) svm = OneSlackSSVM(pbl, C=10) svm.fit(X_train, Y_train) predict_no_class_weight = svm.predict(X_test) pbl_class_weight = MultiClassClf(n_features=3, n_classes=3, class_weight=np.ones(3)) svm_class_weight = OneSlackSSVM(pbl_class_weight, C=10) svm_class_weight.fit(X_train, Y_train) predict_class_weight = svm_class_weight.predict(X_test) assert_array_equal(predict_no_class_weight, predict_class_weight) assert_array_almost_equal(svm.w, svm_class_weight.w)
def test_class_weights(): X, Y = make_blobs(n_samples=210, centers=3, random_state=1, cluster_std=3, shuffle=False) X = np.hstack([X, np.ones((X.shape[0], 1))]) X, Y = X[:170], Y[:170] pbl = MultiClassClf(n_features=3, n_classes=3) svm = OneSlackSSVM(pbl, C=10) svm.fit(X, Y) weights = 1. / np.bincount(Y) weights *= len(weights) / np.sum(weights) pbl_class_weight = MultiClassClf(n_features=3, n_classes=3, class_weight=weights) svm_class_weight = OneSlackSSVM(pbl_class_weight, C=10) svm_class_weight.fit(X, Y) assert_greater(f1_score(Y, svm_class_weight.predict(X)), f1_score(Y, svm.predict(X)))
def test_one_slack_attractive_potentials(): # test that submodular SSVM can learn the block dataset X, Y = toy.generate_blocks(n_samples=10) crf = GridCRF() submodular_clf = OneSlackSSVM(model=crf, max_iter=200, C=100, verbose=1, check_constraints=True, positive_constraint=[5], n_jobs=-1) submodular_clf.fit(X, Y) Y_pred = submodular_clf.predict(X) assert_array_equal(Y, Y_pred) assert_true(submodular_clf.w[5] < 0) # don't ask me about signs
def test_class_weights(): # test that equal class weight is the same as no class weight X, Y = make_blobs(n_samples=210, centers=3, random_state=1, cluster_std=3, shuffle=False) X = np.hstack([X, np.ones((X.shape[0], 1))]) X, Y = X[:170], Y[:170] pbl = CrammerSingerSVMModel(n_features=3, n_classes=3) svm = OneSlackSSVM(pbl, verbose=10, C=10) svm.fit(X, Y) weights = 1. / np.bincount(Y) weights *= len(weights) / np.sum(weights) pbl_class_weight = CrammerSingerSVMModel(n_features=3, n_classes=3, class_weight=weights) svm_class_weight = OneSlackSSVM(pbl_class_weight, verbose=10, C=10) svm_class_weight.fit(X, Y) assert_greater(f1_score(Y, svm_class_weight.predict(X)), f1_score(Y, svm.predict(X)))
def test_one_slack_repellent_potentials(): # test non-submodular problem with and without submodularity constraint # dataset is checkerboard X, Y = generate_checker() crf = GridCRF(inference_method=inference_method) clf = OneSlackSSVM(model=crf, max_iter=10, C=0.01, check_constraints=True) clf.fit(X, Y) Y_pred = clf.predict(X) # standard crf can predict perfectly assert_array_equal(Y, Y_pred) submodular_clf = OneSlackSSVM( model=crf, max_iter=10, C=0.01, check_constraints=True, negativity_constraint=[4, 5, 6] ) submodular_clf.fit(X, Y) Y_pred = submodular_clf.predict(X) assert_less(submodular_clf.score(X, Y), 0.99) # submodular crf can not do better than unaries for i, x in enumerate(X): y_pred_unaries = crf.inference(x, np.array([1, 0, 0, 1, 0, 0, 0])) assert_array_equal(y_pred_unaries, Y_pred[i])
def test_one_slack_attractive_potentials(): # test that submodular SSVM can learn the block dataset X, Y = generate_blocks(n_samples=10) crf = GridCRF(inference_method=inference_method) submodular_clf = OneSlackSSVM(model=crf, max_iter=200, C=1, check_constraints=True, negativity_constraint=[5], inference_cache=50) submodular_clf.fit(X, Y) Y_pred = submodular_clf.predict(X) assert_array_equal(Y, Y_pred) assert_true(submodular_clf.w[5] < 0)
def test_multinomial_blocks_one_slack(): #testing cutting plane ssvm on easy multinomial dataset X, Y = toy.generate_blocks_multinomial(n_samples=10, noise=0.3, seed=0) n_labels = len(np.unique(Y)) for inference_method in ['lp']: crf = GridCRF(n_states=n_labels, inference_method=inference_method) clf = OneSlackSSVM(problem=crf, max_iter=50, C=100, verbose=100, check_constraints=True, break_on_bad=True) clf.fit(X, Y) Y_pred = clf.predict(X) assert_array_equal(Y, Y_pred)
def test_blobs_2d_one_slack(): # make two gaussian blobs X, Y = make_blobs(n_samples=80, centers=3, random_state=42) # we have to add a constant 1 feature by hand :-/ X = np.hstack([X, np.ones((X.shape[0], 1))]) X_train, X_test, Y_train, Y_test = X[:40], X[40:], Y[:40], Y[40:] pbl = MultiClassClf(n_features=3, n_classes=3) svm = OneSlackSSVM(pbl, check_constraints=True, C=1000) svm.fit(X_train, Y_train) assert_array_equal(Y_test, np.hstack(svm.predict(X_test)))
def test_one_slack_repellent_potentials(): # test non-submodular problem with and without submodularity constraint # dataset is checkerboard X, Y = generate_checker() crf = GridCRF(inference_method=inference_method) clf = OneSlackSSVM(model=crf, max_iter=10, C=.01, check_constraints=True) clf.fit(X, Y) Y_pred = clf.predict(X) # standard crf can predict perfectly assert_array_equal(Y, Y_pred) submodular_clf = OneSlackSSVM(model=crf, max_iter=10, C=.01, check_constraints=True, negativity_constraint=[4, 5, 6]) submodular_clf.fit(X, Y) Y_pred = submodular_clf.predict(X) assert_less(submodular_clf.score(X, Y), .99) # submodular crf can not do better than unaries for i, x in enumerate(X): y_pred_unaries = crf.inference(x, np.array([1, 0, 0, 1, 0, 0, 0])) assert_array_equal(y_pred_unaries, Y_pred[i])
def test_blobs_2d_one_slack(): # make two gaussian blobs X, Y = make_blobs(n_samples=80, centers=2, random_state=1) Y = 2 * Y - 1 # we have to add a constant 1 feature by hand :-/ X = np.hstack([X, np.ones((X.shape[0], 1))]) X_train, X_test, Y_train, Y_test = X[:40], X[40:], Y[:40], Y[40:] pbl = BinarySVMModel(n_features=3) svm = OneSlackSSVM(pbl, verbose=30, C=1000) svm.fit(X_train, Y_train) assert_array_equal(Y_test, np.hstack(svm.predict(X_test)))
def test_one_slack_repellent_potentials(): # test non-submodular learning with and without positivity constraint # dataset is checkerboard X, Y = toy.generate_checker() for inference_method in ["lp", "qpbo", "ad3"]: crf = GridCRF(inference_method=inference_method) clf = OneSlackSSVM(model=crf, max_iter=10, C=100, verbose=0, check_constraints=True, n_jobs=-1) clf.fit(X, Y) Y_pred = clf.predict(X) # standard crf can predict perfectly assert_array_equal(Y, Y_pred) submodular_clf = OneSlackSSVM(model=crf, max_iter=10, C=100, verbose=0, check_constraints=True, positive_constraint=[4, 5, 6], n_jobs=-1) submodular_clf.fit(X, Y) Y_pred = submodular_clf.predict(X) # submodular crf can not do better than unaries for i, x in enumerate(X): y_pred_unaries = crf.inference(x, np.array([1, 0, 0, 1, 0, 0, 0])) assert_array_equal(y_pred_unaries, Y_pred[i])
def test_class_weights(): X, Y = make_blobs(n_samples=210, centers=3, random_state=1, cluster_std=3, shuffle=False) X = np.hstack([X, np.ones((X.shape[0], 1))]) X, Y = X[:170], Y[:170] pbl = MultiClassClf(n_features=3, n_classes=3) svm = OneSlackSSVM(pbl, C=10) svm.fit(X, Y) weights = 1. / np.bincount(Y) weights *= len(weights) / np.sum(weights) pbl_class_weight = MultiClassClf(n_features=3, n_classes=3, class_weight=weights) svm_class_weight = OneSlackSSVM(pbl_class_weight, C=10) svm_class_weight.fit(X, Y) assert_greater(f1_score(Y, svm_class_weight.predict(X), average='macro'), f1_score(Y, svm.predict(X), average='macro'))
def fit_crf(self): for C in self.C_range: print("Testing C value: {}".format(C)) model = EdgeFeatureGraphCRF(inference_method="ad3") ssvm = OneSlackSSVM(model, inference_cache=50, C=C, tol=self.tol, max_iter=self.max_iter, n_jobs=4, verbose=False) ssvm.fit(self.X_train, self.y_train) predictions = [x for x in ssvm.predict(self.X_dev)] self.evaluate_predictions(predictions, self.y_dev) # Fit against the whole dataset except test # Is this approach correct? model = EdgeFeatureGraphCRF(inference_method="ad3") ssvm = OneSlackSSVM(model, inference_cache=50, C=0.03, tol=self.tol, max_iter=self.max_iter, n_jobs=4, verbose=False) X_train_dev = np.concatenate([self.X_train, self.X_dev]) y_train_dev = np.concatenate([self.y_train, self.y_dev]) ssvm.fit(X_train_dev, y_train_dev) if self.eval_against_test: predictions = [x for x in ssvm.predict(self.X_test)] print("Test set evaluation") self.evaluate_predictions(predictions, self.y_test) self.model = ssvm return ssvm
def test_standard_svm_blobs_2d(): # no edges, reduce to crammer-singer svm X, Y = make_blobs(n_samples=80, centers=3, random_state=42) # we have to add a constant 1 feature by hand :-/ X = np.hstack([X, np.ones((X.shape[0], 1))]) X_train, X_test, Y_train, Y_test = X[:40], X[40:], Y[:40], Y[40:] X_train_graphs = [(x[np.newaxis, :], np.empty((0, 2), dtype=np.int)) for x in X_train] X_test_graphs = [(x[np.newaxis, :], np.empty((0, 2), dtype=np.int)) for x in X_test] pbl = GraphCRF(n_features=3, n_states=3) svm = OneSlackSSVM(pbl, verbose=10, check_constraints=True, C=1000) svm.fit(X_train_graphs, Y_train[:, np.newaxis]) assert_array_equal(Y_test, np.hstack(svm.predict(X_test_graphs)))
def msrc_test(): # test model on different train set sizes basedir = '../data/msrc/trainmasks/' models_basedir = 'models/msrc/' quality = [] Xtest, Ytest = load_msrc('test') Ytest = remove_areas(Ytest) Xtrain, Ytrain = load_msrc('train') Ytrain = remove_areas(Ytrain) for n_train in [20, 40, 80, 160, 276]: crf = EdgeCRF(n_states=24, n_features=2028, n_edge_features=4, inference_method='gco') clf = OneSlackSSVM(crf, max_iter=1000, C=0.01, verbose=0, tol=0.1, n_jobs=4, inference_cache=100) if n_train != 276: train_mask = np.genfromtxt(basedir + 'trainMaskX%d.txt' % n_train) train_mask = train_mask[:277].astype(np.bool) else: train_mask = np.ones(276).astype(np.bool) curX = [] curY = [] for (s, x, y) in zip(train_mask, Xtrain, Ytrain): if s: curX.append(x) curY.append(y) start = time() clf.fit(curX, curY) stop = time() np.savetxt(models_basedir + 'test_model_%d.csv' % n_train, clf.w) with open(models_basedir + 'test_model_%d' % n_train + '.pickle', 'w') as f: pickle.dump(clf, f) Ypred = clf.predict(Xtest) q = 1 - compute_error(Ytest, Ypred) print('n_train=%d, quality=%f, time=%f' % (n_train, q, stop - start)) quality.append(q) np.savetxt('results/msrc/msrc_full.txt', quality)
def msrc_test(): # test model on different train set sizes basedir = '../data/msrc/trainmasks/' models_basedir = 'models/msrc/' quality = [] Xtest, Ytest = load_msrc('test') Ytest = remove_areas(Ytest) Xtrain, Ytrain = load_msrc('train') Ytrain = remove_areas(Ytrain) for n_train in [20, 40, 80, 160, 276]: crf = EdgeCRF(n_states=24, n_features=2028, n_edge_features=4, inference_method='gco') clf = OneSlackSSVM(crf, max_iter=1000, C=0.01, verbose=0, tol=0.1, n_jobs=4, inference_cache=100) if n_train != 276: train_mask = np.genfromtxt(basedir + 'trainMaskX%d.txt' % n_train) train_mask = train_mask[:277].astype(np.bool) else: train_mask = np.ones(276).astype(np.bool) curX = [] curY = [] for (s, x, y) in zip(train_mask, Xtrain, Ytrain): if s: curX.append(x) curY.append(y) start = time() clf.fit(curX, curY) stop = time() np.savetxt(models_basedir + 'test_model_%d.csv' % n_train, clf.w) with open(models_basedir + 'test_model_%d' % n_train + '.pickle', 'w') as f: cPickle.dump(clf, f) Ypred = clf.predict(Xtest) q = 1 - compute_error(Ytest, Ypred) print 'n_train=%d, quality=%f, time=%f' % (n_train, q, stop - start) quality.append(q) np.savetxt('results/msrc/msrc_full.txt', quality)
def conditional_random_fields(X, y): """ """ X_ = [(np.atleast_2d(x), np.empty((0, 2), dtype=np.int)) for x in X] Y = y.reshape(-1, 1) X_train, X_test, y_train, y_test = train_test_split(X_, Y) pbl = GraphCRF() svm = OneSlackSSVM(pbl) svm.fit(X_train, y_train) y_pred = np.vstack(svm.predict(X_test)) print("Score with pystruct crf svm: %f " % (np.mean(y_pred == y_test))) print classification_report(y_test, y_pred) plot_confusion_matrix(y_test, y_pred)
class PystructSequenceLearner(SequenceLearner): def __init__(self): # the model self.model = ChainCRF(directed=True) # the learner self.learner = OneSlackSSVM(model=self.model, C=.1, inference_cache=50, tol=0.1, n_jobs=1) # self.learner = StructuredPerceptron(model=self.model, average=True) def fit(self, X, y): # Train linear chain CRF self.learner.fit(X, y) def predict(self, X): return self.learner.predict(X)
def test_standard_svm_blobs_2d(): # no edges, reduce to crammer-singer svm X, Y = make_blobs(n_samples=80, centers=3, random_state=42) # we have to add a constant 1 feature by hand :-/ X = np.hstack([X, np.ones((X.shape[0], 1))]) X_train, X_test, Y_train, Y_test = X[:40], X[40:], Y[:40], Y[40:] X_train_graphs = [(x[np.newaxis, :], np.empty((0, 2), dtype=np.int)) for x in X_train] X_test_graphs = [(x[np.newaxis, :], np.empty((0, 2), dtype=np.int)) for x in X_test] pbl = GraphCRF(n_features=3, n_states=3, inference_method='unary') svm = OneSlackSSVM(pbl, check_constraints=True, C=1000) svm.fit(X_train_graphs, Y_train[:, np.newaxis]) assert_array_equal(Y_test, np.hstack(svm.predict(X_test_graphs)))
def test_one_slack_constraint_caching(): #testing cutting plane ssvm on easy multinomial dataset X, Y = toy.generate_blocks_multinomial(n_samples=10, noise=0.5, seed=0, size_x=9) n_labels = len(np.unique(Y)) crf = GridCRF(n_states=n_labels) clf = OneSlackSSVM(model=crf, max_iter=150, C=1, check_constraints=True, break_on_bad=True, inference_cache=50, inactive_window=0) clf.fit(X, Y) Y_pred = clf.predict(X) assert_array_equal(Y, Y_pred) assert_equal(len(clf.inference_cache_), len(X)) # there should be 21 constraints, which are less than the 94 iterations # that are done assert_equal(len(clf.inference_cache_[0]), 21) # check that we didn't change the behavior of how we construct the cache constraints_per_sample = [len(cache) for cache in clf.inference_cache_] assert_equal(np.max(constraints_per_sample), 21) assert_equal(np.min(constraints_per_sample), 21)
def test_one_slack_constraint_caching(): #testing cutting plane ssvm on easy multinomial dataset X, Y = toy.generate_blocks_multinomial(n_samples=10, noise=0.3, seed=0) n_labels = len(np.unique(Y)) crf = GridCRF(n_states=n_labels, inference_method='lp') clf = OneSlackSSVM(problem=crf, max_iter=50, C=100, verbose=100, check_constraints=True, break_on_bad=True, inference_cache=50) clf.fit(X, Y) Y_pred = clf.predict(X) assert_array_equal(Y, Y_pred) assert_equal(len(clf.inference_cache_), len(X)) # there should be 9 constraints, which are less than the 16 iterations # that are done assert_equal(len(clf.inference_cache_[0]), 9) # check that we didn't change the behavior of how we construct the cache constraints_per_sample = [len(cache) for cache in clf.inference_cache_] assert_equal(np.max(constraints_per_sample), 10) assert_equal(np.min(constraints_per_sample), 8)
def CRF_pred_prepro(xtrain, y, xtest, C=0.9, weight_shift=0, max_iter=1000, fs=128): y_train = y.values # CRF Model Preprocessing xtrain_ = xtrain ytrain_classes = np.reshape(y_train, (y_train.shape[0], )) ytrain_ = y_train xtest_ = xtest xtrain_crf = np.reshape( xtrain_, (3, -1, xtrain_.shape[1])) # Reshape so that it works with CRF ytrain_crf = np.reshape(ytrain_, (3, -1)) - 1 # Reshape so that it works with CRF # CRF Model fitting: classes = np.unique(ytrain_) weights_crf = compute_class_weight("balanced", list(classes), list(ytrain_classes)) weights_crf[0] = weights_crf[0] + (2.5 * weight_shift) weights_crf[1] = weights_crf[1] + (1.5 * weight_shift) model = ChainCRF(class_weight=weights_crf) ssvm = OneSlackSSVM(model=model, C=C, max_iter=max_iter) ssvm.fit(xtrain_crf, ytrain_crf) # Test on the third guy xtest_crf = np.reshape(xtest_, (2, -1, xtest_.shape[1])) y_pred_crf = ssvm.predict(xtest_crf) y_pred_crf = np.asarray(y_pred_crf).reshape(-1) + 1 return y_pred_crf
fw_bc_svm = FrankWolfeSSVM(model, C=.1, max_iter=50) fw_batch_svm = FrankWolfeSSVM(model, C=.1, max_iter=50, batch_mode=True) # n-slack cutting plane ssvm start = time() n_slack_svm.fit(X_train_bias, y_train) time_n_slack_svm = time() - start y_pred = np.hstack(n_slack_svm.predict(X_test_bias)) print("Score with pystruct n-slack ssvm: %f (took %f seconds)" % (np.mean(y_pred == y_test), time_n_slack_svm)) ## 1-slack cutting plane ssvm start = time() one_slack_svm.fit(X_train_bias, y_train) time_one_slack_svm = time() - start y_pred = np.hstack(one_slack_svm.predict(X_test_bias)) print("Score with pystruct 1-slack ssvm: %f (took %f seconds)" % (np.mean(y_pred == y_test), time_one_slack_svm)) #online subgradient ssvm start = time() subgradient_svm.fit(X_train_bias, y_train) time_subgradient_svm = time() - start y_pred = np.hstack(subgradient_svm.predict(X_test_bias)) print("Score with pystruct subgradient ssvm: %f (took %f seconds)" % (np.mean(y_pred == y_test), time_subgradient_svm)) # the standard one-vs-rest multi-class would probably be as good and faster # but solving a different model libsvm = LinearSVC(multi_class='crammer_singer', C=.1)
## string labels to label ids label2id = {l:i for i,l in enumerate(set(y_train_flat))} id2label = {l:i for i,l in label2id.items()} y_train = [np.array([label2id[tok[2]] for tok in sentence]) for sentence in iob_train] if args.oneslack: ### fit oneslack ssvm crf = ChainCRF() os_ssvm = OneSlackSSVM(crf, verbose=args.verbose, n_jobs=-1, use_memmapping_pool=0, show_loss_every=20, tol=0.01, cache_tol=0.1) os_ssvm.fit(list(X_train_tsvd), y_train) test_os_ssvm_preds = [[id2label[i] for i in sent] for sent in os_ssvm.predict(X_test_tsvd)] test_conll_os_ssvm = conlleval_fmt(iob_test, test_os_ssvm_preds) test_conll_os_ssvm_file = open('test_conll_os_ssvm.txt', 'wb') for sentence in test_conll_os_ssvm: test_conll_os_ssvm_file.write(bytes(sentence, 'UTF-8')) test_conll_os_ssvm_file.close() print(conlleval_results('test_conll_os_ssvm.txt')) if args.subgrad: ### fit subgradient ssvm crf = ChainCRF() sg_ssvm = SubgradientSSVM(crf, max_iter=200, verbose=args.verbose, n_jobs=-1, use_memmapping_pool=0, show_loss_every=20, shuffle=True) sg_ssvm.fit(list(X_train_tsvd), y_train) test_sg_ssvm_preds = [[id2label[i] for i in sent]
def main(): print("Please be patient. Will take 5-20 minutes.") snakes = load_snakes() X_train, Y_train = snakes['X_train'], snakes['Y_train'] X_train = [one_hot_colors(x) for x in X_train] Y_train_flat = [y_.ravel() for y_ in Y_train] X_train_directions, X_train_edge_features = prepare_data(X_train) if 'ogm' in get_installed(): inference = ('ogm', {'alg': 'fm'}) else: inference = 'qpbo' # first, train on X with directions only: crf = EdgeFeatureGraphCRF(inference_method=inference) ssvm = OneSlackSSVM(crf, inference_cache=50, C=.1, tol=.1, max_iter=100, n_jobs=1) ssvm.fit(X_train_directions, Y_train_flat) # Evaluate using confusion matrix. # Clearly the middel of the snake is the hardest part. X_test, Y_test = snakes['X_test'], snakes['Y_test'] X_test = [one_hot_colors(x) for x in X_test] Y_test_flat = [y_.ravel() for y_ in Y_test] X_test_directions, X_test_edge_features = prepare_data(X_test) Y_pred = ssvm.predict(X_test_directions) print("Results using only directional features for edges") print("Test accuracy: %.3f" % accuracy_score(np.hstack(Y_test_flat), np.hstack(Y_pred))) print(confusion_matrix(np.hstack(Y_test_flat), np.hstack(Y_pred))) # now, use more informative edge features: crf = EdgeFeatureGraphCRF(inference_method=inference) ssvm = OneSlackSSVM(crf, inference_cache=50, C=.1, tol=.1, switch_to='ad3', n_jobs=1) ssvm.fit(X_train_edge_features, Y_train_flat) Y_pred2 = ssvm.predict(X_test_edge_features) print("Results using also input features for edges") print("Test accuracy: %.3f" % accuracy_score(np.hstack(Y_test_flat), np.hstack(Y_pred2))) print(confusion_matrix(np.hstack(Y_test_flat), np.hstack(Y_pred2))) # plot stuff fig, axes = plt.subplots(2, 2) axes[0, 0].imshow(snakes['X_test'][0], interpolation='nearest') axes[0, 0].set_title('Input') y = Y_test[0].astype(np.int) bg = 2 * (y != 0) # enhance contrast axes[0, 1].matshow(y + bg, cmap=plt.cm.Greys) axes[0, 1].set_title("Ground Truth") axes[1, 0].matshow(Y_pred[0].reshape(y.shape) + bg, cmap=plt.cm.Greys) axes[1, 0].set_title("Prediction w/o edge features") axes[1, 1].matshow(Y_pred2[0].reshape(y.shape) + bg, cmap=plt.cm.Greys) axes[1, 1].set_title("Prediction with edge features") for a in axes.ravel(): a.set_xticks(()) a.set_yticks(()) plt.show() from IPython.core.debugger import Tracer Tracer()()
inference = ('ogm', {'alg': 'fm'}) else: inference = 'qpbo' # first, train on X with directions only: crf = EdgeFeatureGraphCRF(inference_method=inference) ssvm = OneSlackSSVM(crf, inference_cache=50, C=.1, tol=.1, max_iter=100, n_jobs=1) ssvm.fit(X_train_directions, Y_train_flat) # Evaluate using confusion matrix. # Clearly the middel of the snake is the hardest part. X_test, Y_test = snakes['X_test'], snakes['Y_test'] X_test = [one_hot_colors(x) for x in X_test] Y_test_flat = [y_.ravel() for y_ in Y_test] X_test_directions, X_test_edge_features = prepare_data(X_test) Y_pred = ssvm.predict(X_test_directions) print("Results using only directional features for edges") print("Test accuracy: %.3f" % accuracy_score(np.hstack(Y_test_flat), np.hstack(Y_pred))) print(confusion_matrix(np.hstack(Y_test_flat), np.hstack(Y_pred))) # now, use more informative edge features: crf = EdgeFeatureGraphCRF(inference_method=inference) ssvm = OneSlackSSVM(crf, inference_cache=50, C=.1, tol=.1, switch_to='ad3', n_jobs=1) ssvm.fit(X_train_edge_features, Y_train_flat) Y_pred2 = ssvm.predict(X_test_edge_features) print("Results using also input features for edges") print("Test accuracy: %.3f" % accuracy_score(np.hstack(Y_test_flat), np.hstack(Y_pred2))) print(confusion_matrix(np.hstack(Y_test_flat), np.hstack(Y_pred2)))
yeast = fetch_mldata("yeast") X = yeast.data X = np.hstack([X, np.ones((X.shape[0], 1))]) y = yeast.target.toarray().astype(np.int).T X_train, X_test = X[:1500], X[1500:] y_train, y_test = y[:1500], y[1500:] else: scene = load_scene() X_train, X_test = scene['X_train'], scene['X_test'] y_train, y_test = scene['y_train'], scene['y_test'] n_labels = y_train.shape[1] full = np.vstack([x for x in itertools.combinations(range(n_labels), 2)]) tree = chow_liu_tree(y_train) #tree_model = MultiLabelClf(edges=tree, inference_method=('ogm', {'alg': 'dyn'})) tree_model = MultiLabelClf(edges=tree, inference_method='max-product') tree_ssvm = OneSlackSSVM(tree_model, inference_cache=50, C=.1, tol=0.01) print("fitting tree model...") tree_ssvm.fit(X_train, y_train) print("Training loss tree model: %f" % hamming_loss(y_train, np.vstack(tree_ssvm.predict(X_train)))) print("Test loss tree model: %f" % hamming_loss(y_test, np.vstack(tree_ssvm.predict(X_test))))
for num in new_index_list: new_array = Xtest_scaler[num[0]:num[1], :] Xtest_scale.append(new_array) Xtest_scale = np.array(Xtest_scale) print Xtest_scale.shape print 'Test set scaled' ssvm.fit(Xtrain_scale, y_train) print 'Model fit' # Storing model weights for plot of transition probabilities if type(transition_states) == str: transition_states = ssvm.w[-49:].reshape(7, 7) else: transition_states = transition_states + ssvm.w[-49:].reshape(7, 7) predicted = ssvm.predict(Xtest_scale) print 'Predictions made' # Metrics fold_cm = confusion_matrix(np.hstack(y_test), np.hstack(predicted)) if type(total_confusion_matrix) == str: total_confusion_matrix = fold_cm else: total_confusion_matrix = total_confusion_matrix + fold_cm report = classification_report(np.hstack(y_test), np.hstack(predicted)) for line in report.split('\n'): print line print 'Metrics written' # Code for plotting matrix depicting transition probabilities # for the CRF model averaged across 10 folds of CV.
class SSVM: """Structured SVM wrapper""" def __init__(self, inference_train, inference_pred, dat_obj, C=1.0, share_params=True, multi_label=True, poi_info=None, debug=False): assert (C > 0) self.C = C self.inference_train = inference_train self.inference_pred = inference_pred self.share_params = share_params self.multi_label = multi_label self.dat_obj = dat_obj self.debug = debug self.trained = False if poi_info is None: self.poi_info = None else: self.poi_info = poi_info self.scaler_node = MinMaxScaler(feature_range=(-1, 1), copy=False) self.scaler_edge = MinMaxScaler(feature_range=(-1, 1), copy=False) def train(self, trajid_list, n_jobs=4): if self.poi_info is None: self.poi_info = self.dat_obj.calc_poi_info(trajid_list) # build POI_ID <--> POI__INDEX mapping for POIs used to train CRF # which means only POIs in traj such that len(traj) >= 2 are included poi_set = { p for tid in trajid_list for p in self.dat_obj.traj_dict[tid] if len(self.dat_obj.traj_dict[tid]) >= 2 } self.poi_list = sorted(poi_set) self.poi_id_dict, self.poi_id_rdict = dict(), dict() for idx, poi in enumerate(self.poi_list): self.poi_id_dict[poi] = idx self.poi_id_rdict[idx] = poi # generate training data train_traj_list = [ self.dat_obj.traj_dict[k] for k in trajid_list if len(self.dat_obj.traj_dict[k]) >= 2 ] node_features_list = Parallel(n_jobs=n_jobs)( delayed(calc_node_features)(tr[0], len(tr), self.poi_list, self.poi_info, self.dat_obj) for tr in train_traj_list) edge_features = calc_edge_features(trajid_list, self.poi_list, self.poi_info, self.dat_obj) # feature scaling: node features # should each example be flattened to one vector before scaling? self.fdim_node = node_features_list[0].shape X_node_all = np.vstack(node_features_list) X_node_all = self.scaler_node.fit_transform(X_node_all) X_node_all = X_node_all.reshape(-1, self.fdim_node[0], self.fdim_node[1]) # feature scaling: edge features fdim_edge = edge_features.shape edge_features = self.scaler_edge.fit_transform( edge_features.reshape(fdim_edge[0] * fdim_edge[1], -1)) self.edge_features = edge_features.reshape(fdim_edge) assert (len(train_traj_list) == X_node_all.shape[0]) X_train = [(X_node_all[k, :, :], self.edge_features.copy(), (self.poi_id_dict[train_traj_list[k][0]], len(train_traj_list[k]))) for k in range(len(train_traj_list))] y_train = [ np.array([self.poi_id_dict[k] for k in tr]) for tr in train_traj_list ] assert (len(X_train) == len(y_train)) # train sm = MyModel(inference_train=self.inference_train, inference_pred=self.inference_pred, share_params=self.share_params, multi_label=self.multi_label) if self.debug is True: print('C:', self.C) verbose = 1 if self.debug is True else 0 self.osssvm = OneSlackSSVM(model=sm, C=self.C, n_jobs=n_jobs, verbose=verbose) try: self.osssvm.fit(X_train, y_train, initialize=True) self.trained = True print('SSVM training finished.') # except ValueError: except: self.trained = False sys.stderr.write('SSVM training FAILED.\n') # raise return self.trained def predict(self, startPOI, nPOI): assert (self.trained is True) if startPOI not in self.poi_list: return None X_node_test = calc_node_features(startPOI, nPOI, self.poi_list, self.poi_info, self.dat_obj) # feature scaling # should each example be flattened to one vector before scaling? # X_node_test = X_node_test.reshape(1, -1) # flatten test example to a vector X_node_test = self.scaler_node.transform(X_node_test) # X_node_test = X_node_test.reshape(self.fdim) X_test = [(X_node_test, self.edge_features, (self.poi_id_dict[startPOI], nPOI))] y_hat_list = self.osssvm.predict(X_test)[0] # print(y_hat_list) return [ np.array([self.poi_id_rdict[x] for x in y_hat]) for y_hat in y_hat_list ]
crf = EdgeFeatureGraphCRF(inference_method=inference) ssvm = OneSlackSSVM(crf, inference_cache=50, C=.1, tol=.1, max_iter=100, n_jobs=1) ssvm.fit(X_train_directions, Y_train_flat) # Evaluate using confusion matrix. # Clearly the middel of the snake is the hardest part. X_test, Y_test = snakes['X_test'], snakes['Y_test'] X_test = [one_hot_colors(x) for x in X_test] Y_test_flat = [y_.ravel() for y_ in Y_test] X_test_directions, X_test_edge_features = prepare_data(X_test) Y_pred = ssvm.predict(X_test_directions) print("Results using only directional features for edges") print("Test accuracy: %.3f" % accuracy_score(np.hstack(Y_test_flat), np.hstack(Y_pred))) print(confusion_matrix(np.hstack(Y_test_flat), np.hstack(Y_pred))) # now, use more informative edge features: crf = EdgeFeatureGraphCRF(inference_method=inference) ssvm = OneSlackSSVM(crf, inference_cache=50, C=.1, tol=.1, switch_to='ad3', n_jobs=1) ssvm.fit(X_train_edge_features, Y_train_flat) Y_pred2 = ssvm.predict(X_test_edge_features)
#inference = 'qpbo' #I'm interested in AD3 inference. inference = 'ad3' # first, train on X with directions only: crf = EdgeFeatureGraphCRF(inference_method=inference) ssvm = OneSlackSSVM(crf, inference_cache=50, C=.1, tol=.1, max_iter=100, n_jobs=1) t0 = time.time() ssvm.fit(X_train_directions, Y_train_flat) print("Model EdgeFeatureGraphCRF fitted. %.1fs"%(time.time()-t0)) Y_GT = np.hstack(Y_test_flat) print("- Results using only directional features for edges. %.1fs"%(time.time()-t0)) t0 = time.time() Y_pred = ssvm.predict(X_test_directions) REPORT(Y_GT, Y_pred, time.time()-t0) print "- Result with binarized graph" t0 = time.time() Y_pred = ssvm.predict(X_test_directions, [True]*len(X_test_directions)) REPORT(Y_GT, Y_pred, time.time()-t0) #Predict under constraints def buildConstraints(X, bOne=True): """ We iterate over each graph, and make sure that for each, we constrain to have a single instances of classes 1 to 9 (or atmost one) The constraints must be a list of tuples like ( <operator>, <unaries>, <states>, <negated> )
print("val end export") Xtrain = np.loadtxt(vfnumpypath + "Xtrain.txt") Ytrain = np.loadtxt(vfnumpypath + "Ytrain.txt", dtype="int") print("train end export") #independent Model independent_model = MultiLabelClf(inference_method='unary') independent_ssvm = OneSlackSSVM(independent_model, C=.1, tol=0.01) print("fitting independent model...") independent_ssvm.fit(Xtrain, Ytrain) #print np.vstack(independent_ssvm.predict(Xval))[1,:] print("Test exact matching ratio: %f" % check_exactmatchratio( Yval, np.vstack(independent_ssvm.predict(Xval)), datatotest)) print( f1_score(Yval[3, :], np.vstack(independent_ssvm.predict(Xtrain))[3, :], average='macro')) ''' print("Training loss independent model: %f" % hamming_loss(Ytrain, np.vstack(independent_ssvm.predict(Xtrain)))) print("Test loss independent model: %f" % hamming_loss(Yval, np.vstack(independent_ssvm.predict(Xval)))) print Yval.shape print Yval[1,:] '''
print("Training SSVM") inference = 'qpbo' # first, train on X with directions only: crf = EdgeFeatureGraphCRF(inference_method=inference) ssvm = OneSlackSSVM(crf, inference_cache=50, C=1., tol=.1, max_iter=500, n_jobs=4) Y_flat = [y_.ravel() for y_ in Y] Y_flat = np.asarray( [Y_flat[i][j] for i in range(len(Y_flat)) for j in range(len(Y_flat[i]))]) ssvm.fit(X, Y) Z_bin = ssvm.predict(X) Z_flat = np.asarray( [Z_bin[i][j] for i in range(len(Z_bin)) for j in range(len(Z_bin[i]))]) f1_score = metrics.f1_score(Y_flat, Z_flat, average='weighted') conf_mat = metrics.confusion_matrix(Y_flat, Z_flat) TPR = conf_mat[0][0] / (conf_mat[0][0] + conf_mat[0][1]) FPR = conf_mat[1][0] / (conf_mat[1][0] + conf_mat[1][1]) print('Results with classifier: ' + ssvm.__class__.__name__) print('TPR/FPR = ' + str(TPR) + '/' + str(FPR)) print('F1-score = ' + str(f1_score)) my_classifier = ssvm # Run prediction on the img_idx-th image img_idx = 0 the_img = imgs[img_idx] the_gt = gt_imgs[img_idx]