def test_binary_blocks_cutting_plane():
#testing cutting plane ssvm on easy binary dataset
# generate graphs explicitly for each example
for inference_method in get_installed(["dai", "lp", "qpbo", "ad3", 'ogm']):
print("testing %s" % inference_method)
X, Y = generate_blocks(n_samples=3)
crf = GraphCRF(inference_method=inference_method)
clf = NSlackSSVM(model=crf,
max_iter=20,
C=100,
check_constraints=True,
break_on_bad=False,
n_jobs=1)
x1, x2, x3 = X
y1, y2, y3 = Y
n_states = len(np.unique(Y))
# delete some rows to make it more fun
x1, y1 = x1[:, :-1], y1[:, :-1]
x2, y2 = x2[:-1], y2[:-1]
# generate graphs
X_ = [x1, x2, x3]
G = [make_grid_edges(x) for x in X_]
# reshape / flatten x and y
X_ = [x.reshape(-1, n_states) for x in X_]
Y = [y.ravel() for y in [y1, y2, y3]]
X = zip(X_, G)
clf.fit(X, Y)
Y_pred = clf.predict(X)
for y, y_pred in zip(Y, Y_pred):
assert_array_equal(y, y_pred)
def test_binary_blocks_cutting_plane():
#testing cutting plane ssvm on easy binary dataset
# generate graphs explicitly for each example
for inference_method in get_installed(["lp", "qpbo", "ad3", 'ogm']):
X, Y = generate_blocks(n_samples=3)
crf = GraphCRF(inference_method=inference_method)
clf = NSlackSSVM(model=crf, max_iter=20, C=100, check_constraints=True,
break_on_bad=False, n_jobs=1)
x1, x2, x3 = X
y1, y2, y3 = Y
n_states = len(np.unique(Y))
# delete some rows to make it more fun
x1, y1 = x1[:, :-1], y1[:, :-1]
x2, y2 = x2[:-1], y2[:-1]
# generate graphs
X_ = [x1, x2, x3]
G = [make_grid_edges(x) for x in X_]
# reshape / flatten x and y
X_ = [x.reshape(-1, n_states) for x in X_]
Y = [y.ravel() for y in [y1, y2, y3]]
X = list(zip(X_, G))
clf.fit(X, Y)
Y_pred = clf.predict(X)
for y, y_pred in zip(Y, Y_pred):
assert_array_equal(y, y_pred)
def test_binary_blocks_batches_n_slack():
#testing cutting plane ssvm on easy binary dataset
X, Y = generate_blocks(n_samples=5)
crf = GridCRF(inference_method=inference_method)
clf = NSlackSSVM(model=crf, max_iter=20, batch_size=1, C=100)
clf.fit(X, Y)
Y_pred = clf.predict(X)
assert_array_equal(Y, Y_pred)
def test_multinomial_checker_cutting_plane():
X, Y = generate_checker_multinomial(n_samples=10, noise=.1)
n_labels = len(np.unique(Y))
crf = GridCRF(n_states=n_labels, inference_method=inference_method)
clf = NSlackSSVM(model=crf, max_iter=20, C=100000, check_constraints=True)
clf.fit(X, Y)
Y_pred = clf.predict(X)
assert_array_equal(Y, Y_pred)
def test_binary_blocks_batches_n_slack():
#testing cutting plane ssvm on easy binary dataset
X, Y = toy.generate_blocks(n_samples=5)
crf = GridCRF()
clf = NSlackSSVM(model=crf, max_iter=20, C=100, check_constraints=True,
break_on_bad=False, n_jobs=1, batch_size=1)
clf.fit(X, Y)
Y_pred = clf.predict(X)
assert_array_equal(Y, Y_pred)
def test_binary_blocks_cutting_plane():
#testing cutting plane ssvm on easy binary dataset
X, Y = generate_blocks(n_samples=5)
crf = GridCRF(inference_method=inference_method)
clf = NSlackSSVM(model=crf, max_iter=20, C=100,
check_constraints=True, break_on_bad=False)
clf.fit(X, Y)
Y_pred = clf.predict(X)
assert_array_equal(Y, Y_pred)
def test_multinomial_blocks_cutting_plane():
#testing cutting plane ssvm on easy multinomial dataset
X, Y = generate_blocks_multinomial(n_samples=40, noise=0.5, seed=0)
n_labels = len(np.unique(Y))
crf = GridCRF(n_states=n_labels, inference_method=inference_method)
clf = NSlackSSVM(model=crf, max_iter=100, C=100, check_constraints=False,
batch_size=1)
clf.fit(X, Y)
Y_pred = clf.predict(X)
assert_array_equal(Y, Y_pred)
def test_simple_1d_dataset_cutting_plane():
# 10 1d datapoints between 0 and 1
X = np.random.uniform(size=(30, 1))
Y = (X.ravel() > 0.5).astype(np.int)
# we have to add a constant 1 feature by hand :-/
X = np.hstack([X, np.ones((X.shape[0], 1))])
pbl = MultiClassClf(n_features=2)
svm = NSlackSSVM(pbl, check_constraints=True, C=10000)
svm.fit(X, Y)
assert_array_equal(Y, np.hstack(svm.predict(X)))
def test_simple_1d_dataset_cutting_plane():
# 10 1d datapoints between 0 and 1
X = np.random.uniform(size=(30, 1))
Y = (X.ravel() > 0.5).astype(np.int)
# we have to add a constant 1 feature by hand :-/
X = np.hstack([X, np.ones((X.shape[0], 1))])
pbl = MultiClassClf(n_features=2)
svm = NSlackSSVM(pbl, check_constraints=True, C=10000)
svm.fit(X, Y)
assert_array_equal(Y, np.hstack(svm.predict(X)))
def test_multinomial_blocks_directional():
# testing cutting plane ssvm with directional CRF on easy multinomial
# dataset
X, Y = toy.generate_blocks_multinomial(n_samples=10, noise=0.3, seed=0)
n_labels = len(np.unique(Y))
crf = DirectionalGridCRF(n_states=n_labels)
clf = NSlackSSVM(model=crf, max_iter=100, C=100, verbose=0,
check_constraints=True, batch_size=1)
clf.fit(X, Y)
Y_pred = clf.predict(X)
assert_array_equal(Y, Y_pred)
def test_binary_ssvm_attractive_potentials():
# test that submodular SSVM can learn the block dataset
X, Y = toy.generate_blocks(n_samples=10)
crf = GridCRF()
submodular_clf = NSlackSSVM(model=crf, max_iter=200, C=100,
check_constraints=True,
positive_constraint=[5])
submodular_clf.fit(X, Y)
Y_pred = submodular_clf.predict(X)
assert_array_equal(Y, Y_pred)
assert_true(submodular_clf.w[5] < 0) # don't ask me about signs
def test_binary_ssvm_attractive_potentials():
# test that submodular SSVM can learn the block dataset
X, Y = generate_blocks(n_samples=10)
crf = GridCRF(inference_method=inference_method)
submodular_clf = NSlackSSVM(model=crf, max_iter=200, C=100,
check_constraints=True,
negativity_constraint=[5])
submodular_clf.fit(X, Y)
Y_pred = submodular_clf.predict(X)
assert_array_equal(Y, Y_pred)
assert_true(submodular_clf.w[5] < 0)
def CRF_oneNode(x_train, x_test, y_train, y_test):
pbl = GraphCRF(n_states = 4,n_features=20)
svm = NSlackSSVM(pbl,max_iter=200, C=10,n_jobs=2)
svm.fit(x_train,y_train)
y_pred = svm.predict(x_test)
target_names = ['Start','Mid','End','Others']
#eclf = EnsembleClassifier(clfs=[pipe1, pipe2],voting='soft',weights=[0.5,0.2])
#eclf.fit(x_train,y_train)
#y_pred = eclf.predict(x_test)
print classification_report(y_test, y_pred, target_names=target_names)
def test_simple_1d_dataset_cutting_plane():
# 10 1d datapoints between 0 and 1
X = np.random.uniform(size=(30, 1))
# linearly separable labels
Y = 1 - 2 * (X.ravel() < .5)
# we have to add a constant 1 feature by hand :-/
X = np.hstack([X, np.ones((X.shape[0], 1))])
pbl = BinaryClf(n_features=2)
svm = NSlackSSVM(pbl, check_constraints=True, C=1000)
svm.fit(X, Y)
assert_array_equal(Y, np.hstack(svm.predict(X)))
def test_simple_1d_dataset_cutting_plane():
# 10 1d datapoints between 0 and 1
X = np.random.uniform(size=(30, 1))
# linearly separable labels
Y = 1 - 2 * (X.ravel() < .5)
# we have to add a constant 1 feature by hand :-/
X = np.hstack([X, np.ones((X.shape[0], 1))])
pbl = BinaryClf(n_features=2)
svm = NSlackSSVM(pbl, check_constraints=True, C=1000)
svm.fit(X, Y)
assert_array_equal(Y, np.hstack(svm.predict(X)))
def test_multinomial_blocks_directional():
# testing cutting plane ssvm with directional CRF on easy multinomial
# dataset
X, Y = generate_blocks_multinomial(n_samples=10, noise=0.3, seed=0)
n_labels = len(np.unique(Y))
crf = DirectionalGridCRF(n_states=n_labels,
inference_method=inference_method)
clf = NSlackSSVM(model=crf, max_iter=100, C=100, verbose=0,
check_constraints=True, batch_size=1)
clf.fit(X, Y)
Y_pred = clf.predict(X)
assert_array_equal(Y, Y_pred)
def test_binary_ssvm_repellent_potentials():
# test non-submodular problem with and without submodularity constraint
# dataset is checkerboard
X, Y = generate_checker()
crf = GridCRF(inference_method=inference_method)
clf = NSlackSSVM(model=crf, max_iter=10, C=100,
check_constraints=True)
clf.fit(X, Y)
Y_pred = clf.predict(X)
# standard crf can predict perfectly
assert_array_equal(Y, Y_pred)
submodular_clf = NSlackSSVM(model=crf, max_iter=10, C=100,
check_constraints=True,
negativity_constraint=[4, 5, 6])
submodular_clf.fit(X, Y)
Y_pred = submodular_clf.predict(X)
# submodular crf can not do better than unaries
for i, x in enumerate(X):
y_pred_unaries = crf.inference(x, np.array([1, 0, 0, 1, 0, 0, 0]))
assert_array_equal(y_pred_unaries, Y_pred[i])
def test_multinomial_blocks_cutting_plane():
#testing cutting plane ssvm on easy multinomial dataset
X, Y = toy.generate_blocks_multinomial(n_samples=10, noise=0.3,
seed=0)
n_labels = len(np.unique(Y))
for inference_method in get_installed(['lp', 'qpbo', 'ad3']):
crf = GridCRF(n_states=n_labels, inference_method=inference_method)
clf = NSlackSSVM(model=crf, max_iter=10, C=100,
check_constraints=False)
clf.fit(X, Y)
Y_pred = clf.predict(X)
assert_array_equal(Y, Y_pred)
def test_blobs_2d_cutting_plane():
# make two gaussian blobs
X, Y = make_blobs(n_samples=80, centers=2, random_state=1)
Y = 2 * Y - 1
# we have to add a constant 1 feature by hand :-/
X = np.hstack([X, np.ones((X.shape[0], 1))])
X_train, X_test, Y_train, Y_test = X[:40], X[40:], Y[:40], Y[40:]
pbl = BinaryClf(n_features=3)
svm = NSlackSSVM(pbl, check_constraints=True, C=1000)
svm.fit(X_train, Y_train)
assert_array_equal(Y_test, np.hstack(svm.predict(X_test)))
def test_blobs_2d_cutting_plane():
# make two gaussian blobs
X, Y = make_blobs(n_samples=80, centers=3, random_state=42)
# we have to add a constant 1 feature by hand :-/
X = np.hstack([X, np.ones((X.shape[0], 1))])
X_train, X_test, Y_train, Y_test = X[:40], X[40:], Y[:40], Y[40:]
pbl = MultiClassClf(n_features=3, n_classes=3)
svm = NSlackSSVM(pbl, check_constraints=True, C=1000, batch_size=1)
svm.fit(X_train, Y_train)
assert_array_equal(Y_test, np.hstack(svm.predict(X_test)))
def test_multinomial_blocks_directional_simple():
# testing cutting plane ssvm with directional CRF on easy multinomial
# dataset
X_, Y_ = generate_blocks_multinomial(n_samples=10, noise=0.3, seed=0)
G = [make_grid_edges(x, return_lists=True) for x in X_]
edge_features = [edge_list_to_features(edge_list) for edge_list in G]
edges = [np.vstack(g) for g in G]
X = zip([x.reshape(-1, 3) for x in X_], edges, edge_features)
Y = [y.ravel() for y in Y_]
crf = EdgeFeatureGraphCRF(n_states=3, n_edge_features=2)
clf = NSlackSSVM(model=crf, max_iter=10, C=1, check_constraints=False)
clf.fit(X, Y)
Y_pred = clf.predict(X)
assert_array_equal(Y, Y_pred)
def test_multinomial_blocks_directional_simple():
# testing cutting plane ssvm with directional CRF on easy multinomial
# dataset
X_, Y_ = generate_blocks_multinomial(n_samples=10, noise=0.3, seed=0)
G = [make_grid_edges(x, return_lists=True) for x in X_]
edge_features = [edge_list_to_features(edge_list) for edge_list in G]
edges = [np.vstack(g) for g in G]
X = list(zip([x.reshape(-1, 3) for x in X_], edges, edge_features))
Y = [y.ravel() for y in Y_]
crf = EdgeFeatureGraphCRF(n_states=3, n_edge_features=2)
clf = NSlackSSVM(model=crf, max_iter=10, C=1, check_constraints=False)
clf.fit(X, Y)
Y_pred = clf.predict(X)
assert_array_equal(Y, Y_pred)
def test_binary_blocks_cutting_plane_latent_node():
#testing cutting plane ssvm on easy binary dataset
# we use the LatentNodeCRF without latent nodes and check that it does the
# same as GraphCRF
X, Y = generate_blocks(n_samples=3)
crf = GraphCRF()
clf = NSlackSSVM(model=crf,
max_iter=20,
C=100,
check_constraints=True,
break_on_bad=False,
n_jobs=1)
x1, x2, x3 = X
y1, y2, y3 = Y
n_states = len(np.unique(Y))
# delete some rows to make it more fun
x1, y1 = x1[:, :-1], y1[:, :-1]
x2, y2 = x2[:-1], y2[:-1]
# generate graphs
X_ = [x1, x2, x3]
G = [make_grid_edges(x) for x in X_]
# reshape / flatten x and y
X_ = [x.reshape(-1, n_states) for x in X_]
Y = [y.ravel() for y in [y1, y2, y3]]
X = zip(X_, G)
clf.fit(X, Y)
Y_pred = clf.predict(X)
for y, y_pred in zip(Y, Y_pred):
assert_array_equal(y, y_pred)
latent_crf = LatentNodeCRF(n_labels=2, n_hidden_states=0)
latent_svm = LatentSSVM(NSlackSSVM(model=latent_crf,
max_iter=20,
C=100,
check_constraints=True,
break_on_bad=False,
n_jobs=1),
latent_iter=3)
X_latent = zip(X_, G, np.zeros(len(X_)))
latent_svm.fit(X_latent, Y, H_init=Y)
Y_pred = latent_svm.predict(X_latent)
for y, y_pred in zip(Y, Y_pred):
assert_array_equal(y, y_pred)
assert_array_almost_equal(latent_svm.w, clf.w)
def runIt(train_list):
X_org = list2features(train_list)
X = np.array(X_org)
y = list2labels_sleep(train_list)
y_org = np.array(y)
Y = y_org.reshape(-1, 1)
X_ = [(np.atleast_2d(x), np.empty((0, 2), dtype=np.int)) for x in X]
X_train, X_test, y_train, y_test = train_test_split(X_, Y, test_size=.5)
pbl = GraphCRF(inference_method='unary')
svm = NSlackSSVM(pbl, C=100)
start = time()
svm.fit(X_train, y_train)
time_svm = time() - start
y_pred = np.vstack(svm.predict(X_test))
print("Score with pystruct crf svm: %f (took %f seconds)"
% (np.mean(y_pred == y_test), time_svm))
def test_multinomial_blocks_directional_anti_symmetric():
# testing cutting plane ssvm with directional CRF on easy multinomial
# dataset
X_, Y_ = generate_blocks_multinomial(n_samples=10, noise=0.3, seed=0)
G = [make_grid_edges(x, return_lists=True) for x in X_]
edge_features = [edge_list_to_features(edge_list) for edge_list in G]
edges = [np.vstack(g) for g in G]
X = list(zip([x.reshape(-1, 3) for x in X_], edges, edge_features))
Y = [y.ravel() for y in Y_]
crf = EdgeFeatureGraphCRF(symmetric_edge_features=[0], antisymmetric_edge_features=[1])
clf = NSlackSSVM(model=crf, C=100)
clf.fit(X, Y)
Y_pred = clf.predict(X)
assert_array_equal(Y, Y_pred)
pairwise_params = clf.w[-9 * 2 :].reshape(2, 3, 3)
sym = pairwise_params[0]
antisym = pairwise_params[1]
assert_array_equal(sym, sym.T)
assert_array_equal(antisym, -antisym.T)
def test_binary_blocks_cutting_plane_latent_node():
#testing cutting plane ssvm on easy binary dataset
# we use the LatentNodeCRF without latent nodes and check that it does the
# same as GraphCRF
X, Y = generate_blocks(n_samples=3)
crf = GraphCRF()
clf = NSlackSSVM(model=crf, max_iter=20, C=100, check_constraints=True,
break_on_bad=False, n_jobs=1)
x1, x2, x3 = X
y1, y2, y3 = Y
n_states = len(np.unique(Y))
# delete some rows to make it more fun
x1, y1 = x1[:, :-1], y1[:, :-1]
x2, y2 = x2[:-1], y2[:-1]
# generate graphs
X_ = [x1, x2, x3]
G = [make_grid_edges(x) for x in X_]
# reshape / flatten x and y
X_ = [x.reshape(-1, n_states) for x in X_]
Y = [y.ravel() for y in [y1, y2, y3]]
X = zip(X_, G)
clf.fit(X, Y)
Y_pred = clf.predict(X)
for y, y_pred in zip(Y, Y_pred):
assert_array_equal(y, y_pred)
latent_crf = LatentNodeCRF(n_labels=2, n_hidden_states=0)
latent_svm = LatentSSVM(NSlackSSVM(model=latent_crf, max_iter=20, C=100,
check_constraints=True,
break_on_bad=False, n_jobs=1),
latent_iter=3)
X_latent = zip(X_, G, np.zeros(len(X_)))
latent_svm.fit(X_latent, Y, H_init=Y)
Y_pred = latent_svm.predict(X_latent)
for y, y_pred in zip(Y, Y_pred):
assert_array_equal(y, y_pred)
assert_array_almost_equal(latent_svm.w, clf.w)
def test_multinomial_blocks_directional_anti_symmetric():
# testing cutting plane ssvm with directional CRF on easy multinomial
# dataset
X_, Y_ = generate_blocks_multinomial(n_samples=10, noise=0.3, seed=0)
G = [make_grid_edges(x, return_lists=True) for x in X_]
edge_features = [edge_list_to_features(edge_list) for edge_list in G]
edges = [np.vstack(g) for g in G]
X = zip([x.reshape(-1, 3) for x in X_], edges, edge_features)
Y = [y.ravel() for y in Y_]
crf = EdgeFeatureGraphCRF(symmetric_edge_features=[0],
antisymmetric_edge_features=[1])
clf = NSlackSSVM(model=crf, C=100)
clf.fit(X, Y)
Y_pred = clf.predict(X)
assert_array_equal(Y, Y_pred)
pairwise_params = clf.w[-9 * 2:].reshape(2, 3, 3)
sym = pairwise_params[0]
antisym = pairwise_params[1]
assert_array_equal(sym, sym.T)
assert_array_equal(antisym, -antisym.T)
# --- Train model --- #
model = EdgeFeatureGraphCRF(n_states, n_features, n_edge_features)
ssvm = NSlackSSVM(model=model,
C=0.1,
tol=0.001,
verbose=0,
show_loss_every=10)
# ssvm = OneSlackSSVM(model=model, C=.1, inference_cache=50, tol=0.1, verbose=0,show_loss_every=10)
ssvm.fit(X_train, Y_train)
# --- Test with pystruct --- #
# print("Test score with graph CRF: %f" % ssvm.score(X_test, Y_test))
# --- Test manually - get contingency tables --- #
prediction = ssvm.predict(X_test)
contingency = np.array([0, 0, 0, 0])
for i in xrange(len(test)):
pred = prediction[i]
true = Y_test[i]
contingency = contingency + get_contingency(pred, true)
TP, FP, TN, FN = contingency[0], contingency[1], contingency[
2], contingency[3]
# sens = float(TP)/(TP+FN)
# sens_all.append(sens)
# spec = float(TN)/(FP+TN)
# spec_all.append(spec)
acc = float(TP + TN) / (TP + FP + TN + FN)
X, Y = make_simple_2x2(seed=1)
# flatten X and Y
X_flat = [x.reshape(-1, 1).astype(np.float) for x in X]
Y_flat = [y.ravel() for y in Y]
# first, use standard graph CRF. Can't do much, high loss.
crf = GraphCRF()
svm = NSlackSSVM(model=crf, max_iter=200, C=1, n_jobs=1)
G = [make_grid_edges(x) for x in X]
X_grid_edges = list(zip(X_flat, G))
svm.fit(X_grid_edges, Y_flat)
plot_boxes(svm.predict(X_grid_edges), title="Non-latent SSVM predictions")
print("Training score binary grid CRF: %f" % svm.score(X_grid_edges, Y_flat))
# using one latent variable for each 2x2 rectangle
latent_crf = LatentNodeCRF(n_labels=2,
n_features=1,
n_hidden_states=2,
inference_method='lp')
ssvm = OneSlackSSVM(model=latent_crf,
max_iter=200,
C=100,
n_jobs=-1,
show_loss_every=10,
inference_cache=50)
latent_svm = LatentSSVM(ssvm)
X = zip([x.reshape(-1, 3) for x in X_], edges, edge_features) Y = [y.ravel() for y in Y_] Xtrain = np.array(X[0:int(np.floor(N/2))]) Ytrain = np.array(Y[0:int(np.floor(N/2))]) Xtest = np.array(X[int(np.floor(N/2)):]) Ytest = np.array(Y[int(np.floor(N/2)):]) crf = EdgeFeatureGraphCRF(n_states=3, n_edge_features=2) ## SSVM ssvm_logger = './logExampleSSVM.pickle' ssvm = NSlackSSVM(model=crf, max_iter=10, C=1, check_constraints=False, logger=SaveLogger(ssvm_logger,save_every=100) ) ssvm.fit(Xtrain, Ytrain) Ypred_ssvm = ssvm.predict(Xtest) ## CSSVM M = 5 # number of models in ensemble gamma = 0.001 # diversity weight C = 1 # regularizer weight cssvm = SubgradientMBestSSVM(crf, verbose=1, C=C, max_iter=100, n_jobs=1, M=M, gamma=gamma, initialize_w_by=ssvm_logger, momentum=0.0, learning_rate='auto', break_on_no_constraints=True, batch_size=None, decay_t0=1, decay_exponent=0.5, averaging=None, shuffle=False, sample_assignment_strategy='all', online=False, break_on_no_loss_improvement=2,parallel_inference=True, zero_constraint=None, force_moment=1) cssvm.fit(Xtrain, Ytrain) Ypreds_cssvm = cssvm.predict(Xtest)
def crf_postprocess(X_train, y_train, X_test, train_examples=2000):
clf = NSlackSSVM(MultiLabelClf(), verbose=1, n_jobs=-1, show_loss_every=1)
clf.fit(X_train, y_train)
pred = clf.predict(X_test)
pred = np.array(pred)
return pred
X, Y = make_simple_2x2(seed=1)
# flatten X and Y
X_flat = [x.reshape(-1, 1).astype(np.float) for x in X]
Y_flat = [y.ravel() for y in Y]
# first, use standard graph CRF. Can't do much, high loss.
crf = GraphCRF()
svm = NSlackSSVM(model=crf, max_iter=200, C=1, n_jobs=1)
G = [make_grid_edges(x) for x in X]
X_grid_edges = list(zip(X_flat, G))
svm.fit(X_grid_edges, Y_flat)
plot_boxes(svm.predict(X_grid_edges), title="Non-latent SSVM predictions")
print("Training score binary grid CRF: %f" % svm.score(X_grid_edges, Y_flat))
# using one latent variable for each 2x2 rectangle
latent_crf = LatentNodeCRF(n_labels=2, n_features=1, n_hidden_states=2,
inference_method='lp')
ssvm = OneSlackSSVM(model=latent_crf, max_iter=200, C=100,
n_jobs=-1, show_loss_every=10, inference_cache=50)
latent_svm = LatentSSVM(ssvm)
# make edges for hidden states:
edges = []
node_indices = np.arange(4 * 4).reshape(4, 4)
for i, (x, y) in enumerate(itertools.product([0, 2], repeat=2)):
for j in range(x, x + 2):
def test_final_segmentations(out_dir):
test_size = .75
#ssvm_c = [5, .3, .1, .01, .001]
ssvm_c = [50]
ssvm_tol = 0.001
ssvm_iter = 50
which_ssvm = 'nslack'
which_solver = ('ogm', {'alg': 'gc'})
which_combo = which_ssvm + '_ogm-' + which_solver[1]['alg']
out_dir = join_path(out_dir, 'test_final_segmentations', which_combo)
create_out_path(out_dir, except_on_exist=False)
cd = JaneliaData(dummy_data=False, only_nice_volumes=True)
subs = cd.subvolumes
demo_volumes = ['an197522_2013_03_10_13002', 'an197522_2013_03_08_06003', 'an229719_2013_12_05_03004', 'an229719_2013_12_05_07003', 'an229719_2013_12_05_08004']
img_x, img_y = cd.subvolumes[0].dims
edges = create_2d_edge_graph(img_x, img_y)
feats_unaries = ['feats_pm_++', 'feats_pm_+++', 'feats_pm_+++x']
feats_pairwise = 'feats_xcorr_green'
train_labelings = ['gt_active_++', 'gt_active_+++', 'gt_active_+++x']
test_labelings = ['gt_active_++', 'gt_active_+++', 'gt_active_+++x']
failed_combos = []
sp = ShuffleSplit(len(subs), random_state=42, test_size=test_size)
train_idxs, test_idxs = next(iter(sp))
for feats_unary in feats_unaries:
X = ssvm_construct_X(subs, feats_unary, feats_pairwise, edges)
for train_labeling in train_labelings:
Y_full_train = ssvm_construct_Y(train_labeling, subs)
class_weights = compute_class_weights(Y_full_train)
crf_graph = EdgeFeatureGraphCRF(inference_method=which_solver, class_weight=class_weights)
for test_labeling in test_labelings:
X_train = [X[i] for i in train_idxs]
Y_train = [Y_full_train[i] for i in train_idxs]
Y_full_test = ssvm_construct_Y(test_labeling, subs)
X_test = [X[i] for i in test_idxs]
Y_test = [Y_full_test[i] for i in test_idxs]
for c in ssvm_c:
try:
test_name = '%s_u=%s_p=%s_train-gt=%s_test-gt=%s_C=%.1e' % (which_ssvm, feats_unary, feats_pairwise, train_labeling, test_labeling, c)
test_out_dir = join_path(out_dir, test_name)
create_out_path(test_out_dir, except_on_exist=True)
print '\n=================================================================='
print '-->' + test_name
except IOError as e:
print e
continue
try:
logger = pystruct_logger(join_path(test_out_dir, 'ssvm_model.logger'), save_every=10)
if which_ssvm == 'nslack':
print "* creating NSlackSSVM"
model = NSlackSSVM(crf_graph, verbose=1, C=c, max_iter=ssvm_iter, n_jobs=-1, tol=ssvm_tol, show_loss_every=1,
inactive_threshold=1e-3, inactive_window=10, batch_size=1000, logger=logger)
ssvm_name = 'NSlackSSVM'
if which_ssvm == 'oneslack':
print "* creating OneSlackSSVM"
model = OneSlackSSVM(crf_graph, verbose=1, C=c, max_iter=ssvm_iter, n_jobs=-1, tol=ssvm_tol,
show_loss_every=1, inactive_threshold=1e-3, inactive_window=10, logger=logger)
ssvm_name = 'OneSlackSSVM'
if which_ssvm == 'frankwolfe':
print "* creating FrankWolfeSSVM"
model = FrankWolfeSSVM(crf_graph, verbose=1, C=c, max_iter=ssvm_iter, tol=ssvm_tol, line_search=True,
check_dual_every=10, do_averaging=True, sample_method='perm', random_state=None, logger=logger)
ssvm_name = 'FrankWolfeSSVM'
print '* fitting model...'
model.fit(X_train, Y_train)
print '--> model trained'
print '* scoring model...'
score = model.score(X_test, Y_test)
print score
print '* saving results for test run'
fig, _ = ssvm_plot_learning(model, title=test_name, time=False)
fig.savefig(join_path(test_out_dir, 'ssvm_plot_learning.png'), dpi=300)
plt.close('all')
print '--> saved learning plots'
### save results to JSON ###
with open(join_path(test_out_dir, 'ssvm_stats.json'), 'w') as json_out:
stats = {
'test_accuracy' : float(score),
'test_size' : float(test_size),
'ssvm_c' : float(c),
'ssvm_iter' : ssvm_iter,
'ssvm_tol' : float(ssvm_tol),
'feats_unary' : feats_unary,
'feats_pairwise' : feats_pairwise,
'which_ssvm' : which_ssvm,
'which_solver' : which_solver[0],
'which_alg' : which_solver[1]['alg'],
'train_labeling' : train_labeling,
'test_labeling' : test_labeling
}
json_out.write(json.dumps(stats, indent=0, sort_keys=True))
print '--> saved SSVM stats to JSON'
except:
print 'Combo %s failed, removing its out folder.' % test_out_dir
failed_combos.append(test_out_dir)
os.rmdir(test_out_dir)
create_out_path('FAILED_' + test_out_dir)
continue
print '--> saving vizzz for demo volumes and computing scores...'
demo_out_dir = join_path(test_out_dir, 'demo_vols')
create_out_path(demo_out_dir, except_on_exist=True)
demo_scores = {}
for demo_sub in demo_volumes:
demo_sub = cd.get_subvolume_by_name(demo_sub)
print ' -', demo_sub.name
unary_feats = demo_sub.get(feats_unary)
pairwise_feats = demo_sub.get(feats_pairwise)
x = (unary_feats, edges, pairwise_feats)
pred = model.predict([x])
pred = pred[0].reshape(img_x, img_y)
gt_train = demo_sub.get(train_labeling).astype('uint8')
gt_test = demo_sub.get(test_labeling).astype('uint8')
rgb_vis_train = plot_it_like_ferran(gt_train, pred)
rgb_vis_test = plot_it_like_ferran(gt_test, pred)
plt.imsave(join_path(demo_out_dir, '%s_ssvm_prediction.png' % demo_sub.name), pred)
plt.imsave(join_path(demo_out_dir, '%s_%s.png' % (demo_sub.name, train_labeling)), gt_train)
plt.imsave(join_path(demo_out_dir, '%s_%s.png' % (demo_sub.name, test_labeling)), gt_test)
plt.imsave(join_path(demo_out_dir, '%s_rgb_vis_train.png' % demo_sub.name), rgb_vis_train)
plt.imsave(join_path(demo_out_dir, '%s_rgb_vis_test.png' % demo_sub.name), rgb_vis_test)
labeled_pm_ax = demo_sub.plot_activity_labels(image=pred, cmap='gray')
plt.savefig(join_path(demo_out_dir, '%s_labeled_pred.png' % demo_sub.name), dpi=300)
plt.close('all')
score_train = model.score([x], [gt_train.reshape(img_x * img_y)])
score_test = model.score([x], [gt_test.reshape(img_x * img_y)])
scores = {
'accuracy_train' : float(score_train),
'accuracy_test' : float(score_test)
}
demo_scores[demo_sub.name] = scores
with open(join_path(test_out_dir, 'demo_vol_scores.json'), 'w') as json_out:
json_out.write(json.dumps(demo_scores, indent=0, sort_keys=True))
print '--> saved demo volume scores to JSON'
print '\nDone.'
print 'There were issues, the following combos got skipped:'
for combo in failed_combos:
print combo
X, Y = make_simple_2x2(seed=1)
# flatten X and Y
X_flat = [x.reshape(-1, 1).astype(np.float) for x in X]
Y_flat = [y.ravel() for y in Y]
# first, use standard graph CRF. Can't do much, high loss.
crf = GraphCRF()
svm = NSlackSSVM(model=crf, max_iter=200, C=1, n_jobs=1)
G = [make_grid_edges(x) for x in X]
asdf = zip(X_flat, G)
svm.fit(asdf, Y_flat)
plot_boxes(svm.predict(asdf), title="Non-latent SSVM predictions")
print("Training score binary grid CRF: %f" % svm.score(asdf, Y_flat))
# using one latent variable for each 2x2 rectangle
latent_crf = LatentNodeCRF(n_labels=2, n_features=1, n_hidden_states=2,
inference_method='lp')
ssvm = OneSlackSSVM(model=latent_crf, max_iter=200, C=100, verbose=1,
n_jobs=-1, show_loss_every=10, inference_cache=50)
latent_svm = LatentSSVM(ssvm)
# make edges for hidden states:
edges = []
node_indices = np.arange(4 * 4).reshape(4, 4)
for i, (x, y) in enumerate(itertools.product([0, 2], repeat=2)):
for j in xrange(x, x + 2):
X, Y = make_simple_2x2(seed=1)
# flatten X and Y
X_flat = [x.reshape(-1, 1).astype(np.float) for x in X]
Y_flat = [y.ravel() for y in Y]
# first, use standard graph CRF. Can't do much, high loss.
crf = GraphCRF()
svm = NSlackSSVM(model=crf, max_iter=200, C=1, n_jobs=1)
G = [make_grid_edges(x) for x in X]
asdf = zip(X_flat, G)
svm.fit(asdf, Y_flat)
plot_boxes(svm.predict(asdf), title="Non-latent SSVM predictions")
print("Training score binary grid CRF: %f" % svm.score(asdf, Y_flat))
# using one latent variable for each 2x2 rectangle
latent_crf = LatentNodeCRF(n_labels=2,
n_features=1,
n_hidden_states=2,
inference_method='lp')
ssvm = OneSlackSSVM(model=latent_crf,
max_iter=200,
C=100,
verbose=1,
n_jobs=-1,
show_loss_every=10,
inference_cache=50)
asdf2 = zip(X_flat2,G2)
#%%
folders = os.listdir(folderName)
num_models = len(folders)
print "Number of models :",num_models
modelTestErrors=[]
for i in range(num_models):
errTest=0
iteration = int(re.findall(r'\d+',folders[i])[0])
with open(folderName+folders[i], "rb") as f:
svm = pickle.load(f)
predTest = svm.predict(asdf2)
for j in range(len(predTest)):
errTest += accuracy(predTest[j],Y_flat2[j])
errTest = errTest/float(len(predTest))
modelTestErrors.append((iteration,errTest))
modelTestErrors.sort(key=lambda x:x[0])
for i in range(len(modelTestErrors)):
print "Iteration : %d , Dice Score : %f"%(modelTestErrors[i][0],modelTestErrors[i][1])
#%%
plt.plot([x[0] for x in modelTestErrors], [x[1] for x in modelTestErrors])
asdf2 = zip(X_flat2, G2)
#%%
folders = os.listdir(folderName)
num_models = len(folders)
print "Number of models :", num_models
modelTestErrors = []
for i in range(num_models):
errTest = 0
iteration = int(re.findall(r'\d+', folders[i])[0])
with open(folderName + folders[i], "rb") as f:
svm = pickle.load(f)
predTest = svm.predict(asdf2)
for j in range(len(predTest)):
errTest += accuracy(predTest[j], Y_flat2[j])
errTest = errTest / float(len(predTest))
modelTestErrors.append((iteration, errTest))
modelTestErrors.sort(key=lambda x: x[0])
for i in range(len(modelTestErrors)):
print "Iteration : %d , Dice Score : %f" % (modelTestErrors[i][0],
modelTestErrors[i][1])
#%%
edgeFeatures2 = []
for i in range(len(X_flat2)):
feature = []
for j in range(len(edgeList)):
feature.append(
np.append(X_flat2[i][edgeList[j][0]], X_flat2[i][edgeList[j][1]]))
edgeFeatures2.append(feature)
edgeFeatures2 = np.array(edgeFeatures2)
#asdf2 = zip(X_flat2,G2,edgeFeatures2)
asdf2 = zip(X_flat2, G2)
#%%
predTrain = svm.predict(asdf)
errTrain = 0
for i in range(len(predTrain)):
errTrain += accuracy(predTrain[i], Y_flat[i])
errTrain = errTrain / float(len(predTrain))
#%%
predTest = svm.predict(asdf2)
errTest = 0
for i in range(len(predTest)):
errTest += accuracy(predTest[i], Y_flat2[i])
errTest = errTest / float(len(predTest))
print "The train set DICE is %f" % (errTrain)
print "The test set DICE is %f" % (errTest)
def test_multivariate_ssvm():
def loss_fn(tn, fp, fn, tp):
return tp / (tp + .5 * (fp + fn))
#def loss_fn(tn, fp, fn, tp):
# return (fp + fn * 1.) / (tn + fp + fn + tp)
n_classes = 2
n_features = 3
n_examples = 600
df, feature_cols, label_col = get_center_dataset(n_classes=n_classes,
n_features=n_features,
n_examples=n_examples)
# Break into training / test
df_train, df_test = train_test_split(df, test_size=0.15, random_state=22)
print('\nSetup')
print("*" * 15)
print('loss=f1_score')
print('n_train_examples = %s' % df_train.shape[0])
print('n_test_examples = %s' % df_test.shape[0])
print('n_classes = %s' % n_classes)
print('n_features = %s' % n_features)
# Select out right columns
X_train, X_test = df_train[feature_cols].values, df_test[
feature_cols].values
y_train, y_test = df_train[label_col].values, df_test[label_col].values
# Let's normalize our data
mean, std = X_train.mean(axis=0), X_train.std(axis=0)
X_train = (X_train - mean) / std
X_test = (X_test - mean) / std
# Handy for later
n_train_examples = X_train.shape[0]
n_test_examples = X_test.shape[0]
print('\nRandom Algorithm')
print("*" * 15)
n = len(y_test.ravel())
losses = []
for _ in range(1000):
y_pred = np.random.choice([-1, 1], n)
tn, fp, fn, tp = confusion_matrix(y_test.ravel(), y_pred).ravel()
losses.append(loss_fn(tn, fp, fn, tp))
lower = np.percentile(losses, 5)
mid = np.percentile(losses, 50)
upper = np.percentile(losses, 95)
print('test_loss_mid=%s' % lower)
print('test_loss_lower=%s' % mid)
print('test_loss_upper=%s' % upper)
# Pystruct SSVM
print('\n Pystruct SSM Algorithm')
print('*' * 15)
model = PystructSSVM(X_train, y_train, X_test, y_test, loss_fn)
clf = NSlackSSVM(model, C=1.0, batch_size=-1)
clf.fit(X_train, y_train)
clf.predict(X_test)
tn, fp, fn, tp = confusion_matrix(y_test.ravel(), y_pred).ravel()
print('test_loss={}' % loss_fn(tn, fp, fn, tp))
# Simple SVM
print('\nSVM Algorithm')
print("*" * 15)
from sklearn import svm
C = 1.
clf = svm.SVC(C=C)
print('Fitting SVM. C={}' % C)
clf.fit(X_train, y_train)
y_pred = clf.predict(X_test)
tn, fp, fn, tp = confusion_matrix(y_test.ravel(), y_pred).ravel()
print('test_loss={}' % loss_fn(tn, fp, fn, tp))
"""
X_train, X_test = features[fidx == 0], features[fidx == 1]
y_train, y_test = labels[fidx == 0], labels[fidx == 1]
model = ChainCRF()
#model = BinaryClf(X_train.shape[0])
model = MultiLabelClf()
ssvm = NSlackSSVM(model=model, max_iter=500)
X_train_l = [row for row in X_train]
y_train_l = [row for row in y_train]
X_test_l = [row for row in X_test]
ssvm.fit(X_train, y_train)
y_pred_l = ssvm.predict(X_test_l)
y_pred = np.array(y_pred_l)
#for pred in y_pred:
# print(pred)
#print(y_pred)
assert y_pred.shape == y_test.shape
for i in range(y_pred.shape[1]):
print("bACC [{0:d}] = {1:0.3f}".format(i, balanced_accuracy_score(y_test[:, i], y_pred[:, i])))
ssvm.fit(X_train, Y_train)
train_time = time.time() - start
print("Training completed and testing started")
print("Time used for training: {}".format(train_time))
pickle.dump(
ssvm,
open(
os.path.join(param_fdp_results,
"results_{}_{}_model.p".format(k, pattern)),
"wb"))
start = time.time()
Y_pred = ssvm.predict(X_test)
test_time = time.time() - start
acc = accuracy_score(np.hstack(Y_test), np.hstack(Y_pred))
cm = confusion_matrix(np.hstack(Y_test), np.hstack(Y_pred))
pickle.dump(
cm,
open(
os.path.join(param_fdp_results,
"results_{}_{}_cm.p".format(k, pattern)), "wb"))
print("Test completed with an accuracy: %.3f" % acc)
print(cm)
print("Time used for testing: {}".format(test_time))
results.append([
Y_flat2 = np.array(testLabels[0:n_test])
edgeFeatures2=[]
for i in range(len(X_flat2)):
feature=[]
for j in range(len(edgeList)):
feature.append( np.append(X_flat2[i][edgeList[j][0]] , X_flat2[i][edgeList[j][1]]) )
edgeFeatures2.append(feature)
edgeFeatures2=np.array(edgeFeatures2)
#asdf2 = zip(X_flat2,G2,edgeFeatures2)
asdf2 = zip(X_flat2,G2)
#%%
predTrain = svm.predict(asdf)
errTrain = 0
for i in range(len(predTrain)):
errTrain += accuracy(predTrain[i],Y_flat[i])
errTrain = errTrain/float(len(predTrain))
#%%
predTest = svm.predict(asdf2)
errTest = 0
for i in range(len(predTest)):
errTest += accuracy(predTest[i],Y_flat2[i])
errTest = errTest/float(len(predTest))
print "The train set DICE is %f" %(errTrain)
print "The test set DICE is %f" %(errTest)
Y_test = [labels[j] for j in test]
# if verbose:
# print np.mean(map(np.mean,Y_train)), 'pm',np.var(map(np.mean,Y_train))
# print np.mean(map(np.mean,Y_test)), 'pm',np.var(map(np.mean,Y_test))
# --- Train model --- #
model = EdgeFeatureGraphCRF(n_states,n_features,n_edge_features)
ssvm = NSlackSSVM(model=model, C=0.1, tol=0.001, verbose=0,show_loss_every=10)
# ssvm = OneSlackSSVM(model=model, C=.1, inference_cache=50, tol=0.1, verbose=0,show_loss_every=10)
ssvm.fit(X_train, Y_train)
# --- Test with pystruct --- #
# print("Test score with graph CRF: %f" % ssvm.score(X_test, Y_test))
# --- Test manually - get contingency tables --- #
prediction = ssvm.predict(X_test)
contingency = np.array([0,0,0,0])
for i in xrange(len(test)):
pred = prediction[i]
true = Y_test[i]
contingency = contingency+get_contingency(pred,true)
TP, FP, TN, FN = contingency[0], contingency[1], contingency[2], contingency[3]
# sens = float(TP)/(TP+FN)
# sens_all.append(sens)
# spec = float(TN)/(FP+TN)
# spec_all.append(spec)
acc = float(TP+TN)/(TP+FP+TN+FN)
acc_all.append(acc)
model = MultiClassClf(n_features=X_train_bias.shape[1], n_classes=10)
n_slack_svm = NSlackSSVM(model, verbose=2, check_constraints=False, C=0.1,
batch_size=100, tol=1e-2)
one_slack_svm = OneSlackSSVM(model, verbose=2, C=.10, tol=.001)
subgradient_svm = SubgradientSSVM(model, C=0.1, learning_rate=0.000001,
max_iter=1000, verbose=0)
fw_bc_svm = FrankWolfeSSVM(model, C=.1, max_iter=50)
fw_batch_svm = FrankWolfeSSVM(model, C=.1, max_iter=50, batch_mode=True)
# n-slack cutting plane ssvm
start = time()
n_slack_svm.fit(X_train_bias, y_train)
time_n_slack_svm = time() - start
y_pred = np.hstack(n_slack_svm.predict(X_test_bias))
print("Score with pystruct n-slack ssvm: %f (took %f seconds)"
% (np.mean(y_pred == y_test), time_n_slack_svm))
## 1-slack cutting plane ssvm
start = time()
one_slack_svm.fit(X_train_bias, y_train)
time_one_slack_svm = time() - start
y_pred = np.hstack(one_slack_svm.predict(X_test_bias))
print("Score with pystruct 1-slack ssvm: %f (took %f seconds)"
% (np.mean(y_pred == y_test), time_one_slack_svm))
#online subgradient ssvm
start = time()
subgradient_svm.fit(X_train_bias, y_train)
time_subgradient_svm = time() - start
"""
from time import time
import numpy as np
from sklearn.datasets import load_iris
from sklearn.cross_validation import train_test_split
from pystruct.models import GraphCRF
from pystruct.learners import NSlackSSVM
iris = load_iris()
X, y = iris.data, iris.target
# make each example into a tuple of a single feature vector and an empty edge
# list
X_ = [(np.atleast_2d(x), np.empty((0, 2), dtype=np.int)) for x in X]
Y = y.reshape(-1, 1)
X_train, X_test, y_train, y_test = train_test_split(X_, Y)
pbl = GraphCRF(inference_method='unary')
svm = NSlackSSVM(pbl, C=100)
start = time()
svm.fit(X_train, y_train)
time_svm = time() - start
y_pred = np.vstack(svm.predict(X_test))
print("Score with pystruct crf svm: %f (took %f seconds)" %
(np.mean(y_pred == y_test), time_svm))
from time import time
import numpy as np
from sklearn.datasets import load_iris
from sklearn.cross_validation import train_test_split
from pystruct.models import GraphCRF
from pystruct.learners import NSlackSSVM
iris = load_iris()
X, y = iris.data, iris.target
# make each example into a tuple of a single feature vector and an empty edge
# list
X_ = [(np.atleast_2d(x), np.empty((0, 2), dtype=np.int)) for x in X]
Y = y.reshape(-1, 1)
X_train, X_test, y_train, y_test = train_test_split(X_, Y)
pbl = GraphCRF(n_features=4, n_states=3, inference_method='lp')
svm = NSlackSSVM(pbl, verbose=1, check_constraints=True, C=100, n_jobs=1)
start = time()
svm.fit(X_train, y_train)
time_svm = time() - start
y_pred = np.vstack(svm.predict(X_test))
print("Score with pystruct crf svm: %f (took %f seconds)"
% (np.mean(y_pred == y_test), time_svm))
X_flat = [x.reshape(-1, 1).astype(np.float) for x in X]
Y_flat = [y.ravel() for y in Y]
# first, use standard graph CRF. Can't do much, high loss.
crf = GraphCRF(n_states=2, n_features=1, inference_method='lp')
svm = NSlackSSVM(model=crf, max_iter=200, C=1, verbose=0,
check_constraints=True, break_on_bad=False, n_jobs=1)
# make dataset from X and graph without edges
#G_ = [np.zeros((0, 2), dtype=np.int) for x in X]
G = [make_grid_edges(x) for x in X]
asdf = zip(X_flat, G)
svm.fit(asdf, Y_flat)
plot_boxes(svm.predict(asdf))
print("Training score multiclass svm CRF: %f" % svm.score(asdf, Y_flat))
# using one latent variable for each 2x2 rectangle
latent_crf = LatentNodeCRF(n_labels=2, n_features=1, inference_method='lp',
n_hidden_states=2)
#latent_svm = LatentSSVM(model=latent_crf, max_iter=200, C=10, verbose=10,
#check_constraints=True, break_on_bad=True, n_jobs=1,
#latent_iter=10, base_svm='subgradient', tol=-1,
#inactive_window=0, learning_rate=0.01, momentum=0)
latent_svm = LatentSubgradientSSVM(model=latent_crf, max_iter=200, C=100,
verbose=1, n_jobs=1, show_loss_every=10,
learning_rate=0.01, momentum=0)
# make edges for hidden states:
edges = []
