def test_energy():
# make sure that energy as computed by ssvm is the same as by lp
np.random.seed(0)
for inference_method in ["lp", "ad3"]:
found_fractional = False
crf = EdgeFeatureGraphCRF(n_states=3,
inference_method=inference_method,
n_edge_features=2)
while not found_fractional:
x = np.random.normal(size=(7, 8, 3))
edge_list = make_grid_edges(x, 4, return_lists=True)
edges = np.vstack(edge_list)
edge_features = edge_list_to_features(edge_list)
x = (x.reshape(-1, 3), edges, edge_features)
unary_params = np.random.normal(size=(3, 3))
pw1 = np.random.normal(size=(3, 3))
pw2 = np.random.normal(size=(3, 3))
w = np.hstack([unary_params.ravel(), pw1.ravel(), pw2.ravel()])
res, energy = crf.inference(x, w, relaxed=True, return_energy=True)
found_fractional = np.any(np.max(res[0], axis=-1) != 1)
psi = crf.psi(x, res)
energy_svm = np.dot(psi, w)
assert_almost_equal(energy, -energy_svm)
if not found_fractional:
# exact discrete labels, test non-relaxed version
res, energy = crf.inference(x, w, relaxed=False,
return_energy=True)
psi = crf.psi(x, res)
energy_svm = np.dot(psi, w)
assert_almost_equal(energy, -energy_svm)
def test_inference():
# Test inference with different weights in different directions
X, Y = generate_blocks_multinomial(noise=2, n_samples=1, seed=1)
x, y = X[0], Y[0]
n_states = x.shape[-1]
edge_list = make_grid_edges(x, 4, return_lists=True)
edges = np.vstack(edge_list)
pw_horz = -1 * np.eye(n_states)
xx, yy = np.indices(pw_horz.shape)
# linear ordering constraint horizontally
pw_horz[xx > yy] = 1
# high cost for unequal labels vertically
pw_vert = -1 * np.eye(n_states)
pw_vert[xx != yy] = 1
pw_vert *= 10
# generate edge weights
edge_weights_horizontal = np.repeat(pw_horz[np.newaxis, :, :],
edge_list[0].shape[0],
axis=0)
edge_weights_vertical = np.repeat(pw_vert[np.newaxis, :, :],
edge_list[1].shape[0],
axis=0)
edge_weights = np.vstack([edge_weights_horizontal, edge_weights_vertical])
# do inference
res = lp_general_graph(-x.reshape(-1, n_states), edges, edge_weights)
edge_features = edge_list_to_features(edge_list)
x = (x.reshape(-1, n_states), edges, edge_features)
y = y.ravel()
for inference_method in get_installed(["lp", "ad3"]):
# same inference through CRF inferface
crf = EdgeFeatureGraphCRF(inference_method=inference_method)
crf.initialize([x], [y])
w = np.hstack([np.eye(3).ravel(), -pw_horz.ravel(), -pw_vert.ravel()])
y_pred = crf.inference(x, w, relaxed=True)
if isinstance(y_pred, tuple):
# ad3 produces an integer result if it found the exact solution
assert_array_almost_equal(res[1], y_pred[1])
assert_array_almost_equal(res[0], y_pred[0].reshape(-1, n_states))
assert_array_equal(y, np.argmax(y_pred[0], axis=-1))
for inference_method in get_installed(["lp", "ad3", "qpbo"]):
# again, this time discrete predictions only
crf = EdgeFeatureGraphCRF(n_states=3,
inference_method=inference_method,
n_edge_features=2)
crf.initialize([x], [y])
w = np.hstack([np.eye(3).ravel(), -pw_horz.ravel(), -pw_vert.ravel()])
y_pred = crf.inference(x, w, relaxed=False)
assert_array_equal(y, y_pred)
def test_inference():
# Test inference with different weights in different directions
X, Y = toy.generate_blocks_multinomial(noise=2, n_samples=1, seed=1)
x, y = X[0], Y[0]
n_states = x.shape[-1]
edge_list = make_grid_edges(x, 4, return_lists=True)
edges = np.vstack(edge_list)
pw_horz = -1 * np.eye(n_states)
xx, yy = np.indices(pw_horz.shape)
# linear ordering constraint horizontally
pw_horz[xx > yy] = 1
# high cost for unequal labels vertically
pw_vert = -1 * np.eye(n_states)
pw_vert[xx != yy] = 1
pw_vert *= 10
# generate edge weights
edge_weights_horizontal = np.repeat(pw_horz[np.newaxis, :, :],
edge_list[0].shape[0], axis=0)
edge_weights_vertical = np.repeat(pw_vert[np.newaxis, :, :],
edge_list[1].shape[0], axis=0)
edge_weights = np.vstack([edge_weights_horizontal, edge_weights_vertical])
# do inference
res = lp_general_graph(-x.reshape(-1, n_states), edges, edge_weights)
edge_features = edge_list_to_features(edge_list)
x = (x.reshape(-1, n_states), edges, edge_features)
y = y.ravel()
for inference_method in get_installed(["lp", "ad3"]):
# same inference through CRF inferface
crf = EdgeFeatureGraphCRF(n_states=3,
inference_method=inference_method,
n_edge_features=2)
w = np.hstack([np.eye(3).ravel(), -pw_horz.ravel(), -pw_vert.ravel()])
y_pred = crf.inference(x, w, relaxed=True)
if isinstance(y_pred, tuple):
# ad3 produces an integer result if it found the exact solution
assert_array_almost_equal(res[1], y_pred[1])
assert_array_almost_equal(res[0], y_pred[0].reshape(-1, n_states))
assert_array_equal(y, np.argmax(y_pred[0], axis=-1))
for inference_method in get_installed(["lp", "ad3", "qpbo"]):
# again, this time discrete predictions only
crf = EdgeFeatureGraphCRF(n_states=3,
inference_method=inference_method,
n_edge_features=2)
w = np.hstack([np.eye(3).ravel(), -pw_horz.ravel(), -pw_vert.ravel()])
y_pred = crf.inference(x, w, relaxed=False)
assert_array_equal(y, y_pred)
def test_psi_continuous():
# FIXME
# first make perfect prediction, including pairwise part
X, Y = toy.generate_blocks_multinomial(noise=2, n_samples=1, seed=1)
x, y = X[0], Y[0]
n_states = x.shape[-1]
edge_list = make_grid_edges(x, 4, return_lists=True)
edges = np.vstack(edge_list)
edge_features = edge_list_to_features(edge_list)
x = (x.reshape(-1, 3), edges, edge_features)
y = y.ravel()
pw_horz = -1 * np.eye(n_states)
xx, yy = np.indices(pw_horz.shape)
# linear ordering constraint horizontally
pw_horz[xx > yy] = 1
# high cost for unequal labels vertically
pw_vert = -1 * np.eye(n_states)
pw_vert[xx != yy] = 1
pw_vert *= 10
# create crf, assemble weight, make prediction
for inference_method in ["lp", "ad3"]:
crf = EdgeFeatureGraphCRF(n_states=3,
inference_method=inference_method,
n_edge_features=2)
w = np.hstack([np.eye(3).ravel(), -pw_horz.ravel(), -pw_vert.ravel()])
y_pred = crf.inference(x, w, relaxed=True)
# compute psi for prediction
psi_y = crf.psi(x, y_pred)
assert_equal(psi_y.shape, (crf.size_psi,))
def test_energy_discrete():
for inference_method in get_installed(["qpbo", "ad3"]):
crf = EdgeFeatureGraphCRF(n_states=3,
inference_method=inference_method,
n_edge_features=2)
for i in xrange(10):
x = np.random.normal(size=(7, 8, 3))
edge_list = make_grid_edges(x, 4, return_lists=True)
edges = np.vstack(edge_list)
edge_features = edge_list_to_features(edge_list)
x = (x.reshape(-1, 3), edges, edge_features)
unary_params = np.random.normal(size=(3, 3))
pw1 = np.random.normal(size=(3, 3))
pw2 = np.random.normal(size=(3, 3))
w = np.hstack([unary_params.ravel(), pw1.ravel(), pw2.ravel()])
y_hat = crf.inference(x, w, relaxed=False)
energy = compute_energy(crf.get_unary_potentials(x, w),
crf.get_pairwise_potentials(x, w), edges,
y_hat)
psi = crf.psi(x, y_hat)
energy_svm = np.dot(psi, w)
assert_almost_equal(energy, energy_svm)
def test_energy_continuous():
# make sure that energy as computed by ssvm is the same as by lp
np.random.seed(0)
for inference_method in get_installed(["lp", "ad3"]):
found_fractional = False
crf = EdgeFeatureGraphCRF(n_states=3,
inference_method=inference_method,
n_edge_features=2, n_features=3)
while not found_fractional:
x = np.random.normal(size=(7, 8, 3))
edge_list = make_grid_edges(x, 4, return_lists=True)
edges = np.vstack(edge_list)
edge_features = edge_list_to_features(edge_list)
x = (x.reshape(-1, 3), edges, edge_features)
unary_params = np.random.normal(size=(3, 3))
pw1 = np.random.normal(size=(3, 3))
pw2 = np.random.normal(size=(3, 3))
w = np.hstack([unary_params.ravel(), pw1.ravel(), pw2.ravel()])
res, energy = crf.inference(x, w, relaxed=True, return_energy=True)
found_fractional = np.any(np.max(res[0], axis=-1) != 1)
joint_feature = crf.joint_feature(x, res)
energy_svm = np.dot(joint_feature, w)
assert_almost_equal(energy, -energy_svm)
def test_energy_discrete():
for inference_method in get_installed(["qpbo", "ad3"]):
crf = EdgeFeatureGraphCRF(n_states=3,
inference_method=inference_method,
n_edge_features=2, n_features=3)
for i in xrange(10):
x = np.random.normal(size=(7, 8, 3))
edge_list = make_grid_edges(x, 4, return_lists=True)
edges = np.vstack(edge_list)
edge_features = edge_list_to_features(edge_list)
x = (x.reshape(-1, 3), edges, edge_features)
unary_params = np.random.normal(size=(3, 3))
pw1 = np.random.normal(size=(3, 3))
pw2 = np.random.normal(size=(3, 3))
w = np.hstack([unary_params.ravel(), pw1.ravel(), pw2.ravel()])
y_hat = crf.inference(x, w, relaxed=False)
energy = compute_energy(crf._get_unary_potentials(x, w),
crf._get_pairwise_potentials(x, w), edges,
y_hat)
joint_feature = crf.joint_feature(x, y_hat)
energy_svm = np.dot(joint_feature, w)
assert_almost_equal(energy, energy_svm)
def test_energy_continuous():
# make sure that energy as computed by ssvm is the same as by lp
np.random.seed(0)
for inference_method in get_installed(["lp", "ad3"]):
found_fractional = False
crf = EdgeFeatureGraphCRF(n_states=3,
inference_method=inference_method,
n_edge_features=2, n_features=3)
while not found_fractional:
x = np.random.normal(size=(7, 8, 3))
edge_list = make_grid_edges(x, 4, return_lists=True)
edges = np.vstack(edge_list)
edge_features = edge_list_to_features(edge_list)
x = (x.reshape(-1, 3), edges, edge_features)
unary_params = np.random.normal(size=(3, 3))
pw1 = np.random.normal(size=(3, 3))
pw2 = np.random.normal(size=(3, 3))
w = np.hstack([unary_params.ravel(), pw1.ravel(), pw2.ravel()])
res, energy = crf.inference(x, w, relaxed=True, return_energy=True)
found_fractional = np.any(np.max(res[0], axis=-1) != 1)
joint_feature = crf.joint_feature(x, res)
energy_svm = np.dot(joint_feature, w)
assert_almost_equal(energy, -energy_svm)
def test_joint_feature_continuous():
# FIXME
# first make perfect prediction, including pairwise part
X, Y = generate_blocks_multinomial(noise=2, n_samples=1, seed=1)
x, y = X[0], Y[0]
n_states = x.shape[-1]
edge_list = make_grid_edges(x, 4, return_lists=True)
edges = np.vstack(edge_list)
edge_features = edge_list_to_features(edge_list)
x = (x.reshape(-1, 3), edges, edge_features)
y = y.ravel()
pw_horz = -1 * np.eye(n_states)
xx, yy = np.indices(pw_horz.shape)
# linear ordering constraint horizontally
pw_horz[xx > yy] = 1
# high cost for unequal labels vertically
pw_vert = -1 * np.eye(n_states)
pw_vert[xx != yy] = 1
pw_vert *= 10
# create crf, assemble weight, make prediction
for inference_method in get_installed(["lp", "ad3"]):
crf = EdgeFeatureGraphCRF(inference_method=inference_method)
w = np.hstack([np.eye(3).ravel(), -pw_horz.ravel(), -pw_vert.ravel()])
crf.initialize([x], [y])
y_pred = crf.inference(x, w, relaxed=True)
# compute joint_feature for prediction
joint_feature_y = crf.joint_feature(x, y_pred)
assert_equal(joint_feature_y.shape, (crf.size_joint_feature,))
class Model:
def __init__(self, X, y, learners, learners_parameters, models, models_parameters, total_label_one_avg, row_threshold):
saved_args = locals()
del saved_args['X'] # preventing from logging
del saved_args['y'] # preventing from logging
Arguments.logArguments('Model', saved_args)
self.X = X
self.y = y
self.learners = learners
self.learners_parameters = learners_parameters
self.models = models
self.models_parameters = models_parameters
self.total_label_one_avg = total_label_one_avg
self.row_threshold = row_threshold # needed to define graph model
self.total_accuracy_list = list() # each image and relevance accuracy
return
def run(self):
self.check_input()
self.define_model()
self.define_learners()
self.fit_model()
# self.inference()
# self.train_accuracy()
return
def check_input(self):
if self.learners not in ['OneSlackSSVM', 'SubgradientSSVM', 'StructuredPerceptron']:
raise('learners is not supported: ' + str(self.learners))
if self.models not in ['GraphCRF', 'GridCRF', 'EdgeFeatureGraphCRF']:
raise ('learners is not supported: ' + str(self.learners))
if self.models_parameters['neighborhood'] not in [4,8]:
raise ('neighborhood number must be 4/8')
return
def define_model(self):
if self.models == 'GridCRF':
from pystruct.models import GridCRF
self.crf = GridCRF(
neighborhood=self.models_parameters['neighborhood'],
inference_method=self.models_parameters['inference']
)
if self.models == 'GraphCRF':
self.prepare_data_to_graph_crf()
logging.info('Class weight: ' + str([self.total_label_one_avg,1-self.total_label_one_avg]))
from pystruct.models import GraphCRF
self.crf = GraphCRF(
inference_method=self.models_parameters['inference'],
directed = self.models_parameters['directed'],
class_weight = [self.total_label_one_avg,1-self.total_label_one_avg]
# class_weight=[0.01, 0.99]
)
if self.models == 'EdgeFeatureGraphCRF':
self.prepare_data_to_edge_feature_graph_crf()
logging.info('Class weight: ' + str([self.total_label_one_avg, 1 - self.total_label_one_avg]))
from pystruct.models import EdgeFeatureGraphCRF
self.crf = EdgeFeatureGraphCRF(
inference_method=self.models_parameters['inference'],
# directed=self.models_parameters['directed'],
class_weight=[self.total_label_one_avg, 1 - self.total_label_one_avg]
# class_weight=[0.01, 0.99]
)
return
def define_learners(self):
if self.learners == 'OneSlackSSVM':
import pystruct.learners as ssvm
self.clf = ssvm.OneSlackSSVM(
model=self.crf,
# C=100,
# inference_cache=100,
# tol=.1,
verbose=self.learners_parameters['verbose'],
max_iter=self.learners_parameters['max_iter'],
n_jobs=self.learners_parameters['n_jobs']
)
if self.learners == 'SubgradientSSVM':
import pystruct.learners as ssvm
self.clf = ssvm.OneSlackSSVM(
model=self.crf,
# C=100,
# inference_cache=100,
# tol=.1,
verbose=self.learners_parameters['verbose'],
max_iter=self.learners_parameters['max_iter'],
n_jobs=self.learners_parameters['n_jobs'],
show_loss_every = self.learners_parameters['show_loss_every']
)
if self.learners == 'StructuredPerceptron':
import pystruct.learners as structured_perceptron
self.clf = structured_perceptron.StructuredPerceptron(
model=self.crf,
# C=100,
# inference_cache=100,
# tol=.1,
verbose=self.learners_parameters['verbose'],
max_iter=self.learners_parameters['max_iter'],
n_jobs=self.learners_parameters['n_jobs'],
# show_loss_every=self.learners_parameters['show_loss_every']
)
return
def fit_model(self):
logging.info("Start fit model")
self.clf.fit(self.X, self.y)
self.w = self.clf.w
# self.objective_curve_ = self.clf.objective_curve_
# self.training_timestamps = self.clf.timestamps_
# self.loss_curve_ = self.clf.loss_curve_
logging.info('weight after fit:')
logging.info(self.w)
# logging.info('objective curve: (cutting plane objective)')
# logging.info(self.objective_curve_)
# logging.info('loss curve: (current loss)')
# logging.info(self.loss_curve_)
logging.info('training time each iteration:')
# logging.info(self.training_timestamps)
# logging.info("Finish fit model")
return
# not in use (in eval class we use inference)
def inference(self):
logging.info("Start inference - crf.inference")
for index, cur_nd_array in enumerate(self.X):
predict_picture = self.crf.inference(cur_nd_array, self.w)
logging.info('inference result number: ' + str(index))
# logging.info(predict_picture)
# logging.info(self.y[index])
accuracy = self.calculate_metrics(predict_picture, self.y[index])
self.total_accuracy_list.append(accuracy)
logging.info('accuracy: ' + str(accuracy))
# logging.info('F1 macro: ' + str(f1_score(predict_picture, self.y[index], average='macro')))
# logging.info('F1 micro: ' + str(f1_score(predict_picture, self.y[index], average='micro')))
logging.info('Total accuracy: ' + str(sum(self.total_accuracy_list)/len(self.total_accuracy_list)))
logging.info("Finish inference")
return
def calculate_metrics(self, predict_picture, real_picture):
row_loss = list()
for i, c_list in enumerate(predict_picture):
row_loss.append(zero_one_loss(c_list, real_picture[i]))
return 1- sum(row_loss)/len(row_loss)
def prepare_data_to_graph_crf(self):
from pystruct.utils import make_grid_edges, edge_list_to_features
self.X_flatten = []
self.y_flatten = []
for pic_i, pic_nd_array in enumerate(self.X):
pic_item = list()
cell_index_place = 0
for row_i, row_val in enumerate(pic_nd_array): # pic item
for col_i, cell_features in enumerate(row_val): # pic row iteration cell by cell
pic_item.append(cell_features)
if self.models_parameters['neighborhood'] == 4:
right, down = make_grid_edges(pic_nd_array, neighborhood=4, return_lists=True)
# right, down, upright, downright = make_grid_edges(pic_nd_array, neighborhood=8, return_lists=True)
edges = np.vstack([right, down])
elif self.models_parameters['neighborhood'] == 8:
right, down, upright, downright = make_grid_edges(pic_nd_array, neighborhood=8, return_lists=True)
edges = np.vstack([right, down, upright, downright])
# for val in range
# Guy version - old
# edges_item = list()
# max_cell_index = self.row_threshold * self.row_threshold
# last_row_first_index = max_cell_index - self.row_threshold # e.g. 36-6
# for i in range(0, max_cell_index):
# if i<last_row_first_index: # except last row
# edges_item.append(np.array([i, i + self.row_threshold]))
#
# if (i+1)%self.row_threshold != 0: # except last col
# edges_item.append(np.array([i, i + 1]))
# finish iterate picture
self.X_flatten.append((np.array(pic_item), edges))
for pic_i, pic_nd_array in enumerate(self.y):
pic_item = list()
for row_i, row_val in enumerate(pic_nd_array): # pic item
for col_i, cell_features in enumerate(row_val): # pic row iteration cell by cell
pic_item.append(cell_features)
self.y_flatten.append(pic_item)
self.X = np.array(self.X_flatten)
self.y = np.array(self.y_flatten)
return
def prepare_data_to_edge_feature_graph_crf(self):
from pystruct.utils import make_grid_edges, edge_list_to_features
self.X_flatten = []
self.y_flatten = []
for pic_i, pic_nd_array in enumerate(self.X):
pic_item = list()
for row_i, row_val in enumerate(pic_nd_array): # pic item
for col_i, cell_features in enumerate(row_val): # pic row iteration cell by cell
pic_item.append(cell_features)
if self.models_parameters['neighborhood'] == 4:
# 4 neigh and cross type
if 'type' in self.models_parameters and self.models_parameters['type'] == 'cross_edge':
upright, downright = self.cross_make_grid_edges(pic_nd_array, neighborhood=4, return_lists=True)
edges = np.vstack([upright, downright])
edge_features_directions = self.edge_list_to_features([upright, downright], 4)
else: # regular
right, down = make_grid_edges(pic_nd_array, neighborhood=4, return_lists=True)
edges = np.vstack([right, down])
edge_features_directions = self.edge_list_to_features([right, down], 4)
elif self.models_parameters['neighborhood'] == 8:
right, down, upright, downright = make_grid_edges(pic_nd_array, neighborhood=8, return_lists=True)
edges = np.vstack([right, down, upright, downright])
edge_features_directions = self.edge_list_to_features([right, down, upright, downright], 8)
# finish iterate picture - (pixel feature (list), edge (pixel-pixel) list)
self.X_flatten.append((np.array(pic_item), edges, edge_features_directions))
for pic_i, pic_nd_array in enumerate(self.y):
pic_item = list()
for row_i, row_val in enumerate(pic_nd_array): # pic item
for col_i, cell_features in enumerate(row_val): # pic row iteration cell by cell
pic_item.append(cell_features)
self.y_flatten.append(pic_item)
self.X = np.array(self.X_flatten)
self.y = np.array(self.y_flatten)
return
# add direction feature to each edge
def edge_list_to_features(self, edge_list, neighborhood):
edges = np.vstack(edge_list)
if neighborhood == 4:
edge_features = np.zeros((edges.shape[0], 2))
edge_features[:len(edge_list[0]), 0] = 1
edge_features[len(edge_list[0]):, 1] = 1
elif neighborhood == 8:
limit_0 = len(edge_list[0])
limit_1 = len(edge_list[0]) + len(edge_list[1])
limit_2 = len(edge_list[0]) + len(edge_list[1]) + len(edge_list[2])
limit_3 = len(edge_list[0]) + len(edge_list[1]) + len(edge_list[2]) + + len(edge_list[3])
edge_features = np.zeros((edges.shape[0], 4))
edge_features[:limit_0, 0] = 1
edge_features[limit_0:limit_1, 1] = 1
edge_features[limit_1:limit_2, 2] = 1
edge_features[limit_2:limit_3, 3] = 1
else:
raise('number neighborhood is not supported 4/8')
return edge_features
def cross_make_grid_edges(self, x, neighborhood=4, return_lists=False):
if neighborhood not in [4]:
raise ValueError("neighborhood can only be '4', got %s" %
repr(neighborhood))
inds = np.arange(x.shape[0] * x.shape[1]).reshape(x.shape[:2])
inds = inds.astype(np.int64)
upright = np.c_[inds[1:, :-1].ravel(), inds[:-1, 1:].ravel()]
downright = np.c_[inds[:-1, :-1].ravel(), inds[1:, 1:].ravel()]
edges = [upright, downright]
if return_lists:
return edges
return np.vstack(edges)
Y.extend(folds[i]['Y'])
print "Training Size: ", len(X), len(Y)
ssvm.fit(X, Y)
print 'Train Score: ' + str(ssvm.score(X, Y))
# w = Weight(ssvm.w, node_features, edge_features, dictLength)
fold_correct = 0
fold_total = 0
testX = folds[fold]['X']
testY = folds[fold]['Y']
for i in range(len(testX)):
print "Instance: " + str(i)
print folds[fold]['threads'][i].id
Yi = testY[i]
print list(Yi)
infY = crf.inference(testX[i], ssvm.w)
print list(infY)
for py in range(len(Yi)):
recall[Yi[py]][1] += 1
if infY[py] == Yi[py]:
recall[Yi[py]][0] += 1
for py in range(len(infY)):
precision[infY[py]][1] += 1
if infY[py] == Yi[py]:
precision[infY[py]][0] += 1
fold_total += len(Yi)
for r in range(len(Yi)):
if Yi[r] == infY[r]:
fold_correct += 1
total_correct += fold_correct
total += fold_total
