def test_multilabel_fully():
# test inference and energy with fully connected model
n_features = 5
n_labels = 4
edges = np.vstack([x for x in itertools.combinations(range(n_labels), 2)])
model = MultiLabelClf(n_labels=n_labels, n_features=n_features,
edges=edges)
rnd = np.random.RandomState(0)
x = rnd.normal(size=n_features)
w = rnd.normal(size=n_features * n_labels + 4 * len(edges))
y = model.inference(x, w)
# test joint_feature / energy
joint_feature = model.joint_feature(x, y)
energy = compute_energy(model._get_unary_potentials(x, w),
model._get_pairwise_potentials(x, w), edges, y)
assert_almost_equal(energy, np.dot(joint_feature, w))
# for continuous y
#y_cont = model.inference(x, w, relaxed=True)
y_continuous = np.zeros((n_labels, 2))
pairwise_marginals = []
for edge in edges:
# indicator of one of four possible states of the edge
pw = np.zeros((2, 2))
pw[y[edge[0]], y[edge[1]]] = 1
pairwise_marginals.append(pw)
pairwise_marginals = np.vstack(pairwise_marginals)
y_continuous[np.arange(n_labels), y] = 1
assert_array_almost_equal(
joint_feature, model.joint_feature(x, (y_continuous, pairwise_marginals)))
def test_energy_discrete():
for inference_method in get_installed(["qpbo", "ad3"]):
crf = EdgeFeatureGraphCRF(n_states=3,
inference_method=inference_method,
n_edge_features=2)
for i in xrange(10):
x = np.random.normal(size=(7, 8, 3))
edge_list = make_grid_edges(x, 4, return_lists=True)
edges = np.vstack(edge_list)
edge_features = edge_list_to_features(edge_list)
x = (x.reshape(-1, 3), edges, edge_features)
unary_params = np.random.normal(size=(3, 3))
pw1 = np.random.normal(size=(3, 3))
pw2 = np.random.normal(size=(3, 3))
w = np.hstack([unary_params.ravel(), pw1.ravel(), pw2.ravel()])
y_hat = crf.inference(x, w, relaxed=False)
energy = compute_energy(crf.get_unary_potentials(x, w),
crf.get_pairwise_potentials(x, w), edges,
y_hat)
psi = crf.psi(x, y_hat)
energy_svm = np.dot(psi, w)
assert_almost_equal(energy, energy_svm)
def test_multilabel_independent():
# test inference and energy with independent model
edges = np.zeros((0, 2), dtype=np.int)
n_features = 5
n_labels = 4
model = MultiLabelClf(n_labels=n_labels, n_features=n_features,
edges=edges)
rnd = np.random.RandomState(0)
x = rnd.normal(size=5)
w = rnd.normal(size=n_features * n_labels)
# test inference
y = model.inference(x, w)
y_ = np.dot(w.reshape(n_labels, n_features), x) > 0
assert_array_equal(y, y_)
# test joint_feature / energy
joint_feature = model.joint_feature(x, y)
energy = compute_energy(model._get_unary_potentials(x, w),
model._get_pairwise_potentials(x, w), edges, y)
assert_almost_equal(energy, np.dot(joint_feature, w))
# for continuous y
y_continuous = np.zeros((n_labels, 2))
y_continuous[np.arange(n_labels), y] = 1
assert_array_almost_equal(
joint_feature, model.joint_feature(x, (y_continuous, np.zeros((0, n_labels, n_labels)))))
def test_energy_discrete():
for inference_method in get_installed(["qpbo", "ad3"]):
crf = EdgeFeatureGraphCRF(n_states=3,
inference_method=inference_method,
n_edge_features=2, n_features=3)
for i in xrange(10):
x = np.random.normal(size=(7, 8, 3))
edge_list = make_grid_edges(x, 4, return_lists=True)
edges = np.vstack(edge_list)
edge_features = edge_list_to_features(edge_list)
x = (x.reshape(-1, 3), edges, edge_features)
unary_params = np.random.normal(size=(3, 3))
pw1 = np.random.normal(size=(3, 3))
pw2 = np.random.normal(size=(3, 3))
w = np.hstack([unary_params.ravel(), pw1.ravel(), pw2.ravel()])
y_hat = crf.inference(x, w, relaxed=False)
energy = compute_energy(crf._get_unary_potentials(x, w),
crf._get_pairwise_potentials(x, w), edges,
y_hat)
joint_feature = crf.joint_feature(x, y_hat)
energy_svm = np.dot(joint_feature, w)
assert_almost_equal(energy, energy_svm)
def test_energy_discrete():
# for inference_method in get_installed(["qpbo", "ad3"]):
# crf = EdgeFeatureGraphCRF(n_states=3,
# inference_method=inference_method,
# n_edge_features=2, n_features=3)
crf = NodeTypeEdgeFeatureGraphCRF(1, [3], [3], [[2]])
for i in range(10):
x = np.random.normal(size=(7, 8, 3))
edge_list = make_grid_edges(x, 4, return_lists=True)
edges = np.vstack(edge_list)
edge_features = edge_list_to_features(edge_list)
x = ([x.reshape(-1, 3)], [edges], [edge_features])
unary_params = np.random.normal(size=(3, 3))
pw1 = np.random.normal(size=(3, 3))
pw2 = np.random.normal(size=(3, 3))
w = np.hstack([unary_params.ravel(), pw1.ravel(), pw2.ravel()])
crf.initialize(x)
y_hat = crf.inference(x, w, relaxed=False)
#flat_edges = crf._index_all_edges(x)
energy = compute_energy(crf._get_unary_potentials(x, w)[0],
crf._get_pairwise_potentials(x, w)[0], edges, #CAUTION: pass the flatened edges!!
y_hat)
joint_feature = crf.joint_feature(x, y_hat)
energy_svm = np.dot(joint_feature, w)
assert_almost_equal(energy, energy_svm)
def test_demo():
frm_num=3
size=frm_num
n_states=2
unaries= np.array([[0.6, 0.5], [1, 0], [.4, .6]])
edges= np.array([[0, 1], [1, 2]])
pairwise=np.array([[0, 0], [0, 0]])
print unaries.shape
print edges.shape
print pairwise.shape
fig, ax = plt.subplots(1, 2, figsize=(3, 1))
##for a, inference_method in zip(ax, ['ad3', 'qpbo', 'max-product',
## ('max-product', {'max_iter': 10}), 'lp']):
for a, inference_method in zip(ax, ['lp']):
start = time()
print a, inference_method
y = inference_dispatch(unaries, pairwise, edges,
inference_method=inference_method) ##(400)
took = time() - start
a.matshow(y.reshape(size, 1))
print y.shape
print y
energy = compute_energy(unaries, pairwise, edges, y)
a.set_title(str(inference_method) + "\n time: %.2f energy %.2f" % (took, energy))
##a.set_xticks(())
##a.set_yticks(())
plt.show()
def test_multilabel_independent():
# test inference and energy with independent model
edges = np.zeros((0, 2), dtype=np.int)
n_features = 5
n_labels = 4
model = MultiLabelClf(n_labels=n_labels,
n_features=n_features,
edges=edges)
rnd = np.random.RandomState(0)
x = rnd.normal(size=5)
w = rnd.normal(size=n_features * n_labels)
# test inference
y = model.inference(x, w)
y_ = np.dot(w.reshape(n_labels, n_features), x) > 0
assert_array_equal(y, y_)
# test joint_feature / energy
joint_feature = model.joint_feature(x, y)
energy = compute_energy(model._get_unary_potentials(x, w),
model._get_pairwise_potentials(x, w), edges, y)
assert_almost_equal(energy, np.dot(joint_feature, w))
# for continuous y
y_continuous = np.zeros((n_labels, 2))
y_continuous[np.arange(n_labels), y] = 1
assert_array_almost_equal(
joint_feature,
model.joint_feature(x, (y_continuous, np.zeros(
(0, n_labels, n_labels)))))
def generate_Potts(shape=(10, 10),
ncolors=2,
beta=1.0,
inference='max-product'):
"""Generate Potts image."""
# Generate initial normal image
x = rnd.normal(size=(*shape, ncolors))
# Unary potentials
unaries = x.reshape(-1, ncolors)
# Pairwise potentials
pairwise = beta*np.eye(ncolors)
# Generate edge matrix
edges = make_grid_edges(x)
# Start clock
start = time()
# Infer image
y = inference_dispatch(unaries, pairwise, edges,
inference_method=inference)
# End clock
took = time() - start
print('Inference took ' + str(took) + ' seconds')
# Compute energy
energy = compute_energy(unaries, pairwise, edges, y)
# Return inferred image and energy
return np.reshape(y, shape), energy
def risk(self, flatten_w, x, y_true, y_true_labels, dim=0, k=0):
n, _ = x.shape[0], x.shape[1]
if dim == 0 and k == 0:
dim = self.dim
k = self.k
W = flatten_w[:dim * k].reshape(k, dim).T
A = flatten_w[dim * k:dim * k + k * k].reshape(k, k).T
b = flatten_w[dim * k + k * k:]
y_pred = self.loss_augmented_decoding(W, A, b, x, y_true_labels, k)
S = np.dot(x, W) + b
edges = make_grid_edges(S.reshape(1, n, k))
pairwise = A
unaries = S
loss = self.hamming_loss_base(y_true_labels, y_pred)
score_y = compute_energy(unaries, pairwise, edges, y_true_labels)
score_y_pred = compute_energy(unaries, pairwise, edges, y_pred)
return loss - score_y + score_y_pred
def test_multilabel_fully():
# test inference and energy with fully connected model
n_features = 5
n_labels = 4
edges = np.vstack([x for x in itertools.combinations(range(n_labels), 2)])
model = MultiLabelClf(n_labels=n_labels,
n_features=n_features,
edges=edges)
rnd = np.random.RandomState(0)
x = rnd.normal(size=n_features)
w = rnd.normal(size=n_features * n_labels + 4 * len(edges))
y = model.inference(x, w)
# test joint_feature / energy
joint_feature = model.joint_feature(x, y)
energy = compute_energy(model._get_unary_potentials(x, w),
model._get_pairwise_potentials(x, w), edges, y)
assert_almost_equal(energy, np.dot(joint_feature, w))
# for continuous y
#y_cont = model.inference(x, w, relaxed=True)
y_continuous = np.zeros((n_labels, 2))
pairwise_marginals = []
for edge in edges:
# indicator of one of four possible states of the edge
pw = np.zeros((2, 2))
pw[y[edge[0]], y[edge[1]]] = 1
pairwise_marginals.append(pw)
pairwise_marginals = np.vstack(pairwise_marginals)
y_continuous[np.arange(n_labels), y] = 1
assert_array_almost_equal(
joint_feature,
model.joint_feature(x, (y_continuous, pairwise_marginals)))
def crf_infer(fw_masks,cur_masks,bw_masks):
print '================================using CRF to do inference=========================================================='
frm_num=len(cur_masks)
n_nodes=frm_num
states_per_frm=3
n_states=frm_num*states_per_frm
mask_sample=cur_masks[0]
w_mask=mask_sample.shape[1]
h_mask=mask_sample.shape[0]
area_mask=w_mask*h_mask ## area_mask is fixed here, but can be different later.
mask1=fw_masks[0]
mask2=cur_masks[0]
match_mask=np.logical_and(mask1, mask2)
##np.logical_and([True, False], [False, False])
##put all three kinds of mask in one category
state_masks=[]
for node_id in xrange(n_nodes):
state_masks.append(fw_masks[node_id])
state_masks.append(cur_masks[node_id])
state_masks.append(bw_masks[node_id])
##(1) define edges
edges=np.zeros((n_nodes-1,2),dtype=int)
##edges=np.zeros((2,2),dtype=int) ##(node-1,2)
for node_id in xrange(n_nodes-1):
edges[node_id,:]=[node_id,node_id+1]
##(2) define unaries
## calculate unary values within the same frame
# unary_vals=np.zeros((n_nodes,n_states))
# for node_id in xrange(n_nodes):
# mask1=fw_masks[node_id]
# mask2=cur_masks[node_id]
# mask3=bw_masks[node_id]
# inter_1_2=inter_2_1=np.logical_and(mask1,mask2)
# inter_2_3=inter_3_2=np.logical_and(mask2,mask3)
# inter_1_3=inter_3_1=np.logical_and(mask1,mask3)
# val1=np.sum(inter_1_2)+np.sum(inter_1_3)
# val2=np.sum(inter_2_1)+np.sum(inter_2_3)
# val3=np.sum(inter_3_1)+np.sum(inter_3_2)
# val_sum=val1+val2+val3+0.0
# norm_val1=val1/val_sum
# norm_val2=val2/val_sum
# norm_val3=val3/val_sum
# t_nodes=[node_id*states_per_frm,node_id*states_per_frm+1,node_id*states_per_frm+2]
# unary_vals[node_id,t_nodes]=[norm_val1,norm_val2,norm_val3]
# unaries=unary_vals
##ignore unary terms:
unary_vals=np.ones((n_nodes,n_states))
unaries=unary_vals
# ##(2) define unaries
#unaries=np.zeros((n_nodes,n_states))
# for node_id in xrange(n_nodes):
# t_nodes=[node_id*states_per_frm,node_id*states_per_frm+1,node_id*states_per_frm+2]
# unaries[node_id,t_nodes]=1.0/3
##(3) define pairwise
pairwise=np.zeros((n_states, n_states))
t_base=np.zeros((n_nodes,n_nodes))
for state_id1 in xrange(n_states):
t1=state_id1/states_per_frm
for state_id2 in xrange(n_states):
t2=state_id2/states_per_frm
if t1==t2 or abs(t2-t1)>1 :
continue
else:
mask1=state_masks[state_id1]
mask2=state_masks[state_id2]
iou_1_2=np.logical_and(mask1,mask2)
val_1_2=np.sum(iou_1_2)
pairwise[state_id1,state_id2]=val_1_2
t_base[t1,t2]=val_1_2+t_base[t1,t2]
##s1: normalize based on two consective frames(9 pairs)
for state_id1 in xrange(n_states):
t1=state_id1/states_per_frm
for state_id2 in xrange(n_states):
t2=state_id2/states_per_frm
if t1==t2 or abs(t2-t1)>1 :
continue
else:
pairwise[state_id1,state_id2]=pairwise[state_id1,state_id2]/t_base[t1,t2]
pairwise=pairwise
##s2: normalize on the whole sequence.
##print 't_base:\n', t_base
##print 'unaries:\n',unaries
##print 'pairwise:\n',pairwise
print 'unaries.shape:',unaries.shape
print 'edges.shape:',edges.shape
print 'pairwise.shape:',pairwise.shape
fig, ax = plt.subplots(1, 2, figsize=(n_nodes, 1))
##for a, inference_method in zip(ax, ['ad3', 'qpbo', 'max-product',
## ('max-product', {'max_iter': 10}), 'lp']):
for a, inference_method in zip(ax, ['lp']):
start = time()
##print a, inference_method
y = inference_dispatch(unaries, pairwise, edges,
inference_method=inference_method) ##(400)
took = time() - start
a.matshow(y.reshape(n_nodes, 1))
##print y.shape
energy = compute_energy(unaries, pairwise, edges, y)
##a.set_title(str(inference_method) + "\n time: %.2f energy %.2f" % (took, energy))
##a.set_xticks(())
##a.set_yticks(())
##plt.show()
arr_state_masks=np.asarray(state_masks)
picked_masks= arr_state_masks[y]
return y,picked_masks
import numpy as np
import matplotlib.pyplot as plt
from time import time
from pystruct.inference import inference_dispatch, compute_energy
from pystruct.utils import make_grid_edges
size = 20
n_states = 5
rnd = np.random.RandomState(2)
x = rnd.normal(size=(size, size, n_states))
pairwise = np.eye(n_states)
edges = make_grid_edges(x)
unaries = x.reshape(-1, n_states)
fig, ax = plt.subplots(1, 5, figsize=(20, 5))
for a, inference_method in zip(ax, ['ad3', 'qpbo', 'max-product',
('max-product', {'max_iter': 10}), 'lp']):
start = time()
y = inference_dispatch(unaries, pairwise, edges,
inference_method=inference_method)
took = time() - start
a.matshow(y.reshape(size, size))
energy = compute_energy(unaries, pairwise, edges, y)
a.set_title(str(inference_method) + "\n time: %.2f energy %.2f" % (took, energy))
a.set_xticks(())
a.set_yticks(())
plt.show()
