def staged_latent_objective(self, X, Y):
for i in xrange(self.iter_done):
w = self.w_history_[i]
Y = Parallel(n_jobs=self.n_jobs, verbose=0, max_nbytes=1e8)(
delayed(latent)(self.model, x, y, w) for x, y in zip(X, Y))
yield objective_primal(self.model, w, X, Y, self.C,
'one_slack', self.njobs)
def test_ssvm_objectives():
# test that the algorithms provide consistent objective curves.
# this is not that strong a test now but at least makes sure that
# the objective function is called.
X, Y = generate_blocks_multinomial(n_samples=10, noise=1.5, seed=0)
n_labels = len(np.unique(Y))
crf = GridCRF(n_states=n_labels, inference_method=inference_method)
# once for n-slack
clf = NSlackSSVM(model=crf, max_iter=5, C=1, tol=.1)
clf.fit(X, Y)
primal_objective = objective_primal(clf.model, clf.w, X, Y, clf.C)
assert_almost_equal(clf.primal_objective_curve_[-1], primal_objective)
# once for one-slack
clf = OneSlackSSVM(model=crf, max_iter=5, C=1, tol=.1)
clf.fit(X, Y)
primal_objective = objective_primal(clf.model, clf.w, X, Y, clf.C,
variant='one_slack')
assert_almost_equal(clf.primal_objective_curve_[-1], primal_objective)
# now subgradient. Should also work in batch-mode.
clf = SubgradientSSVM(model=crf, max_iter=5, C=1, batch_size=-1)
clf.fit(X, Y)
primal_objective = objective_primal(clf.model, clf.w, X, Y, clf.C)
assert_almost_equal(clf.objective_curve_[-1], primal_objective)
# frank wolfe
clf = FrankWolfeSSVM(model=crf, max_iter=5, C=1, batch_mode=True)
clf.fit(X, Y)
primal_objective = objective_primal(clf.model, clf.w, X, Y, clf.C)
assert_almost_equal(clf.primal_objective_curve_[-1], primal_objective)
# block-coordinate Frank-Wolfe
clf = FrankWolfeSSVM(model=crf, max_iter=5, C=1, batch_mode=False)
clf.fit(X, Y)
primal_objective = objective_primal(clf.model, clf.w, X, Y, clf.C)
assert_almost_equal(clf.primal_objective_curve_[-1], primal_objective)
def fit(self,
X,
Y,
initialize=True,
continued=False,
warm_start=False,
save_inner_w=False):
"""Learn parameters using the concave-convex procedure.
Parameters
----------
X : iterable
Traing instances. Contains the structured input objects.
No requirement on the particular form of entries of X is made.
Y : iterable
Training labels. Contains the strctured labels (full and weak)
for inputs in X. Needs to have the same length as X.
initialize : boolean
If True initialize w by running SSVM on full-labeled examples.
Otherwise initialize w by running SSVM on full-labeled and
randomly initialized weak-labeled examples.
continued : boolean
If True than it is assumed that every internal model data are set up.
And we continue learning. It may be used to perform additional iterations
without restarting the method.
"""
self.save_inner_w = save_inner_w
if not continued:
w = np.zeros(self.model.size_joint_feature)
start_time = time()
self.w_history_ = []
self.number_of_iterations_ = []
self.number_of_changes_ = []
self.timestamps_ = []
self.qp_time_ = []
self.inference_time_ = []
self.number_of_constraints_ = []
self.objective_curve_ = []
self.primal_objective_curve_ = []
self.inference_calls_ = []
self.latent_objective_ = []
if self.save_inner_w:
self.inner_w = []
self.inner_sz = []
self.inner_objective = []
self.inner_primal = []
self.inner_staged_inference = []
self.inner_timestamps = []
# all data is fully labeled, quit
# fixme: it should not work!
if np.all([y.full_labeled for y in Y]):
self.base_ssvm.fit(X, Y)
self.w_history_ = np.array([self.base_ssvm.w])
self.number_of_iterations_ = np.array(
[len(self.base_ssvm.primal_objective_curve_)])
self.number_of_changes_ = np.array([])
self.timestamps_ = np.array([time() - start_time])
self.qp_time_ = np.array([self.base_ssvm.qp_time])
self.inference_time_ = np.array(
[self.base_ssvm.inference_time])
self.number_of_constraints_ = np.array(
[len(self.base_ssvm.constraints_)])
self.objective_curve_ = np.array(
[self.base_ssvm.objective_curve_[-1]])
self.primal_objective_curve_ = np.array(
[self.base_ssvm.primal_objective_curve_[-1]])
self.iter_done = 1
return
X1, Y1 = [], []
if initialize:
for x, y in zip(X, Y):
if y.full_labeled:
X1.append(x)
Y1.append(y)
else:
# we have some fully labeled examples, others are somehow initialized
X1, Y1 = X, Y
# now it's a dirty hack
# we'd like to find a good initialstarting point, let ssvm converge
old_max_iter = None
if warm_start:
old_max_iter = self.base_ssvm.max_iter
self.base_ssvm.max_iter = 10000
self.base_ssvm.fit(X1, Y1, save_history=self.save_inner_w)
if warm_start:
self.base_ssvm.max_iter = old_max_iter
w = self.base_ssvm.w
self.w_history_.append(w)
self.number_of_iterations_.append(self.base_ssvm.iterations_done)
self.timestamps_.append(time() - start_time)
self.qp_time_.append(self.base_ssvm.qp_time)
self.inference_time_.append(self.base_ssvm.inference_time)
self.number_of_constraints_.append(len(
self.base_ssvm.constraints_))
self.objective_curve_.append(self.base_ssvm.objective_curve_[-1])
self.primal_objective_curve_.append(
self.base_ssvm.primal_objective_curve_[-1])
self.inference_calls_.append(self.base_ssvm.inference_calls)
if self.save_inner_w:
self.inner_w.append(self.base_ssvm.w_history)
self.inner_sz.append(len(self.base_ssvm.objective_curve_))
self.inner_objective += self.base_ssvm.objective_curve_
self.inner_primal += self.base_ssvm.primal_objective_curve_
self.inner_staged_inference += self.base_ssvm.staged_inference_calls
self.inner_timestamps += self.base_ssvm.timestamps_
gap = self.primal_objective_curve_[-1] - self.objective_curve_[-1]
if self.verbose:
print("Final primal objective: %f" %
self.primal_objective_curve_[-1])
print("Final cutting-plane objective: %f" %
self.objective_curve_[-1])
print("Duality gap: %f" % gap)
print("Finished in %d iterations" %
self.number_of_iterations_[-1])
print("Time elapsed: %f s" % (self.timestamps_[-1]))
print("Time spent by QP: %f s" % self.base_ssvm.qp_time)
print("Time spent by inference: %f s" %
self.base_ssvm.inference_time)
print("Number of constraints: %d" %
self.number_of_constraints_[-1])
print("----------------------------------------")
begin = 0
else:
begin = self.iter_done
w = self.w_history_[-1]
start_time = time()
# reset strong labels
# from simple_dataset import Label
# Y_new = [Label(None, y.weak, False) for y in Y]
# Y = Y_new
too_small_changes = False
try:
for iteration in xrange(begin, self.latent_iter):
if self.verbose:
print("LATENT SVM ITERATION %d" % iteration)
# complete latent variables
Y_new = Parallel(n_jobs=self.n_jobs, verbose=0,
max_nbytes=1e8)(
delayed(latent)(self.model, x, y, w)
for x, y in zip(X, Y))
changes = [
np.any(y_new.full != y.full) for y_new, y in zip(Y_new, Y)
]
if np.sum(changes) <= self.min_changes:
if self.verbose:
print(
"too few changes in latent variables of ground truth."
" stopping.")
too_small_changes = True
if self.verbose:
print("changes in H: %d" % np.sum(changes))
self.number_of_changes_.append(np.sum(changes))
Y = Y_new
latent_objective = objective_primal(self.model, w, X, Y,
self.C, 'one_slack',
self.n_jobs)
self.latent_objective_.append(latent_objective)
if self.verbose:
print("Previous Latent SSVM objective: %f" %
latent_objective)
if too_small_changes:
break
if not warm_start:
self.base_ssvm.fit(X,
Y,
warm_start=False,
initialize=False,
save_history=self.save_inner_w)
else:
if iteration > 0:
self.base_ssvm.fit(X,
Y,
warm_start=warm_start,
initialize=False,
save_history=self.save_inner_w)
else:
self.base_ssvm.fit(X,
Y,
warm_start=False,
initialize=False,
save_history=self.save_inner_w)
w = self.base_ssvm.w
self.w_history_.append(w)
self.number_of_iterations_.append(
self.base_ssvm.iterations_done)
self.timestamps_.append(time() - start_time)
self.qp_time_.append(self.base_ssvm.qp_time)
self.inference_time_.append(self.base_ssvm.inference_time)
self.number_of_constraints_.append(
len(self.base_ssvm.constraints_))
self.objective_curve_.append(
self.base_ssvm.objective_curve_[-1])
self.primal_objective_curve_.append(
self.base_ssvm.primal_objective_curve_[-1])
self.inference_calls_.append(self.base_ssvm.inference_calls)
if self.save_inner_w:
self.inner_w.append(self.base_ssvm.w_history)
self.inner_sz.append(len(self.base_ssvm.objective_curve_))
self.inner_objective += self.base_ssvm.objective_curve_
self.inner_primal += self.base_ssvm.primal_objective_curve_
self.inner_staged_inference += self.base_ssvm.staged_inference_calls
self.inner_timestamps += self.base_ssvm.timestamps_
gap = self.primal_objective_curve_[-1] - self.objective_curve_[
-1]
delta = np.linalg.norm(self.w_history_[-1] -
self.w_history_[-2])
q_delta = np.abs(self.primal_objective_curve_[-1] -
self.primal_objective_curve_[-2])
if self.verbose:
print("|w-w_prev|: %f" % delta)
print("|Q-Q_prev|: %f" % q_delta)
print("Final primal objective: %f" %
self.primal_objective_curve_[-1])
print("Final cutting-plane objective: %f" %
self.objective_curve_[-1])
print("Duality gap: %f" % gap)
print("Finished in %d iterations" %
self.number_of_iterations_[-1])
print("Time elapsed: %f s" %
(self.timestamps_[-1] - self.timestamps_[-2]))
print("Time spent by QP: %f s" % self.base_ssvm.qp_time)
print("Time spent by inference: %f s" %
self.base_ssvm.inference_time)
print("Number of constraints: %d" %
self.number_of_constraints_[-1])
print("----------------------------------------")
if q_delta < self.tol:
if self.verbose:
print("objective value did not change a lot, break")
break
except KeyboardInterrupt:
if self.verbose:
print('interrupted... finishing...')
pass
# some copy paste
if not too_small_changes:
Y_new = Parallel(n_jobs=self.n_jobs, verbose=0, max_nbytes=1e8)(
delayed(latent)(self.model, x, y, w) for x, y in zip(X, Y))
latent_objective = objective_primal(self.model, w, X, Y_new,
self.C, 'one_slack',
self.n_jobs)
changes = [
np.any(y_new.full != y.full) for y_new, y in zip(Y_new, Y)
]
if self.verbose:
print("changes in H: %d" % np.sum(changes))
self.number_of_changes_.append(np.sum(changes))
self.latent_objective_.append(latent_objective)
if self.verbose:
print("Previous Latent SSVM objective: %f" % latent_objective)
#
self.number_of_changes_ = np.array(self.number_of_changes_)
self.w_history_ = np.array(self.w_history_)
self.number_of_iterations_ = np.array(self.number_of_iterations_)
self.timestamps_ = np.array(self.timestamps_)
self.qp_time_ = np.array(self.qp_time_)
self.inference_time_ = np.array(self.inference_time_)
self.number_of_constraints_ = np.array(self.number_of_constraints_)
self.objective_curve_ = np.array(self.objective_curve_)
self.primal_objective_curve_ = np.array(self.primal_objective_curve_)
self.inference_calls_ = np.array(self.inference_calls_)
self.latent_objective_ = np.array(self.latent_objective_)
self.iter_done = self.w_history_.shape[0]
if self.save_inner_w:
self.inner_w = np.vstack(self.inner_w)
self.inner_sz = np.array(self.inner_sz)
self.inner_primal = np.array(self.inner_primal)
self.inner_objective = np.array(self.inner_objective)
self.inner_staged_inference = np.array(self.inner_staged_inference)
self.inner_timestamps = np.array(self.inner_timestamps)
def fit(self, X, Y, initialize=True,
continued=False, warm_start=False,
save_inner_w=False):
"""Learn parameters using the concave-convex procedure.
Parameters
----------
X : iterable
Traing instances. Contains the structured input objects.
No requirement on the particular form of entries of X is made.
Y : iterable
Training labels. Contains the strctured labels (full and weak)
for inputs in X. Needs to have the same length as X.
initialize : boolean
If True initialize w by running SSVM on full-labeled examples.
Otherwise initialize w by running SSVM on full-labeled and
randomly initialized weak-labeled examples.
continued : boolean
If True than it is assumed that every internal model data are set up.
And we continue learning. It may be used to perform additional iterations
without restarting the method.
"""
self.save_inner_w = save_inner_w
if not continued:
w = np.zeros(self.model.size_joint_feature)
start_time = time()
self.w_history_ = []
self.number_of_iterations_ = []
self.number_of_changes_ = []
self.timestamps_ = []
self.qp_time_ = []
self.inference_time_ = []
self.number_of_constraints_ = []
self.objective_curve_ = []
self.primal_objective_curve_ = []
self.inference_calls_ = []
self.latent_objective_ = []
if self.save_inner_w:
self.inner_w = []
self.inner_sz = []
self.inner_objective = []
self.inner_primal = []
self.inner_staged_inference = []
self.inner_timestamps = []
# all data is fully labeled, quit
# fixme: it should not work!
if np.all([y.full_labeled for y in Y]):
self.base_ssvm.fit(X, Y)
self.w_history_ = np.array([self.base_ssvm.w])
self.number_of_iterations_ = np.array([len(self.base_ssvm.primal_objective_curve_)])
self.number_of_changes_ = np.array([])
self.timestamps_ = np.array([time() - start_time])
self.qp_time_ = np.array([self.base_ssvm.qp_time])
self.inference_time_ = np.array([self.base_ssvm.inference_time])
self.number_of_constraints_ = np.array([len(self.base_ssvm.constraints_)])
self.objective_curve_ = np.array([self.base_ssvm.objective_curve_[-1]])
self.primal_objective_curve_ = np.array([self.base_ssvm.primal_objective_curve_[-1]])
self.iter_done = 1
return
X1, Y1 = [], []
if initialize:
for x, y in zip(X, Y):
if y.full_labeled:
X1.append(x)
Y1.append(y)
else:
# we have some fully labeled examples, others are somehow initialized
X1, Y1 = X, Y
# now it's a dirty hack
# we'd like to find a good initialstarting point, let ssvm converge
old_max_iter = None
if warm_start:
old_max_iter = self.base_ssvm.max_iter
self.base_ssvm.max_iter = 10000
self.base_ssvm.fit(X1, Y1, save_history=self.save_inner_w)
if warm_start:
self.base_ssvm.max_iter = old_max_iter
w = self.base_ssvm.w
self.w_history_.append(w)
self.number_of_iterations_.append(self.base_ssvm.iterations_done)
self.timestamps_.append(time() - start_time)
self.qp_time_.append(self.base_ssvm.qp_time)
self.inference_time_.append(self.base_ssvm.inference_time)
self.number_of_constraints_.append(len(self.base_ssvm.constraints_))
self.objective_curve_.append(self.base_ssvm.objective_curve_[-1])
self.primal_objective_curve_.append(self.base_ssvm.primal_objective_curve_[-1])
self.inference_calls_.append(self.base_ssvm.inference_calls)
if self.save_inner_w:
self.inner_w.append(self.base_ssvm.w_history)
self.inner_sz.append(len(self.base_ssvm.objective_curve_))
self.inner_objective += self.base_ssvm.objective_curve_
self.inner_primal += self.base_ssvm.primal_objective_curve_
self.inner_staged_inference += self.base_ssvm.staged_inference_calls
self.inner_timestamps += self.base_ssvm.timestamps_
gap = self.primal_objective_curve_[-1] - self.objective_curve_[-1]
if self.verbose:
print("Final primal objective: %f" % self.primal_objective_curve_[-1])
print("Final cutting-plane objective: %f" % self.objective_curve_[-1])
print("Duality gap: %f" % gap)
print("Finished in %d iterations" % self.number_of_iterations_[-1])
print("Time elapsed: %f s" % (self.timestamps_[-1]))
print("Time spent by QP: %f s" % self.base_ssvm.qp_time)
print("Time spent by inference: %f s" % self.base_ssvm.inference_time)
print("Number of constraints: %d" % self.number_of_constraints_[-1])
print("----------------------------------------")
begin = 0
else:
begin = self.iter_done
w = self.w_history_[-1]
start_time = time()
# reset strong labels
# from simple_dataset import Label
# Y_new = [Label(None, y.weak, False) for y in Y]
# Y = Y_new
too_small_changes = False
try:
for iteration in xrange(begin, self.latent_iter):
if self.verbose:
print("LATENT SVM ITERATION %d" % iteration)
# complete latent variables
Y_new = Parallel(n_jobs=self.n_jobs, verbose=0, max_nbytes=1e8)(
delayed(latent)(self.model, x, y, w) for x, y in zip(X, Y))
changes = [np.any(y_new.full != y.full) for y_new, y in zip(Y_new, Y)]
if np.sum(changes) <= self.min_changes:
if self.verbose:
print("too few changes in latent variables of ground truth."
" stopping.")
too_small_changes = True
if self.verbose:
print("changes in H: %d" % np.sum(changes))
self.number_of_changes_.append(np.sum(changes))
Y = Y_new
latent_objective = objective_primal(self.model, w,
X, Y, self.C, 'one_slack', self.n_jobs)
self.latent_objective_.append(latent_objective)
if self.verbose:
print("Previous Latent SSVM objective: %f" % latent_objective)
if too_small_changes:
break
if not warm_start:
self.base_ssvm.fit(X, Y, warm_start=False,
initialize=False, save_history=self.save_inner_w)
else:
if iteration > 0:
self.base_ssvm.fit(X, Y, warm_start=warm_start,
initialize=False, save_history=self.save_inner_w)
else:
self.base_ssvm.fit(X, Y, warm_start=False,
initialize=False, save_history=self.save_inner_w)
w = self.base_ssvm.w
self.w_history_.append(w)
self.number_of_iterations_.append(self.base_ssvm.iterations_done)
self.timestamps_.append(time() - start_time)
self.qp_time_.append(self.base_ssvm.qp_time)
self.inference_time_.append(self.base_ssvm.inference_time)
self.number_of_constraints_.append(len(self.base_ssvm.constraints_))
self.objective_curve_.append(self.base_ssvm.objective_curve_[-1])
self.primal_objective_curve_.append(self.base_ssvm.primal_objective_curve_[-1])
self.inference_calls_.append(self.base_ssvm.inference_calls)
if self.save_inner_w:
self.inner_w.append(self.base_ssvm.w_history)
self.inner_sz.append(len(self.base_ssvm.objective_curve_))
self.inner_objective += self.base_ssvm.objective_curve_
self.inner_primal += self.base_ssvm.primal_objective_curve_
self.inner_staged_inference += self.base_ssvm.staged_inference_calls
self.inner_timestamps += self.base_ssvm.timestamps_
gap = self.primal_objective_curve_[-1] - self.objective_curve_[-1]
delta = np.linalg.norm(self.w_history_[-1] - self.w_history_[-2])
q_delta = np.abs(self.primal_objective_curve_[-1] - self.primal_objective_curve_[-2])
if self.verbose:
print("|w-w_prev|: %f" % delta)
print("|Q-Q_prev|: %f" % q_delta)
print("Final primal objective: %f" % self.primal_objective_curve_[-1])
print("Final cutting-plane objective: %f" % self.objective_curve_[-1])
print("Duality gap: %f" % gap)
print("Finished in %d iterations" % self.number_of_iterations_[-1])
print("Time elapsed: %f s" % (self.timestamps_[-1] - self.timestamps_[-2]))
print("Time spent by QP: %f s" % self.base_ssvm.qp_time)
print("Time spent by inference: %f s" % self.base_ssvm.inference_time)
print("Number of constraints: %d" % self.number_of_constraints_[-1])
print("----------------------------------------")
if q_delta < self.tol:
if self.verbose:
print("objective value did not change a lot, break")
break
except KeyboardInterrupt:
if self.verbose:
print('interrupted... finishing...')
pass
# some copy paste
if not too_small_changes:
Y_new = Parallel(n_jobs=self.n_jobs, verbose=0, max_nbytes=1e8)(
delayed(latent)(self.model, x, y, w) for x, y in zip(X, Y))
latent_objective = objective_primal(self.model, w, X, Y_new, self.C,
'one_slack', self.n_jobs)
changes = [np.any(y_new.full != y.full) for y_new, y in zip(Y_new, Y)]
if self.verbose:
print("changes in H: %d" % np.sum(changes))
self.number_of_changes_.append(np.sum(changes))
self.latent_objective_.append(latent_objective)
if self.verbose:
print("Previous Latent SSVM objective: %f" % latent_objective)
#
self.number_of_changes_ = np.array(self.number_of_changes_)
self.w_history_ = np.array(self.w_history_)
self.number_of_iterations_ = np.array(self.number_of_iterations_)
self.timestamps_ = np.array(self.timestamps_)
self.qp_time_ = np.array(self.qp_time_)
self.inference_time_ = np.array(self.inference_time_)
self.number_of_constraints_ = np.array(self.number_of_constraints_)
self.objective_curve_ = np.array(self.objective_curve_)
self.primal_objective_curve_ = np.array(self.primal_objective_curve_)
self.inference_calls_ = np.array(self.inference_calls_)
self.latent_objective_ = np.array(self.latent_objective_)
self.iter_done = self.w_history_.shape[0]
if self.save_inner_w:
self.inner_w = np.vstack(self.inner_w)
self.inner_sz = np.array(self.inner_sz)
self.inner_primal = np.array(self.inner_primal)
self.inner_objective = np.array(self.inner_objective)
self.inner_staged_inference = np.array(self.inner_staged_inference)
self.inner_timestamps = np.array(self.inner_timestamps)
