def __init__(self, numpy_rng, base_dnn, rho=0.5, **kwargs):
"""
Modify the objective function so it has this form:
L = (1 - rho) * L' + rho * KL(y, y')
Here L' is the original objective function, rho is the regularization
weight. The larger rho is, the more weight we place on the base model.
"""
DNN.__init__(self, numpy_rng, **kwargs)
self.base_dnn = base_dnn
self.rho = rho
self.finetune_cost = (1 - rho) * self.finetune_cost
self.kld_cost = -T.mean(T.sum(
T.log(self.logLayer.p_y_given_x) * self.base_dnn.logLayer.p_y_given_x,
axis=1
))
self.finetune_cost += rho * self.kld_cost
