def do_theano(self):
""" Compiles all theano functions needed to use the model"""
init_names = dir(self)
###### All fields you don't want to get pickled (e.g., theano functions) should be created below this line
self.init_debug()
# POSITIVE PHASE
pos_h = self.h_given_v(self.input)
self.inference_func = theano.function([self.input], [pos_h])
# SAMPLING: NEGATIVE PHASE
neg_updates = self.neg_sampling_updates(n_steps=self.neg_sample_steps,
use_pcd=True)
self.sample_func = theano.function([], [], updates=neg_updates)
##
# BUILD COST OBJECTS
##
lcost = self.ml_cost(pos_v=self.input, neg_v=neg_updates[self.neg_v])
regcost = self.get_reg_cost(self.l2, self.l1)
##
# COMPUTE GRADIENTS WRT. COSTS
##
main_cost = [lcost, regcost]
learning_grads = costmod.compute_gradients(self.lr, self.lr_mults,
*main_cost)
##
# BUILD UPDATES DICTIONARY FROM GRADIENTS
##
learning_updates = costmod.get_updates(learning_grads)
learning_updates.update(neg_updates)
learning_updates.update({self.iter: self.iter + 1})
# build theano function to train on a single minibatch
self.batch_train_func = function([self.input], [],
updates=learning_updates,
name='train_rbm_func')
#theano.printing.pydotprint(self.batch_train_func, outfile='batch_train_func.png', scan_graphs=True);
#######################
# CONSTRAINT FUNCTION #
#######################
constraint_updates = self.get_constraint_updates()
self.enforce_constraints = theano.function([], [],
updates=constraint_updates)
###### All fields you don't want to get pickled should be created above this line
final_names = dir(self)
self.register_names_to_del(
[name for name in (final_names) if name not in init_names])
# Before we start learning, make sure constraints are enforced
self.enforce_constraints()
def do_theano(self):
""" Compiles all theano functions needed to use the model"""
init_names = dir(self)
###### All fields you don't want to get pickled (e.g., theano functions) should be created below this line
self.init_debug()
# POSITIVE PHASE
pos_h = self.h_given_v(self.input)
self.inference_func = theano.function([self.input], [pos_h])
# SAMPLING: NEGATIVE PHASE
neg_updates = self.neg_sampling_updates(n_steps=self.neg_sample_steps, use_pcd=True)
self.sample_func = theano.function([], [], updates=neg_updates)
##
# BUILD COST OBJECTS
##
lcost = self.ml_cost(
pos_v = self.input,
neg_v = neg_updates[self.neg_v])
regcost = self.get_reg_cost(self.l2, self.l1)
##
# COMPUTE GRADIENTS WRT. COSTS
##
main_cost = [lcost, regcost]
learning_grads = costmod.compute_gradients(self.lr, self.lr_mults, *main_cost)
##
# BUILD UPDATES DICTIONARY FROM GRADIENTS
##
learning_updates = costmod.get_updates(learning_grads)
learning_updates.update(neg_updates)
learning_updates.update({self.iter: self.iter+1})
# build theano function to train on a single minibatch
self.batch_train_func = function([self.input], [],
updates=learning_updates,
name='train_rbm_func')
#theano.printing.pydotprint(self.batch_train_func, outfile='batch_train_func.png', scan_graphs=True);
#######################
# CONSTRAINT FUNCTION #
#######################
constraint_updates = self.get_constraint_updates()
self.enforce_constraints = theano.function([],[], updates=constraint_updates)
###### All fields you don't want to get pickled should be created above this line
final_names = dir(self)
self.register_names_to_del( [ name for name in (final_names) if name not in init_names ])
# Before we start learning, make sure constraints are enforced
self.enforce_constraints()
def do_theano(self):
""" Compiles all theano functions needed to use the model"""
init_names = dir(self)
###### All fields you don't want to get pickled (e.g., theano functions) should be created below this line
self.init_debug()
# STANDARDIZATION OF S
stand_s_updates = OrderedDict()
new_gamma = self.gamma * 0.99 + self.pos_s.std(axis=0) * 0.01
stand_s_updates[self.gamma] = new_gamma
stand_s_updates[self.Wv] = new_gamma / self.gamma * self.Wv
stand_s_updates[self.mu] = self.gamma / new_gamma * self.mu
self.standardize_s = theano.function([], [], updates=stand_s_updates)
# SAMPLING: NEGATIVE PHASE
neg_updates = self.neg_sampling_updates(n_steps=self.neg_sample_steps, use_pcd=True)
self.sample_func = theano.function([], [], updates=neg_updates)
# POSITIVE PHASE
pos_updates = self.pos_phase_updates(
self.input,
n_steps = self.pos_steps)
self.inference_func = theano.function([self.input], [],
updates=pos_updates)
##
# BUILD COST OBJECTS
##
lcost = self.ml_cost(
pos_g = self.pos_g,
pos_h = self.pos_h,
pos_s1 = self.pos_s1,
pos_s0 = self.pos_s0,
pos_v = self.input,
neg_g = neg_updates[self.neg_g],
neg_h = neg_updates[self.neg_h],
neg_s = neg_updates[self.neg_s],
neg_v = neg_updates[self.neg_v])
#spcost = self.get_sparsity_cost(
#pos_g = pos_updates[self.pos_g],
#pos_h = pos_updates[self.pos_h])
regcost = self.get_reg_cost(self.l2, self.l1)
##
# COMPUTE GRADIENTS WRT. COSTS
##
#main_cost = [lcost, spcost, regcost]
main_cost = [lcost, regcost]
learning_grads = costmod.compute_gradients(self.lr, self.lr_mults, *main_cost)
weight_updates = OrderedDict()
if self.flags['wv_true_gradient']:
weight_updates[self.Wv] = true_gradient(self.Wv, -learning_grads[self.Wv])
##
# BUILD UPDATES DICTIONARY FROM GRADIENTS
##
learning_updates = costmod.get_updates(learning_grads)
learning_updates.update(neg_updates)
learning_updates.update({self.iter: self.iter+1})
learning_updates.update(weight_updates)
# build theano function to train on a single minibatch
self.batch_train_func = function([self.input], [],
updates=learning_updates,
name='train_rbm_func')
#theano.printing.pydotprint(self.batch_train_func, outfile='batch_train_func.png', scan_graphs=True);
#######################
# CONSTRAINT FUNCTION #
#######################
constraint_updates = self.get_constraint_updates()
self.enforce_constraints = theano.function([],[], updates=constraint_updates)
###### All fields you don't want to get pickled should be created above this line
final_names = dir(self)
self.register_names_to_del( [ name for name in (final_names) if name not in init_names ])
# Before we start learning, make sure constraints are enforced
self.enforce_constraints()
def do_theano(self):
""" Compiles all theano functions needed to use the model"""
init_names = dir(self)
###### All fields you don't want to get pickled (e.g., theano functions) should be created below this line
# SAMPLING: NEGATIVE PHASE
neg_updates = self.neg_sampling_updates(n_steps=self.neg_sample_steps, use_pcd=True)
self.sample_func = theano.function([], [], updates=neg_updates)
# VARIATIONAL E-STEP
pos_updates = OrderedDict()
if self.pos_mf_steps:
pos_states, mf_updates = self.pos_phase_updates(
self.input,
mean_field = True,
n_steps = self.pos_mf_steps)
pos_updates.update(mf_updates)
# SAMPLING: POSITIVE PHASE
if self.pos_sample_steps:
init_state = pos_states if self.pos_mf_steps else None
pos_states, sample_updates = self.pos_phase_updates(
self.input,
init_state = init_state,
mean_field = False,
n_steps = self.pos_sample_steps)
pos_updates.update(sample_updates)
##
# BUILD COST OBJECTS
##
lcost = self.ml_cost(
pos_g = pos_states['g'],
pos_s = pos_states['s'],
pos_h = pos_states['h'],
pos_t = pos_states['t'],
pos_v = self.input,
neg_g = neg_updates[self.neg_g],
neg_s = neg_updates[self.neg_s],
neg_h = neg_updates[self.neg_h],
neg_t = neg_updates[self.neg_t],
neg_v = neg_updates[self.neg_v])
spcost = self.get_sparsity_cost(
pos_states['g'], pos_states['s'],
pos_states['h'], pos_states['t'])
regcost = self.get_reg_cost(self.l2, self.l1)
##
# COMPUTE GRADIENTS WRT. COSTS
##
main_cost = [lcost, spcost, regcost]
learning_grads = costmod.compute_gradients(self.lr, self.lr_mults, *main_cost)
weight_updates = OrderedDict()
weight_updates[self.Wv] = true_gradient(self.Wv, -learning_grads[self.Wv])
if self.Wg in self.params():
weight_updates[self.Wg] = true_gradient(self.Wg, -learning_grads[self.Wg])
if self.Wh in self.params():
weight_updates[self.Wh] = true_gradient(self.Wh, -learning_grads[self.Wh])
##
# BUILD UPDATES DICTIONARY FROM GRADIENTS
##
learning_updates = costmod.get_updates(learning_grads)
learning_updates.update(pos_updates)
learning_updates.update(neg_updates)
learning_updates.update({self.iter: self.iter+1})
learning_updates.update(weight_updates)
# build theano function to train on a single minibatch
self.batch_train_func = function([self.input], [],
updates=learning_updates,
name='train_rbm_func')
self.energy_fn = function([], self.energy(self.neg_g, self.neg_s, self.neg_h,
self.neg_t, self.neg_v))
self.g_fn = function([], self.g_given_htv(self.neg_h, self.neg_t, self.neg_v))
self.h_fn = function([], self.h_given_gsv(self.neg_g, self.neg_s, self.neg_v))
self.s_fn = function([], self.s_given_ghtv(self.neg_g, self.neg_h, self.neg_t, self.neg_v))
self.t_fn = function([], self.t_given_gshv(self.neg_g, self.neg_s, self.neg_h, self.neg_v))
self.v_fn = function([], self.v_given_gsht(self.neg_g, self.neg_s, self.neg_h, self.neg_t))
self.sample_g_fn = function([], self.sample_g_given_htv(self.neg_h, self.neg_t, self.neg_v))
self.sample_h_fn = function([], self.sample_h_given_gsv(self.neg_g, self.neg_s, self.neg_v))
self.sample_s_fn = function([], self.sample_s_given_ghtv(self.neg_g, self.neg_h, self.neg_t, self.neg_v))
self.sample_t_fn = function([], self.sample_t_given_gshv(self.neg_g, self.neg_s, self.neg_h, self.neg_v))
self.sample_v_fn = function([], self.sample_v_given_gsht(self.neg_g, self.neg_s, self.neg_h, self.neg_t))
#######################
# CONSTRAINT FUNCTION #
#######################
# enforce constraints function
constraint_updates = OrderedDict()
constraint_updates[self.lambd] = T.mean(self.lambd) * T.ones_like(self.lambd)
## clip parameters to maximum values (if applicable)
for (k,v) in self.clip_max.iteritems():
assert k in [param.name for param in self.params()]
param = getattr(self, k)
constraint_updates[param] = T.clip(param, param, v)
## clip parameters to minimum values (if applicable)
for (k,v) in self.clip_min.iteritems():
assert k in [param.name for param in self.params()]
param = getattr(self, k)
constraint_updates[param] = T.clip(constraint_updates.get(param, param), v, param)
self.enforce_constraints = theano.function([],[], updates=constraint_updates)
###### All fields you don't want to get pickled should be created above this line
final_names = dir(self)
self.register_names_to_del( [ name for name in (final_names) if name not in init_names ])
# Before we start learning, make sure constraints are enforced
self.enforce_constraints()
def do_theano(self):
""" Compiles all theano functions needed to use the model"""
init_names = dir(self)
###### All fields you don't want to get pickled (e.g., theano functions) should be created below this line
# build energy function
_gtemp = T.matrix('_gtemp')
_htemp = T.matrix('_htemp')
_vtemp = T.matrix('_vtemp')
energy, cte = self.free_energy(_gtemp, _htemp, _vtemp, mean=False)
self.energy_func = theano.function([_gtemp, _htemp, _vtemp], energy)
# SAMPLING: NEGATIVE PHASE
neg_updates = self.neg_sampling_updates(n_steps=self.neg_sample_steps, use_pcd=True)
self.sample_func = theano.function([], [], updates=neg_updates)
# POSITIVE PHASE
pos_states, pos_updates = self.pos_phase_updates(
self.input,
n_steps = self.pos_steps,
mean_field=self.flags['mean_field'])
self.inference_func = theano.function([self.input], [pos_states['g'], pos_states['h']])
##
# BUILD COST OBJECTS
##
lcost = self.ml_cost(
pos_g = pos_states['g'],
pos_h = pos_states['h'],
pos_v = self.input,
neg_g = neg_updates[self.neg_g],
neg_h = neg_updates[self.neg_h],
neg_v = neg_updates[self.neg_v],
mean_field=self.flags['mean_field'])
spcost = self.get_sparsity_cost(pos_states['g'], pos_states['h'])
regcost = self.get_reg_cost(self.l2, self.l1)
##
# COMPUTE GRADIENTS WRT. COSTS
##
main_cost = [lcost, spcost, regcost]
learning_grads = costmod.compute_gradients(self.lr, self.lr_mults, *main_cost)
weight_updates = OrderedDict()
if self.flags['wg_norm'] == 'unit' and self.Wg in self.params():
weight_updates[self.Wg] = true_gradient(self.Wg, -learning_grads[self.Wg])
if self.flags['wh_norm'] == 'unit' and self.Wh in self.params():
weight_updates[self.Wh] = true_gradient(self.Wh, -learning_grads[self.Wh])
if self.flags['wv_norm'] == 'unit':
weight_updates[self.Wv] = true_gradient(self.Wv, -learning_grads[self.Wv])
##
# BUILD UPDATES DICTIONARY FROM GRADIENTS
##
learning_updates = costmod.get_updates(learning_grads)
learning_updates.update(pos_updates)
learning_updates.update(neg_updates)
learning_updates.update({self.iter: self.iter+1})
learning_updates.update(weight_updates)
# build theano function to train on a single minibatch
self.batch_train_func = function([self.input], [],
updates=learning_updates,
name='train_rbm_func')
#theano.printing.pydotprint(self.batch_train_func, outfile='batch_train_func.png', scan_graphs=True);
#######################
# CONSTRAINT FUNCTION #
#######################
constraint_updates = self.get_constraint_updates()
self.enforce_constraints = theano.function([],[], updates=constraint_updates)
###### All fields you don't want to get pickled should be created above this line
final_names = dir(self)
self.register_names_to_del( [ name for name in (final_names) if name not in init_names ])
# Before we start learning, make sure constraints are enforced
self.enforce_constraints()
def do_theano(self):
""" Compiles all theano functions needed to use the model"""
init_names = dir(self)
###### All fields you don't want to get pickled (e.g., theano functions) should be created below this line
# SAMPLING: NEGATIVE PHASE
neg_updates = self.neg_sampling_updates(n_steps=self.neg_sample_steps, use_pcd=True)
self.sample_func = theano.function([], [], updates=neg_updates)
### BUILD TWO FUNCTIONS: ONE CONDITIONED ON LABELS, ONE WHERE LABELS ARE SAMPLED
self.batch_train_func = {}
for (key, input_label) in zip(['nolabel','label'], [None, self.input_labels]):
# POSITIVE PHASE
pos_states, pos_updates = self.pos_phase_updates(
self.input, l = input_label,
n_steps = self.pos_steps,
mean_field=self.flags['mean_field'])
##
# BUILD COST OBJECTS
##
lcost = self.ml_cost(
pos_g = pos_states['g'],
pos_h = pos_states['h'],
pos_l = pos_states['l'],
pos_v = self.input,
neg_g = neg_updates[self.neg_g],
neg_h = neg_updates[self.neg_h],
neg_v = neg_updates[self.neg_v],
neg_l = neg_updates[self.neg_l],
mean_field=self.flags['mean_field'])
spcost = self.get_sparsity_cost(pos_states['g'], pos_states['h'], pos_states['l'])
regcost = self.get_reg_cost(self.l2, self.l1)
##
# COMPUTE GRADIENTS WRT. COSTS
##
main_cost = [lcost, spcost, regcost]
learning_grads = costmod.compute_gradients(self.lr, self.lr_mults, *main_cost)
weight_updates = OrderedDict()
if self.flags['wg_norm'] == 'unit' and self.Wg in self.params():
weight_updates[self.Wg] = true_gradient(self.Wg, -learning_grads[self.Wg])
if self.flags['wh_norm'] == 'unit' and self.Wh in self.params():
weight_updates[self.Wh] = true_gradient(self.Wh, -learning_grads[self.Wh])
if self.flags['wv_norm'] == 'unit':
weight_updates[self.Wv] = true_gradient(self.Wv, -learning_grads[self.Wv])
##
# BUILD UPDATES DICTIONARY FROM GRADIENTS
##
learning_updates = costmod.get_updates(learning_grads)
learning_updates.update(pos_updates)
learning_updates.update(neg_updates)
learning_updates.update({self.iter: self.iter+1})
learning_updates.update(weight_updates)
# build theano function to train on a single minibatch
inputs = [self.input, self.input_labels] if key == 'label' else [self.input]
self.batch_train_func[key] = function(inputs, [],
updates=learning_updates,
name='train_rbm_func_%s' % key)
#######################
# CONSTRAINT FUNCTION #
#######################
constraint_updates = self.get_constraint_updates()
self.enforce_constraints = theano.function([],[], updates=constraint_updates)
###### All fields you don't want to get pickled should be created above this line
final_names = dir(self)
self.register_names_to_del( [ name for name in (final_names) if name not in init_names ])
# Before we start learning, make sure constraints are enforced
self.enforce_constraints()
def do_theano(self):
""" Compiles all theano functions needed to use the model"""
init_names = dir(self)
###### All fields you don't want to get pickled (e.g., theano functions) should be created below this line
# SAMPLING: NEGATIVE PHASE
neg_updates = self.neg_sampling_updates(n_steps=self.neg_sample_steps, use_pcd=True)
self.sample_func = theano.function([], [], updates=neg_updates)
##
# BUILD COST OBJECTS
##
lcost = self.ml_cost(pos_v = self.input, neg_v = neg_updates[self.neg_v])
spcost = self.get_sparsity_cost()
regcost = self.get_reg_cost(self.l2, self.l1)
##
# COMPUTE GRADIENTS WRT. COSTS
##
main_cost = [lcost, spcost, regcost]
learning_grads = costmod.compute_gradients(self.lr, self.lr_mults, *main_cost)
##
# BUILD UPDATES DICTIONARY FROM GRADIENTS
##
learning_updates = costmod.get_updates(learning_grads)
learning_updates.update(neg_updates)
learning_updates.update({self.iter: self.iter+1})
# build theano function to train on a single minibatch
self.batch_train_func = function([self.input], [],
updates=learning_updates,
name='train_rbm_func')
#######################
# CONSTRAINT FUNCTION #
#######################
# enforce constraints function
constraint_updates = OrderedDict()
## clip parameters to maximum values (if applicable)
for (k,v) in self.clip_max.iteritems():
assert k in [param.name for param in self.params()]
param = getattr(self, k)
constraint_updates[param] = T.clip(param, param, v)
## clip parameters to minimum values (if applicable)
for (k,v) in self.clip_min.iteritems():
assert k in [param.name for param in self.params()]
param = getattr(self, k)
constraint_updates[param] = T.clip(constraint_updates.get(param, param), v, param)
## constrain lambd to be a scalar
if self.flags['scalar_lambd']:
lambd = constraint_updates.get(self.lambd, self.lambd)
constraint_updates[self.lambd] = T.mean(lambd) * T.ones_like(lambd)
self.enforce_constraints = theano.function([],[], updates=constraint_updates)
###### All fields you don't want to get pickled should be created above this line
final_names = dir(self)
self.register_names_to_del( [ name for name in (final_names) if name not in init_names ])
# Before we start learning, make sure constraints are enforced
self.enforce_constraints()
