def _init_exprs(self):
self.exprs = {
'inpt': T.matrix('inpt'),
}
self.exprs['inpt'].tag.test_value = np.zeros(
(10, self.n_inpt)).astype(theano.config.floatX)
P = self.parameters
n_layers = len(self.n_hiddens)
if self.tied_weights:
hidden_to_hiddens = [getattr(P, 'hidden_to_hidden_%i' % i)
for i in range(n_layers / 2)]
hidden_to_hiddens += [i.T for i in reversed(hidden_to_hiddens)]
hidden_to_out = P.in_to_hidden.T
else:
hidden_to_hiddens = [getattr(P, 'hidden_to_hidden_%i' % i)
for i in range(n_layers - 1)]
hidden_to_out = P.hidden_to_out
hidden_biases = [getattr(P, 'hidden_bias_%i' % i)
for i in range(n_layers)]
self.exprs.update(mlp.exprs(
self.exprs['inpt'],
P.in_to_hidden, hidden_to_hiddens, hidden_to_out,
hidden_biases, P.out_bias,
self.hidden_transfers, self.out_transfer))
self.exprs.update(supervised_loss(
self.exprs['inpt'], self.exprs['output'], self.loss))
self.exprs['feature'] = self.exprs['layer-%i-output' % self.code_idx]
def _init_exprs(self):
self.exprs = {
'inpt': T.matrix('inpt'),
}
P = self.parameters
n_layers = len(self.n_hiddens)
if self.tied_weights:
hidden_to_hiddens = [
getattr(P, 'hidden_to_hidden_%i' % i)
for i in range(n_layers / 2)
]
hidden_to_hiddens += [i.T for i in reversed(hidden_to_hiddens)]
hidden_to_out = P.in_to_hidden.T
else:
hidden_to_hiddens = [
getattr(P, 'hidden_to_hidden_%i' % i)
for i in range(n_layers - 1)
]
hidden_to_out = P.hidden_to_out
hidden_biases = [
getattr(P, 'hidden_bias_%i' % i) for i in range(n_layers)
]
self.exprs.update(
mlp.exprs(self.exprs['inpt'], P.in_to_hidden, hidden_to_hiddens,
hidden_to_out, hidden_biases, P.out_bias,
self.hidden_transfers, self.out_transfer))
self.exprs.update(
supervised_loss(self.exprs['inpt'], self.exprs['output'],
self.loss))
self.exprs['feature'] = self.exprs['layer-%i-output' % self.code_idx]
def _init_exprs(self):
self.exprs = {
'inpt': T.matrix('inpt'),
'target': T.matrix('target')
}
if self.imp_weight:
self.exprs['imp_weight'] = T.matrix('imp_weight')
P = self.parameters
n_layers = len(self.n_hiddens)
hidden_to_hiddens = [getattr(P, 'hidden_to_hidden_%i' % i)
for i in range(n_layers - 1)]
hidden_biases = [getattr(P, 'hidden_bias_%i' % i)
for i in range(n_layers)]
self.exprs.update(mlp.exprs(
self.exprs['inpt'],
P.in_to_hidden, hidden_to_hiddens, P.hidden_to_out,
hidden_biases, P.out_bias,
self.hidden_transfers, self.out_transfer))
imp_weight = False if not self.imp_weight else self.exprs['imp_weight']
self.exprs.update(supervised_loss(
self.exprs['target'], self.exprs['output'], self.loss,
imp_weight=imp_weight))
def _init_exprs(self):
E = self.exprs = self._make_start_exprs()
n_dim = E['inpt'].ndim
# Make the expression of the model.
E.update(sgvb.exprs(
E['inpt'],
self._recog_exprs, self._gen_exprs,
self.assumptions.sample_latents,
shortcut_key=self.shortcut))
# TODO this is not going to work with variance propagation.
imp_weight = False if not self.imp_weight else self._fix_imp_weight(n_dim)
rec_loss = supervised_loss(
E['inpt'], E['gen']['output'], self.assumptions.nll_gen_model,
prefix='rec_', coord_axis=n_dim - 1, imp_weight=imp_weight)
# Create the KL divergence part of the loss.
kl_loss = self._make_kl_loss()
E.update(rec_loss)
E.update(kl_loss)
E.update({
'loss_sample_wise': E['kl_sample_wise'] + E['rec_loss_sample_wise'],
'loss': E['kl'] + E['rec_loss'],
})
def _init_exprs(self):
self.exprs = {'inpt': T.matrix('inpt'), 'target': T.matrix('target')}
self.exprs['inpt'].tag.test_value = np.zeros(
(10, self.n_inpt)).astype(theano.config.floatX)
self.exprs['target'].tag.test_value = np.zeros(
(10, self.n_output)).astype(theano.config.floatX)
if self.imp_weight:
self.exprs['imp_weight'] = T.matrix('imp_weight')
self.exprs['imp_weight'].tag.test_value = np.zeros(
(10, self.n_output)).astype(theano.config.floatX)
P = self.parameters
n_layers = len(self.n_hiddens)
hidden_to_hiddens = [
getattr(P, 'hidden_to_hidden_%i' % i) for i in range(n_layers - 1)
]
hidden_biases = [
getattr(P, 'hidden_bias_%i' % i) for i in range(n_layers)
]
self.exprs.update(
mlp.exprs(self.exprs['inpt'], P.in_to_hidden, hidden_to_hiddens,
P.hidden_to_out, hidden_biases, P.out_bias,
self.hidden_transfers, self.out_transfer))
imp_weight = False if not self.imp_weight else self.exprs['imp_weight']
self.exprs.update(
supervised_loss(self.exprs['target'],
self.exprs['output'],
self.loss,
imp_weight=imp_weight))
def _init_exprs(self):
self.exprs = {
'inpt': T.matrix('inpt'),
'target': T.matrix('target')
}
self.exprs['inpt'].tag.test_value = np.zeros(
(10, self.n_inpt)).astype(theano.config.floatX)
self.exprs['target'].tag.test_value = np.zeros(
(10, self.n_output)).astype(theano.config.floatX)
if self.imp_weight:
self.exprs['imp_weight'] = T.matrix('imp_weight')
self.exprs['imp_weight'].tag.test_value = np.zeros(
(10, self.n_output)).astype(theano.config.floatX)
P = self.parameters
n_layers = len(self.n_hiddens)
hidden_to_hiddens = [getattr(P, 'hidden_to_hidden_%i' % i)
for i in range(n_layers - 1)]
hidden_biases = [getattr(P, 'hidden_bias_%i' % i)
for i in range(n_layers)]
self.exprs.update(mlp.exprs(
self.exprs['inpt'],
P.in_to_hidden, hidden_to_hiddens, P.hidden_to_out,
hidden_biases, P.out_bias,
self.hidden_transfers, self.out_transfer))
imp_weight = False if not self.imp_weight else self.exprs['imp_weight']
self.exprs.update(supervised_loss(
self.exprs['target'], self.exprs['output'], self.loss,
imp_weight=imp_weight))
def _init_exprs(self):
E = self.exprs = self._make_start_exprs()
n_dim = E['inpt'].ndim
# Make the expression of the model.
E.update(
sgvb.exprs(E['inpt'],
self._recog_exprs,
self._gen_exprs,
self.assumptions.sample_latents,
shortcut_key=self.shortcut))
# TODO this is not going to work with variance propagation.
imp_weight = False if not self.imp_weight else self._fix_imp_weight(
n_dim)
rec_loss = supervised_loss(E['inpt'],
E['gen']['output'],
self.assumptions.nll_gen_model,
prefix='rec_',
coord_axis=n_dim - 1,
imp_weight=imp_weight)
# Create the KL divergence part of the loss.
kl_loss = self._make_kl_loss()
E.update(rec_loss)
E.update(kl_loss)
E.update({
'loss_sample_wise':
E['kl_sample_wise'] + E['rec_loss_sample_wise'],
'loss':
E['kl'] + E['rec_loss'],
})
def _init_exprs(self):
super(SupervisedLstmRnn, self)._init_exprs()
if self.pooling:
self.exprs['target'] = T.matrix('target')
else:
self.exprs['target'] = T.tensor3('target')
self.exprs.update(supervised_loss(
self.exprs['target'], self.exprs['output'], self.loss, 2))
def _init_exprs(self):
super(SupervisedLstmRnn, self)._init_exprs()
if self.pooling:
self.exprs['target'] = T.matrix('target')
else:
self.exprs['target'] = T.tensor3('target')
self.exprs.update(
supervised_loss(self.exprs['target'], self.exprs['output'],
self.loss, 2))
def _init_exprs(self):
self.exprs = {'inpt': T.matrix('inpt'), 'target': T.matrix('target')}
P = self.parameters
self.exprs.update(
linear.exprs(self.exprs['inpt'], P.in_to_out, P.bias,
self.out_transfer))
self.exprs.update(
supervised_loss(self.exprs['target'], self.exprs['output'],
self.loss))
def _init_exprs(self):
self.exprs = {
'inpt': T.matrix('inpt'),
'target': T.matrix('target')
}
P = self.parameters
self.exprs.update(linear.exprs(
self.exprs['inpt'], P.in_to_out, P.bias, self.out_transfer))
self.exprs.update(supervised_loss(
self.exprs['target'], self.exprs['output'], self.loss))
def _init_exprs(self):
self.exprs = {'inpt': T.matrix('inpt'), 'target': T.matrix('target')}
P = self.parameters
n_layers = len(self.n_hiddens)
hidden_to_hiddens = [
getattr(P, 'hidden_to_hidden_%i' % i) for i in range(n_layers - 1)
]
hidden_biases = [
getattr(P, 'hidden_bias_%i' % i) for i in range(n_layers)
]
self.exprs.update(
mlp.exprs(self.exprs['inpt'], P.in_to_hidden, hidden_to_hiddens,
P.hidden_to_out, hidden_biases, P.out_bias,
self.hidden_transfers, self.out_transfer))
self.exprs.update(
supervised_loss(self.exprs['target'], self.exprs['output'],
self.loss))
def _init_exprs(self):
super(SupervisedRnn, self)._init_exprs()
if self.imp_weight:
self.exprs['imp_weight'] = T.tensor3('imp_weight')
self.exprs['imp_weight'].tag.test_value = np.zeros((5, 2, self.n_output)
).astype(theano.config.floatX)
if self.pooling:
self.exprs['target'] = T.matrix('target')
self.exprs['target'].tag.test_value = np.zeros((2, self.n_output)
).astype(theano.config.floatX)
else:
self.exprs['target'] = T.tensor3('target')
self.exprs['target'].tag.test_value = np.zeros((5, 2, self.n_output)
).astype(theano.config.floatX)
imp_weight = False if not self.imp_weight else self.exprs['imp_weight']
self.exprs.update(supervised_loss(
self.exprs['target'], self.exprs['output'], self.loss, 2,
imp_weight=imp_weight))
def _init_exprs(self):
super(SupervisedRnn, self)._init_exprs()
if self.imp_weight:
self.exprs['imp_weight'] = T.tensor3('imp_weight')
self.exprs['imp_weight'].tag.test_value = np.zeros(
(5, 2, self.n_output)).astype(theano.config.floatX)
if self.pooling:
self.exprs['target'] = T.matrix('target')
self.exprs['target'].tag.test_value = np.zeros(
(2, self.n_output)).astype(theano.config.floatX)
else:
self.exprs['target'] = T.tensor3('target')
self.exprs['target'].tag.test_value = np.zeros(
(5, 2, self.n_output)).astype(theano.config.floatX)
imp_weight = False if not self.imp_weight else self.exprs['imp_weight']
self.exprs.update(
supervised_loss(self.exprs['target'],
self.exprs['output'],
self.loss,
2,
imp_weight=imp_weight))
def _init_exprs(self):
self.exprs = {
'inpt': T.matrix('inpt'),
'target': T.matrix('target')
}
#self.exprs['inpt'].tag.test_value = np.random.rand(5, 784)
#self.exprs['target'].tag.test_value = np.random.rand(5, 10)
if self.imp_weight:
self.exprs['imp_weight'] = T.matrix('imp_weight')
P = self.parameters
n_layers = len(self.n_hiddens)
hidden_to_hiddens = [getattr(P, 'hidden_to_hidden_%i' % i)
for i in range(n_layers - 1)]
hidden_biases = [getattr(P, 'hidden_bias_%i' % i)
for i in range(n_layers)]
for exp_ind in np.arange(self.n_experts):
in_col_from = self.n_expert_hiddens[0]*exp_ind
in_col_to = self.n_expert_hiddens[0]*(exp_ind+1)
out_row_from = self.n_expert_hiddens[-1]*exp_ind
out_row_to = self.n_expert_hiddens[-1]*(exp_ind+1)
out_col_from = self.n_expert_output*exp_ind
out_col_to = self.n_expert_output*(exp_ind+1)
exp_in_to_hidden = P.in_to_hidden[:, in_col_from:in_col_to]
exp_hidden_to_hiddens = [hidden_to_hidden[self.n_expert_hiddens[i]*exp_ind:self.n_expert_hiddens[i]*(exp_ind+1),
self.n_expert_hiddens[i+1]*exp_ind:self.n_expert_hiddens[i+1]*(exp_ind+1)]
for i, hidden_to_hidden in enumerate(hidden_to_hiddens)]
exp_hidden_to_out = P.hidden_to_out[out_row_from:out_row_to, out_col_from:out_col_to]
exp_hidden_biases = [hidden_bias[self.n_expert_hiddens[i]*exp_ind:self.n_expert_hiddens[i]*(exp_ind+1)]
for i, hidden_bias in enumerate(hidden_biases)]
exp_out_bias = P.out_bias[self.n_expert_output*exp_ind:self.n_expert_output*(exp_ind+1)]
exp_dict = mlp.exprs(self.exprs['inpt'], exp_in_to_hidden, exp_hidden_to_hiddens, exp_hidden_to_out,
exp_hidden_biases, exp_out_bias, self.experts_hidden_transfers, self.experts_out_transfer)
for key in exp_dict.keys():
exp_dict[key +'_exp_%d' %exp_ind] = exp_dict[key]
exp_dict.pop(key)
self.exprs.update(exp_dict)
exp_loss_dict = supervised_loss(self.exprs['target'], self.exprs['output_exp_%d' %exp_ind], self.expert_loss)
for key in exp_loss_dict.keys():
exp_loss_dict[key +'_exp_%d' %exp_ind] = exp_loss_dict[key]
exp_loss_dict.pop(key)
self.exprs.update(exp_loss_dict)
man_in_to_hidden = P.in_to_hidden[:, self.n_expert_hiddens[0]*self.n_experts:]
man_hidden_to_hiddens = [hidden_to_hidden[self.n_expert_hiddens[i]*self.n_experts:,
self.n_expert_hiddens[i+1]*self.n_experts:]
for i, hidden_to_hidden in enumerate(hidden_to_hiddens)]
man_hidden_to_out = P.hidden_to_out[self.n_expert_hiddens[-1]*self.n_experts:,
self.n_expert_output*self.n_experts:]
man_hidden_biases = [hidden_bias[self.n_expert_hiddens[i]*self.n_experts:]
for i, hidden_bias in enumerate(hidden_biases)]
man_out_bias = P.out_bias[self.n_expert_output*self.n_experts:]
man_dict = mlp.exprs(self.exprs['inpt'], man_in_to_hidden, man_hidden_to_hiddens, man_hidden_to_out,
man_hidden_biases, man_out_bias, self.hidden_transfers, self.out_transfer)
for key in man_dict.keys():
man_dict[key +'_man'] = man_dict[key]
man_dict.pop(key)
self.exprs.update(man_dict)
self.exprs['loss_sample_wise'] = T.concatenate([T.reshape(self.exprs['loss_sample_wise_exp_%d' %exp], (self.exprs['loss_sample_wise_exp_%d' %exp].shape[0],1))
for exp in np.arange(self.n_experts)], axis=1)
self.exprs['loss_sample_wise'] = T.sum(self.exprs['loss_sample_wise']*self.exprs['output_man'], axis=1)
self.exprs['loss'] = self.exprs['loss_sample_wise'].mean()
chosen_experts = T.argmax(self.exprs['output_man'], axis=1)
selection_matrix = T.zeros((chosen_experts.shape[0],3))
selection_matrix = T.set_subtensor(selection_matrix[T.arange(selection_matrix.shape[0]), chosen_experts], 1)
selection_matrix = T.reshape(selection_matrix, (selection_matrix.shape[0], selection_matrix.shape[1], 1))
expert_outputs = T.concatenate([T.reshape(self.exprs['output_exp_%d' %exp], (self.exprs['output_exp_%d' %exp].shape[0],1,self.n_expert_output)) for exp in np.arange(self.n_experts)], axis=1)
chosen_outputs = selection_matrix * expert_outputs
self.exprs['output'] = T.sum(chosen_outputs, axis=1)