def _init_exprs(self):
self.exprs = {
'inpt': T.matrix('inpt'),
}
self.exprs['inpt'].tag.test_value = np.zeros(
(10, self.n_inpt)).astype(theano.config.floatX)
P = self.parameters
n_layers = len(self.n_hiddens)
if self.tied_weights:
hidden_to_hiddens = [getattr(P, 'hidden_to_hidden_%i' % i)
for i in range(n_layers / 2)]
hidden_to_hiddens += [i.T for i in reversed(hidden_to_hiddens)]
hidden_to_out = P.in_to_hidden.T
else:
hidden_to_hiddens = [getattr(P, 'hidden_to_hidden_%i' % i)
for i in range(n_layers - 1)]
hidden_to_out = P.hidden_to_out
hidden_biases = [getattr(P, 'hidden_bias_%i' % i)
for i in range(n_layers)]
self.exprs.update(mlp.exprs(
self.exprs['inpt'],
P.in_to_hidden, hidden_to_hiddens, hidden_to_out,
hidden_biases, P.out_bias,
self.hidden_transfers, self.out_transfer))
self.exprs.update(supervised_loss(
self.exprs['inpt'], self.exprs['output'], self.loss))
self.exprs['feature'] = self.exprs['layer-%i-output' % self.code_idx]
def _init_exprs(self):
self.exprs = {
'inpt': T.matrix('inpt'),
}
P = self.parameters
n_layers = len(self.n_hiddens)
if self.tied_weights:
hidden_to_hiddens = [
getattr(P, 'hidden_to_hidden_%i' % i)
for i in range(n_layers / 2)
]
hidden_to_hiddens += [i.T for i in reversed(hidden_to_hiddens)]
hidden_to_out = P.in_to_hidden.T
else:
hidden_to_hiddens = [
getattr(P, 'hidden_to_hidden_%i' % i)
for i in range(n_layers - 1)
]
hidden_to_out = P.hidden_to_out
hidden_biases = [
getattr(P, 'hidden_bias_%i' % i) for i in range(n_layers)
]
self.exprs.update(
mlp.exprs(self.exprs['inpt'], P.in_to_hidden, hidden_to_hiddens,
hidden_to_out, hidden_biases, P.out_bias,
self.hidden_transfers, self.out_transfer))
self.exprs.update(
supervised_loss(self.exprs['inpt'], self.exprs['output'],
self.loss))
self.exprs['feature'] = self.exprs['layer-%i-output' % self.code_idx]
def _init_exprs(self):
self.exprs = {'inpt': T.matrix('inpt'), 'target': T.matrix('target')}
self.exprs['inpt'].tag.test_value = np.zeros(
(10, self.n_inpt)).astype(theano.config.floatX)
self.exprs['target'].tag.test_value = np.zeros(
(10, self.n_output)).astype(theano.config.floatX)
if self.imp_weight:
self.exprs['imp_weight'] = T.matrix('imp_weight')
self.exprs['imp_weight'].tag.test_value = np.zeros(
(10, self.n_output)).astype(theano.config.floatX)
P = self.parameters
n_layers = len(self.n_hiddens)
hidden_to_hiddens = [
getattr(P, 'hidden_to_hidden_%i' % i) for i in range(n_layers - 1)
]
hidden_biases = [
getattr(P, 'hidden_bias_%i' % i) for i in range(n_layers)
]
self.exprs.update(
mlp.exprs(self.exprs['inpt'], P.in_to_hidden, hidden_to_hiddens,
P.hidden_to_out, hidden_biases, P.out_bias,
self.hidden_transfers, self.out_transfer))
imp_weight = False if not self.imp_weight else self.exprs['imp_weight']
self.exprs.update(
supervised_loss(self.exprs['target'],
self.exprs['output'],
self.loss,
imp_weight=imp_weight))
def _init_exprs(self):
self.exprs = {
'inpt': T.matrix('inpt'),
'target': T.matrix('target')
}
if self.imp_weight:
self.exprs['imp_weight'] = T.matrix('imp_weight')
P = self.parameters
n_layers = len(self.n_hiddens)
hidden_to_hiddens = [getattr(P, 'hidden_to_hidden_%i' % i)
for i in range(n_layers - 1)]
hidden_biases = [getattr(P, 'hidden_bias_%i' % i)
for i in range(n_layers)]
self.exprs.update(mlp.exprs(
self.exprs['inpt'],
P.in_to_hidden, hidden_to_hiddens, P.hidden_to_out,
hidden_biases, P.out_bias,
self.hidden_transfers, self.out_transfer))
imp_weight = False if not self.imp_weight else self.exprs['imp_weight']
self.exprs.update(supervised_loss(
self.exprs['target'], self.exprs['output'], self.loss,
imp_weight=imp_weight))
def _init_exprs(self):
self.exprs = {
'inpt': T.matrix('inpt'),
'target': T.matrix('target')
}
self.exprs['inpt'].tag.test_value = np.zeros(
(10, self.n_inpt)).astype(theano.config.floatX)
self.exprs['target'].tag.test_value = np.zeros(
(10, self.n_output)).astype(theano.config.floatX)
if self.imp_weight:
self.exprs['imp_weight'] = T.matrix('imp_weight')
self.exprs['imp_weight'].tag.test_value = np.zeros(
(10, self.n_output)).astype(theano.config.floatX)
P = self.parameters
n_layers = len(self.n_hiddens)
hidden_to_hiddens = [getattr(P, 'hidden_to_hidden_%i' % i)
for i in range(n_layers - 1)]
hidden_biases = [getattr(P, 'hidden_bias_%i' % i)
for i in range(n_layers)]
self.exprs.update(mlp.exprs(
self.exprs['inpt'],
P.in_to_hidden, hidden_to_hiddens, P.hidden_to_out,
hidden_biases, P.out_bias,
self.hidden_transfers, self.out_transfer))
imp_weight = False if not self.imp_weight else self.exprs['imp_weight']
self.exprs.update(supervised_loss(
self.exprs['target'], self.exprs['output'], self.loss,
imp_weight=imp_weight))
def _init_exprs(self):
self.exprs = {'inpt': T.matrix('inpt'), 'target': T.matrix('target')}
P = self.parameters
n_layers = len(self.n_hiddens)
hidden_to_hiddens = [
getattr(P, 'hidden_to_hidden_%i' % i) for i in range(n_layers - 1)
]
hidden_biases = [
getattr(P, 'hidden_bias_%i' % i) for i in range(n_layers)
]
self.exprs.update(
mlp.exprs(self.exprs['inpt'], P.in_to_hidden, hidden_to_hiddens,
P.hidden_to_out, hidden_biases, P.out_bias,
self.hidden_transfers, self.out_transfer))
self.exprs.update(
supervised_loss(self.exprs['target'], self.exprs['output'],
self.loss))
def _init_exprs(self):
self.exprs = {
'inpt': T.matrix('inpt'),
'target': T.matrix('target')
}
#self.exprs['inpt'].tag.test_value = np.random.rand(5, 784)
#self.exprs['target'].tag.test_value = np.random.rand(5, 10)
if self.imp_weight:
self.exprs['imp_weight'] = T.matrix('imp_weight')
P = self.parameters
n_layers = len(self.n_hiddens)
hidden_to_hiddens = [getattr(P, 'hidden_to_hidden_%i' % i)
for i in range(n_layers - 1)]
hidden_biases = [getattr(P, 'hidden_bias_%i' % i)
for i in range(n_layers)]
for exp_ind in np.arange(self.n_experts):
in_col_from = self.n_expert_hiddens[0]*exp_ind
in_col_to = self.n_expert_hiddens[0]*(exp_ind+1)
out_row_from = self.n_expert_hiddens[-1]*exp_ind
out_row_to = self.n_expert_hiddens[-1]*(exp_ind+1)
out_col_from = self.n_expert_output*exp_ind
out_col_to = self.n_expert_output*(exp_ind+1)
exp_in_to_hidden = P.in_to_hidden[:, in_col_from:in_col_to]
exp_hidden_to_hiddens = [hidden_to_hidden[self.n_expert_hiddens[i]*exp_ind:self.n_expert_hiddens[i]*(exp_ind+1),
self.n_expert_hiddens[i+1]*exp_ind:self.n_expert_hiddens[i+1]*(exp_ind+1)]
for i, hidden_to_hidden in enumerate(hidden_to_hiddens)]
exp_hidden_to_out = P.hidden_to_out[out_row_from:out_row_to, out_col_from:out_col_to]
exp_hidden_biases = [hidden_bias[self.n_expert_hiddens[i]*exp_ind:self.n_expert_hiddens[i]*(exp_ind+1)]
for i, hidden_bias in enumerate(hidden_biases)]
exp_out_bias = P.out_bias[self.n_expert_output*exp_ind:self.n_expert_output*(exp_ind+1)]
exp_dict = mlp.exprs(self.exprs['inpt'], exp_in_to_hidden, exp_hidden_to_hiddens, exp_hidden_to_out,
exp_hidden_biases, exp_out_bias, self.experts_hidden_transfers, self.experts_out_transfer)
for key in exp_dict.keys():
exp_dict[key +'_exp_%d' %exp_ind] = exp_dict[key]
exp_dict.pop(key)
self.exprs.update(exp_dict)
exp_loss_dict = supervised_loss(self.exprs['target'], self.exprs['output_exp_%d' %exp_ind], self.expert_loss)
for key in exp_loss_dict.keys():
exp_loss_dict[key +'_exp_%d' %exp_ind] = exp_loss_dict[key]
exp_loss_dict.pop(key)
self.exprs.update(exp_loss_dict)
man_in_to_hidden = P.in_to_hidden[:, self.n_expert_hiddens[0]*self.n_experts:]
man_hidden_to_hiddens = [hidden_to_hidden[self.n_expert_hiddens[i]*self.n_experts:,
self.n_expert_hiddens[i+1]*self.n_experts:]
for i, hidden_to_hidden in enumerate(hidden_to_hiddens)]
man_hidden_to_out = P.hidden_to_out[self.n_expert_hiddens[-1]*self.n_experts:,
self.n_expert_output*self.n_experts:]
man_hidden_biases = [hidden_bias[self.n_expert_hiddens[i]*self.n_experts:]
for i, hidden_bias in enumerate(hidden_biases)]
man_out_bias = P.out_bias[self.n_expert_output*self.n_experts:]
man_dict = mlp.exprs(self.exprs['inpt'], man_in_to_hidden, man_hidden_to_hiddens, man_hidden_to_out,
man_hidden_biases, man_out_bias, self.hidden_transfers, self.out_transfer)
for key in man_dict.keys():
man_dict[key +'_man'] = man_dict[key]
man_dict.pop(key)
self.exprs.update(man_dict)
self.exprs['loss_sample_wise'] = T.concatenate([T.reshape(self.exprs['loss_sample_wise_exp_%d' %exp], (self.exprs['loss_sample_wise_exp_%d' %exp].shape[0],1))
for exp in np.arange(self.n_experts)], axis=1)
self.exprs['loss_sample_wise'] = T.sum(self.exprs['loss_sample_wise']*self.exprs['output_man'], axis=1)
self.exprs['loss'] = self.exprs['loss_sample_wise'].mean()
chosen_experts = T.argmax(self.exprs['output_man'], axis=1)
selection_matrix = T.zeros((chosen_experts.shape[0],3))
selection_matrix = T.set_subtensor(selection_matrix[T.arange(selection_matrix.shape[0]), chosen_experts], 1)
selection_matrix = T.reshape(selection_matrix, (selection_matrix.shape[0], selection_matrix.shape[1], 1))
expert_outputs = T.concatenate([T.reshape(self.exprs['output_exp_%d' %exp], (self.exprs['output_exp_%d' %exp].shape[0],1,self.n_expert_output)) for exp in np.arange(self.n_experts)], axis=1)
chosen_outputs = selection_matrix * expert_outputs
self.exprs['output'] = T.sum(chosen_outputs, axis=1)