def _init_exprs(self): self.exprs = {'inpt': T.tensor3('inpt'), 'target': T.tensor3('target')} self.exprs['inpt'].tag.test_value = np.zeros((5, 2, self.n_inpt) ).astype(theano.config.floatX) self.exprs['target'].tag.test_value = np.zeros((5, 2, self.n_output) ).astype(theano.config.floatX) if self.imp_weight: self.exprs['imp_weight'] = T.tensor3('imp_weight') self.exprs['imp_weight'].tag.test_value = np.zeros( (5, 2, self.n_output)).astype(theano.config.floatX) P = self.parameters n_layers = len(self.n_hiddens) hidden_to_hiddens = [getattr(P, 'hidden_to_hidden_%i' % i) for i in range(n_layers - 1)] recurrents = [getattr(P, 'recurrent_%i' % i) for i in range(n_layers)] initial_hidden_means = [getattr(P, 'initial_hidden_means_%i' % i) for i in range(n_layers)] initial_hidden_vars = [getattr(P, 'initial_hidden_vars_%i' % i) ** 2 + 1e-4 for i in range(n_layers)] hidden_biases = [getattr(P, 'hidden_bias_%i' % i) for i in range(n_layers)] if self.skip_to_out: skip_to_outs = [getattr(P, 'hidden_%i_to_out' % i) for i in range(n_layers)] in_to_out = P.in_to_out else: in_to_out = skip_to_outs = None inpt_var = T.zeros_like(self.exprs['inpt']) p_dropouts = ([self.p_dropout_inpt] + self.p_dropout_hiddens + [self.p_dropout_hidden_to_out]) hidden_var_scales_sqrt = [int(i) for i in self.use_varprop_at[:-1]] out_var_scale_sqrt = int(self.use_varprop_at[-1]) self.exprs.update(varprop_rnn.exprs( self.exprs['inpt'], inpt_var, P.in_to_hidden, hidden_to_hiddens, P.hidden_to_out, hidden_biases, hidden_var_scales_sqrt, initial_hidden_means, initial_hidden_vars, recurrents, P.out_bias, out_var_scale_sqrt, self.hidden_transfers, self.out_transfer, in_to_out=in_to_out, skip_to_outs=skip_to_outs, p_dropouts=p_dropouts, hotk_inpt=False)) imp_weight = False if not self.imp_weight else self.exprs['imp_weight'] self.exprs.update(varprop_supervised_loss( self.exprs['target'], self.exprs['output'], self.loss, 2, imp_weight=imp_weight))
def _recog_exprs(self, inpt): """Return the exprssions of the recognition model.""" P = self.parameters.recog n_layers = len(self.n_hiddens_recog) hidden_to_hiddens = [ getattr(P, 'hidden_to_hidden_%i' % i) for i in range(n_layers - 1) ] hidden_biases = [ getattr(P, 'hidden_bias_%i' % i) for i in range(n_layers) ] initial_hidden_means = [ getattr(P, 'initial_hidden_means_%i' % i) for i in range(n_layers) ] initial_hidden_vars = [ getattr(P, 'initial_hidden_vars_%i' % i)**2 + 1e-4 for i in range(n_layers) ] recurrents = [getattr(P, 'recurrent_%i' % i) for i in range(n_layers)] p_dropouts = ([P.p_dropout.inpt] + P.p_dropout.hiddens + [P.p_dropout.hidden_to_out]) # Reparametrize to assert the rates lie in (0.01, 0.5). p_dropouts = [T.nnet.sigmoid(i) * 0.49 + 0.01 for i in p_dropouts] exprs = vprnn.exprs(inpt, T.zeros_like(inpt), P.in_to_hidden, hidden_to_hiddens, P.hidden_to_out, hidden_biases, [1 for _ in hidden_biases], initial_hidden_means, initial_hidden_vars, recurrents, P.out_bias, 1, self.recog_transfers, self.assumptions.statify_latent, p_dropouts=p_dropouts) # TODO also integrate variance! #to_shortcut = self.exprs['inpt'] to_shortcut = exprs['hidden_mean_%i' % (n_layers - 1)] shortcut = T.concatenate( [T.zeros_like(to_shortcut[:1]), to_shortcut[:-1]]) # Hic sunt dracones. # If we do not keep this line, Theano will die with a segfault. shortcut_empty = T.set_subtensor( T.zeros_like(shortcut)[:, :, :], shortcut) exprs['shortcut'] = shortcut_empty return exprs
def _init_exprs(self): self.exprs = {'inpt': T.tensor3('inpt'), 'target': T.tensor3('target')} P = self.parameters n_layers = len(self.n_hiddens) hidden_to_hiddens = [ getattr(P, 'hidden_to_hidden_%i' % i) for i in range(n_layers - 1) ] recurrents = [getattr(P, 'recurrent_%i' % i) for i in range(n_layers)] initial_hiddens = [ getattr(P, 'initial_hiddens_%i' % i) for i in range(n_layers) ] hidden_biases = [ getattr(P, 'hidden_bias_%i' % i) for i in range(n_layers) ] if self.skip_to_out: skip_to_outs = [ getattr(P, 'hidden_%i_to_out' % i) for i in range(n_layers) ] in_to_out = P.in_to_out else: in_to_out = skip_to_outs = None inpt_var = T.zeros_like(self.exprs['inpt']) p_dropouts = ([self.p_dropout_inpt] + self.p_dropout_hiddens + [self.p_dropout_hidden_to_out]) hidden_var_scales_sqrt = [int(i) for i in self.use_varprop_at[:-1]] out_var_scale_sqrt = int(self.use_varprop_at[-1]) self.exprs.update( varprop_rnn.exprs(self.exprs['inpt'], inpt_var, P.in_to_hidden, hidden_to_hiddens, P.hidden_to_out, hidden_biases, hidden_var_scales_sqrt, initial_hiddens, recurrents, P.out_bias, out_var_scale_sqrt, self.hidden_transfers, self.out_transfer, in_to_out=in_to_out, skip_to_outs=skip_to_outs, p_dropouts=p_dropouts, hotk_inpt=False)) self.exprs.update( varprop_supervised_loss(self.exprs['target'], self.exprs['output'], self.loss, 2))
def _gen_exprs(self, inpt): """Return the exprssions of the recognition model.""" P = self.parameters.gen n_layers = len(self.n_hiddens_gen) hidden_to_hiddens = [ getattr(P, 'hidden_to_hidden_%i' % i) for i in range(n_layers - 1) ] hidden_biases = [ getattr(P, 'hidden_bias_%i' % i) for i in range(n_layers) ] initial_hidden_means = [ getattr(P, 'initial_hidden_means_%i' % i) for i in range(n_layers) ] initial_hidden_vars = [ getattr(P, 'initial_hidden_vars_%i' % i)**2 + 1e-4 for i in range(n_layers) ] recurrents = [getattr(P, 'recurrent_%i' % i) for i in range(n_layers)] p_dropout_inpt = T.zeros_like(inpt[:, :, :self.n_latent]) p_dropout_inpt = T.fill(p_dropout_inpt, self.p_dropout_inpt) p_dropout_shortcut = T.zeros_like(inpt[:, :, self.n_latent:]) p_dropout_shortcut = T.fill(p_dropout_shortcut, self.p_dropout_inpt) p_dropout_inpt = T.concatenate([p_dropout_inpt, p_dropout_shortcut], axis=2) p_dropouts = [p_dropout_inpt] + self.p_dropout_hiddens if self.p_dropout_hidden_to_out is None: p_dropouts.append(self.p_dropout_hiddens[-1]) else: p_dropouts.append(self.p_dropout_hidden_to_out) exprs = vprnn.exprs(inpt, T.zeros_like(inpt), P.in_to_hidden, hidden_to_hiddens, P.hidden_to_out, hidden_biases, [1 for _ in hidden_biases], initial_hidden_means, initial_hidden_vars, recurrents, P.out_bias, 1, self.gen_transfers, self.assumptions.statify_visible, p_dropouts=p_dropouts) return exprs
def _recog_exprs(self, inpt): """Return the exprssions of the recognition model.""" P = self.parameters.recog n_layers = len(self.n_hiddens_recog) hidden_to_hiddens = [getattr(P, 'hidden_to_hidden_%i' % i) for i in range(n_layers - 1)] hidden_biases = [getattr(P, 'hidden_bias_%i' % i) for i in range(n_layers)] initial_hidden_means = [getattr(P, 'initial_hidden_means_%i' % i) for i in range(n_layers)] initial_hidden_vars = [getattr(P, 'initial_hidden_vars_%i' % i) ** 2 + 1e-4 for i in range(n_layers)] recurrents = [getattr(P, 'recurrent_%i' % i) for i in range(n_layers)] p_dropouts = ( [P.p_dropout.inpt] + P.p_dropout.hiddens + [P.p_dropout.hidden_to_out]) # Reparametrize to assert the rates lie in (0.025, 1-0.025). p_dropouts = [T.nnet.sigmoid(i) * 0.49 + 0.01 for i in p_dropouts] exprs = vprnn.exprs( inpt, T.zeros_like(inpt), P.in_to_hidden, hidden_to_hiddens, P.hidden_to_out, hidden_biases, [1 for _ in hidden_biases], initial_hidden_means, initial_hidden_vars, recurrents, P.out_bias, 1, self.recog_transfers, self.assumptions.statify_latent, p_dropouts=p_dropouts) # TODO also integrate variance! #to_shortcut = self.exprs['inpt'] to_shortcut = exprs['hidden_mean_%i' % (n_layers - 1)] shortcut = T.concatenate([T.zeros_like(to_shortcut[:1]), to_shortcut[:-1]]) # Hic sunt dracones. # If we do not keep this line, Theano will die with a segfault. shortcut_empty = T.set_subtensor(T.zeros_like(shortcut)[:, :, :], shortcut) exprs['shortcut'] = shortcut_empty return exprs
def _init_exprs(self): self.exprs = {'inpt': T.tensor3('inpt'), 'target': T.tensor3('target')} P = self.parameters n_layers = len(self.n_hiddens) hidden_to_hiddens = [getattr(P, 'hidden_to_hidden_%i' % i) for i in range(n_layers - 1)] recurrents = [getattr(P, 'recurrent_%i' % i) for i in range(n_layers)] initial_hiddens = [getattr(P, 'initial_hiddens_%i' % i) for i in range(n_layers)] hidden_biases = [getattr(P, 'hidden_bias_%i' % i) for i in range(n_layers)] if self.skip_to_out: skip_to_outs = [getattr(P, 'hidden_%i_to_out' % i) for i in range(n_layers)] in_to_out = P.in_to_out else: in_to_out = skip_to_outs = None inpt_var = T.zeros_like(self.exprs['inpt']) p_dropouts = ([self.p_dropout_inpt] + self.p_dropout_hiddens + [self.p_dropout_hidden_to_out]) hidden_var_scales_sqrt = [int(i) for i in self.use_varprop_at[:-1]] out_var_scale_sqrt = int(self.use_varprop_at[-1]) self.exprs.update(varprop_rnn.exprs( self.exprs['inpt'], inpt_var, P.in_to_hidden, hidden_to_hiddens, P.hidden_to_out, hidden_biases, hidden_var_scales_sqrt, initial_hiddens, recurrents, P.out_bias, out_var_scale_sqrt, self.hidden_transfers, self.out_transfer, in_to_out=in_to_out, skip_to_outs=skip_to_outs, p_dropouts=p_dropouts, hotk_inpt=False)) self.exprs.update(varprop_supervised_loss( self.exprs['target'], self.exprs['output'], self.loss, 2))
def _gen_exprs(self, inpt): """Return the exprssions of the recognition model.""" P = self.parameters.gen n_layers = len(self.n_hiddens_gen) hidden_to_hiddens = [getattr(P, 'hidden_to_hidden_%i' % i) for i in range(n_layers - 1)] hidden_biases = [getattr(P, 'hidden_bias_%i' % i) for i in range(n_layers)] initial_hidden_means = [getattr(P, 'initial_hidden_means_%i' % i) for i in range(n_layers)] initial_hidden_vars = [getattr(P, 'initial_hidden_vars_%i' % i) for i in range(n_layers)] recurrents = [getattr(P, 'recurrent_%i' % i) for i in range(n_layers)] shortcut_size = self.n_hiddens_recog[-1] p_dropout_inpt = T.zeros_like(inpt[:, :, :self.n_latent]) p_dropout_inpt = T.fill(p_dropout_inpt, self.p_dropout_inpt) p_dropout_shortcut = T.zeros_like(inpt[:, :, self.n_latent:]) p_dropout_shortcut = T.fill(p_dropout_shortcut, self.p_dropout_inpt) p_dropout_inpt = T.concatenate([p_dropout_inpt, p_dropout_shortcut], axis=2) p_dropouts = [p_dropout_inpt] + self.p_dropout_hiddens if self.p_dropout_hidden_to_out is None: p_dropouts.append(self.p_dropout_hiddens[-1]) else: p_dropouts.append(self.p_dropout_hidden_to_out) exprs = vprnn.exprs( inpt, T.zeros_like(inpt), P.in_to_hidden, hidden_to_hiddens, P.hidden_to_out, hidden_biases, [1 for _ in hidden_biases], initial_hidden_means, initial_hidden_vars, recurrents, P.out_bias, 1, self.gen_transfers, self.assumptions.statify_visible, p_dropouts=p_dropouts) return exprs
def _init_exprs(self): self.exprs = {'inpt': T.tensor3('inpt'), 'target': T.tensor3('target')} self.exprs['inpt'].tag.test_value = np.zeros( (5, 2, self.n_inpt)).astype(theano.config.floatX) self.exprs['target'].tag.test_value = np.zeros( (5, 2, self.n_output)).astype(theano.config.floatX) if self.imp_weight: self.exprs['imp_weight'] = T.tensor3('imp_weight') self.exprs['imp_weight'].tag.test_value = np.zeros( (5, 2, self.n_output)).astype(theano.config.floatX) P = self.parameters n_layers = len(self.n_hiddens) hidden_to_hiddens = [ getattr(P, 'hidden_to_hidden_%i' % i) for i in range(n_layers - 1) ] recurrents = [getattr(P, 'recurrent_%i' % i) for i in range(n_layers)] initial_hidden_means = [ getattr(P, 'initial_hidden_means_%i' % i) for i in range(n_layers) ] initial_hidden_vars = [ getattr(P, 'initial_hidden_vars_%i' % i)**2 + 1e-4 for i in range(n_layers) ] hidden_biases = [ getattr(P, 'hidden_bias_%i' % i) for i in range(n_layers) ] if self.skip_to_out: skip_to_outs = [ getattr(P, 'hidden_%i_to_out' % i) for i in range(n_layers) ] in_to_out = P.in_to_out else: in_to_out = skip_to_outs = None inpt_var = T.zeros_like(self.exprs['inpt']) p_dropouts = ([self.p_dropout_inpt] + self.p_dropout_hiddens + [self.p_dropout_hidden_to_out]) hidden_var_scales_sqrt = [int(i) for i in self.use_varprop_at[:-1]] out_var_scale_sqrt = int(self.use_varprop_at[-1]) self.exprs.update( varprop_rnn.exprs(self.exprs['inpt'], inpt_var, P.in_to_hidden, hidden_to_hiddens, P.hidden_to_out, hidden_biases, hidden_var_scales_sqrt, initial_hidden_means, initial_hidden_vars, recurrents, P.out_bias, out_var_scale_sqrt, self.hidden_transfers, self.out_transfer, in_to_out=in_to_out, skip_to_outs=skip_to_outs, p_dropouts=p_dropouts, hotk_inpt=False)) imp_weight = False if not self.imp_weight else self.exprs['imp_weight'] self.exprs.update( varprop_supervised_loss(self.exprs['target'], self.exprs['output'], self.loss, 2, imp_weight=imp_weight))