def do_theano(self):
        """ Compiles all theano functions needed to use the model"""

        init_names = dir(self)

        ###### All fields you don't want to get pickled (e.g., theano functions) should be created below this line
        self.init_debug()

        # STANDARDIZATION OF S
        stand_s_updates = OrderedDict()
        new_gamma = self.gamma * 0.99 + self.pos_s.std(axis=0) * 0.01
        stand_s_updates[self.gamma] = new_gamma
        stand_s_updates[self.Wv] = new_gamma / self.gamma * self.Wv
        stand_s_updates[self.mu] = self.gamma / new_gamma * self.mu
        self.standardize_s = theano.function([], [], updates=stand_s_updates)

        # SAMPLING: NEGATIVE PHASE
        neg_updates = self.neg_sampling_updates(n_steps=self.neg_sample_steps, use_pcd=True)
        self.sample_func = theano.function([], [], updates=neg_updates)

        # POSITIVE PHASE
        pos_updates = self.pos_phase_updates(
                self.input,
                n_steps = self.pos_steps)

        self.inference_func = theano.function([self.input], [],
                updates=pos_updates)

        ##
        # BUILD COST OBJECTS
        ##
        lcost = self.ml_cost(
                        pos_g = self.pos_g,
                        pos_h = self.pos_h,
                        pos_s1 = self.pos_s1,
                        pos_s0 = self.pos_s0,
                        pos_v = self.input,
                        neg_g = neg_updates[self.neg_g],
                        neg_h = neg_updates[self.neg_h],
                        neg_s = neg_updates[self.neg_s],
                        neg_v = neg_updates[self.neg_v])

        #spcost = self.get_sparsity_cost(
                        #pos_g = pos_updates[self.pos_g],
                        #pos_h = pos_updates[self.pos_h])

        regcost = self.get_reg_cost(self.l2, self.l1)

        ##
        # COMPUTE GRADIENTS WRT. COSTS
        ##
        #main_cost = [lcost, spcost, regcost]
        main_cost = [lcost, regcost]

        learning_grads = costmod.compute_gradients(self.lr, self.lr_mults, *main_cost)

        weight_updates = OrderedDict()
        if self.flags['wv_true_gradient']:
            weight_updates[self.Wv] = true_gradient(self.Wv, -learning_grads[self.Wv])

        ##
        # BUILD UPDATES DICTIONARY FROM GRADIENTS
        ##
        learning_updates = costmod.get_updates(learning_grads)
        learning_updates.update(neg_updates)
        learning_updates.update({self.iter: self.iter+1})
        learning_updates.update(weight_updates)

        # build theano function to train on a single minibatch
        self.batch_train_func = function([self.input], [],
                                         updates=learning_updates,
                                         name='train_rbm_func')
        #theano.printing.pydotprint(self.batch_train_func, outfile='batch_train_func.png', scan_graphs=True);

        #######################
        # CONSTRAINT FUNCTION #
        #######################
        constraint_updates = self.get_constraint_updates()
        self.enforce_constraints = theano.function([],[], updates=constraint_updates)

        ###### All fields you don't want to get pickled should be created above this line
        final_names = dir(self)
        self.register_names_to_del( [ name for name in (final_names) if name not in init_names ])

        # Before we start learning, make sure constraints are enforced
        self.enforce_constraints()
Example #2
0
    def do_theano(self):
        """ Compiles all theano functions needed to use the model"""

        init_names = dir(self)

        ###### All fields you don't want to get pickled (e.g., theano functions) should be created below this line

        # build energy function
        _gtemp = T.matrix('_gtemp')
        _htemp = T.matrix('_htemp')
        _vtemp = T.matrix('_vtemp')
        energy, cte = self.free_energy(_gtemp, _htemp, _vtemp, mean=False)
        self.energy_func = theano.function([_gtemp, _htemp, _vtemp], energy)

        # SAMPLING: NEGATIVE PHASE
        neg_updates = self.neg_sampling_updates(n_steps=self.neg_sample_steps, use_pcd=True)
        self.sample_func = theano.function([], [], updates=neg_updates)

        # POSITIVE PHASE
        pos_states, pos_updates = self.pos_phase_updates(
                self.input,
                n_steps = self.pos_steps,
                mean_field=self.flags['mean_field'])
        self.inference_func = theano.function([self.input], [pos_states['g'], pos_states['h']])

        ##
        # BUILD COST OBJECTS
        ##
        lcost = self.ml_cost(
                        pos_g = pos_states['g'],
                        pos_h = pos_states['h'],
                        pos_v = self.input,
                        neg_g = neg_updates[self.neg_g],
                        neg_h = neg_updates[self.neg_h],
                        neg_v = neg_updates[self.neg_v],
                        mean_field=self.flags['mean_field'])
        spcost = self.get_sparsity_cost(pos_states['g'], pos_states['h'])
        regcost = self.get_reg_cost(self.l2, self.l1)

        ##
        # COMPUTE GRADIENTS WRT. COSTS
        ##
        main_cost = [lcost, spcost, regcost]

        learning_grads = costmod.compute_gradients(self.lr, self.lr_mults, *main_cost)

        weight_updates = OrderedDict()
        if self.flags['wg_norm'] == 'unit' and self.Wg in self.params():
            weight_updates[self.Wg] = true_gradient(self.Wg, -learning_grads[self.Wg])
        if self.flags['wh_norm'] == 'unit' and self.Wh in self.params():
            weight_updates[self.Wh] = true_gradient(self.Wh, -learning_grads[self.Wh])
        if self.flags['wv_norm'] == 'unit':
            weight_updates[self.Wv] = true_gradient(self.Wv, -learning_grads[self.Wv])

        ##
        # BUILD UPDATES DICTIONARY FROM GRADIENTS
        ##
        learning_updates = costmod.get_updates(learning_grads)
        learning_updates.update(pos_updates)
        learning_updates.update(neg_updates)
        learning_updates.update({self.iter: self.iter+1})
        learning_updates.update(weight_updates)

        # build theano function to train on a single minibatch
        self.batch_train_func = function([self.input], [],
                                         updates=learning_updates,
                                         name='train_rbm_func')
        #theano.printing.pydotprint(self.batch_train_func, outfile='batch_train_func.png', scan_graphs=True);

        #######################
        # CONSTRAINT FUNCTION #
        #######################
        constraint_updates = self.get_constraint_updates()
        self.enforce_constraints = theano.function([],[], updates=constraint_updates)

        ###### All fields you don't want to get pickled should be created above this line
        final_names = dir(self)
        self.register_names_to_del( [ name for name in (final_names) if name not in init_names ])

        # Before we start learning, make sure constraints are enforced
        self.enforce_constraints()
    def do_theano(self):
        """ Compiles all theano functions needed to use the model"""

        init_names = dir(self)

        ###### All fields you don't want to get pickled (e.g., theano functions) should be created below this line
        # SAMPLING: NEGATIVE PHASE
        neg_updates = self.neg_sampling_updates(n_steps=self.neg_sample_steps, use_pcd=True)
        self.sample_func = theano.function([], [], updates=neg_updates)

        # VARIATIONAL E-STEP
        pos_updates = OrderedDict()
        if self.pos_mf_steps:
            pos_states, mf_updates = self.pos_phase_updates(
                    self.input,
                    mean_field = True,
                    n_steps = self.pos_mf_steps)
            pos_updates.update(mf_updates)

        # SAMPLING: POSITIVE PHASE
        if self.pos_sample_steps:
            init_state = pos_states if self.pos_mf_steps else None
            pos_states, sample_updates = self.pos_phase_updates(
                    self.input,
                    init_state = init_state,
                    mean_field = False,
                    n_steps = self.pos_sample_steps)
            pos_updates.update(sample_updates)

        ##
        # BUILD COST OBJECTS
        ##
        lcost = self.ml_cost(
                        pos_g = pos_states['g'],
                        pos_s = pos_states['s'],
                        pos_h = pos_states['h'],
                        pos_t = pos_states['t'],
                        pos_v = self.input,
                        neg_g = neg_updates[self.neg_g],
                        neg_s = neg_updates[self.neg_s],
                        neg_h = neg_updates[self.neg_h],
                        neg_t = neg_updates[self.neg_t],
                        neg_v = neg_updates[self.neg_v])

        spcost = self.get_sparsity_cost(
                pos_states['g'], pos_states['s'],
                pos_states['h'], pos_states['t'])

        regcost = self.get_reg_cost(self.l2, self.l1)

        ##
        # COMPUTE GRADIENTS WRT. COSTS
        ##
        main_cost = [lcost, spcost, regcost]
        learning_grads = costmod.compute_gradients(self.lr, self.lr_mults, *main_cost)

        weight_updates = OrderedDict()
        weight_updates[self.Wv] = true_gradient(self.Wv, -learning_grads[self.Wv])
        if self.Wg in self.params():
            weight_updates[self.Wg] = true_gradient(self.Wg, -learning_grads[self.Wg])
        if self.Wh in self.params():
            weight_updates[self.Wh] = true_gradient(self.Wh, -learning_grads[self.Wh])

        ##
        # BUILD UPDATES DICTIONARY FROM GRADIENTS
        ##
        learning_updates = costmod.get_updates(learning_grads)
        learning_updates.update(pos_updates)
        learning_updates.update(neg_updates)
        learning_updates.update({self.iter: self.iter+1})
        learning_updates.update(weight_updates)

        # build theano function to train on a single minibatch
        self.batch_train_func = function([self.input], [],
                                         updates=learning_updates,
                                         name='train_rbm_func')

        self.energy_fn = function([], self.energy(self.neg_g, self.neg_s, self.neg_h,
            self.neg_t, self.neg_v))

        self.g_fn = function([], self.g_given_htv(self.neg_h, self.neg_t, self.neg_v))
        self.h_fn = function([], self.h_given_gsv(self.neg_g, self.neg_s, self.neg_v))
        self.s_fn = function([], self.s_given_ghtv(self.neg_g, self.neg_h, self.neg_t, self.neg_v))
        self.t_fn = function([], self.t_given_gshv(self.neg_g, self.neg_s, self.neg_h, self.neg_v))
        self.v_fn = function([], self.v_given_gsht(self.neg_g, self.neg_s, self.neg_h, self.neg_t))
        self.sample_g_fn = function([], self.sample_g_given_htv(self.neg_h, self.neg_t, self.neg_v))
        self.sample_h_fn = function([], self.sample_h_given_gsv(self.neg_g, self.neg_s, self.neg_v))
        self.sample_s_fn = function([], self.sample_s_given_ghtv(self.neg_g, self.neg_h, self.neg_t, self.neg_v))
        self.sample_t_fn = function([], self.sample_t_given_gshv(self.neg_g, self.neg_s, self.neg_h, self.neg_v))
        self.sample_v_fn = function([], self.sample_v_given_gsht(self.neg_g, self.neg_s, self.neg_h, self.neg_t))

        #######################
        # CONSTRAINT FUNCTION #
        #######################

        # enforce constraints function
        constraint_updates = OrderedDict()
        constraint_updates[self.lambd] = T.mean(self.lambd) * T.ones_like(self.lambd)

        ## clip parameters to maximum values (if applicable)
        for (k,v) in self.clip_max.iteritems():
            assert k in [param.name for param in self.params()]
            param = getattr(self, k)
            constraint_updates[param] = T.clip(param, param, v)

        ## clip parameters to minimum values (if applicable)
        for (k,v) in self.clip_min.iteritems():
            assert k in [param.name for param in self.params()]
            param = getattr(self, k)
            constraint_updates[param] = T.clip(constraint_updates.get(param, param), v, param)
        
        self.enforce_constraints = theano.function([],[], updates=constraint_updates)

        ###### All fields you don't want to get pickled should be created above this line
        final_names = dir(self)
        self.register_names_to_del( [ name for name in (final_names) if name not in init_names ])

        # Before we start learning, make sure constraints are enforced
        self.enforce_constraints()
    def do_theano(self):
        """ Compiles all theano functions needed to use the model"""

        init_names = dir(self)

        ###### All fields you don't want to get pickled (e.g., theano functions) should be created below this line
        # SAMPLING: NEGATIVE PHASE
        neg_updates = self.neg_sampling_updates(n_steps=self.neg_sample_steps, use_pcd=True)
        self.sample_func = theano.function([], [], updates=neg_updates)

        ### BUILD TWO FUNCTIONS: ONE CONDITIONED ON LABELS, ONE WHERE LABELS ARE SAMPLED
        self.batch_train_func = {}
        for (key, input_label) in zip(['nolabel','label'], [None, self.input_labels]):

            # POSITIVE PHASE
            pos_states, pos_updates = self.pos_phase_updates(
                    self.input, l = input_label,
                    n_steps = self.pos_steps,
                    mean_field=self.flags['mean_field'])

            ##
            # BUILD COST OBJECTS
            ##
            lcost = self.ml_cost(
                            pos_g = pos_states['g'],
                            pos_h = pos_states['h'],
                            pos_l = pos_states['l'],
                            pos_v = self.input,
                            neg_g = neg_updates[self.neg_g],
                            neg_h = neg_updates[self.neg_h],
                            neg_v = neg_updates[self.neg_v],
                            neg_l = neg_updates[self.neg_l],
                            mean_field=self.flags['mean_field'])
            spcost = self.get_sparsity_cost(pos_states['g'], pos_states['h'], pos_states['l'])
            regcost = self.get_reg_cost(self.l2, self.l1)

            ##
            # COMPUTE GRADIENTS WRT. COSTS
            ##
            main_cost = [lcost, spcost, regcost]

            learning_grads = costmod.compute_gradients(self.lr, self.lr_mults, *main_cost)

            weight_updates = OrderedDict()
            if self.flags['wg_norm'] == 'unit' and self.Wg in self.params():
                weight_updates[self.Wg] = true_gradient(self.Wg, -learning_grads[self.Wg])
            if self.flags['wh_norm'] == 'unit' and self.Wh in self.params():
                weight_updates[self.Wh] = true_gradient(self.Wh, -learning_grads[self.Wh])
            if self.flags['wv_norm'] == 'unit':
                weight_updates[self.Wv] = true_gradient(self.Wv, -learning_grads[self.Wv])

            ##
            # BUILD UPDATES DICTIONARY FROM GRADIENTS
            ##
            learning_updates = costmod.get_updates(learning_grads)
            learning_updates.update(pos_updates)
            learning_updates.update(neg_updates)
            learning_updates.update({self.iter: self.iter+1})
            learning_updates.update(weight_updates)

            # build theano function to train on a single minibatch
            inputs = [self.input, self.input_labels] if key == 'label' else [self.input]
            self.batch_train_func[key] = function(inputs, [],
                                             updates=learning_updates,
                                             name='train_rbm_func_%s' % key)

        #######################
        # CONSTRAINT FUNCTION #
        #######################
        constraint_updates = self.get_constraint_updates()
        self.enforce_constraints = theano.function([],[], updates=constraint_updates)

        ###### All fields you don't want to get pickled should be created above this line
        final_names = dir(self)
        self.register_names_to_del( [ name for name in (final_names) if name not in init_names ])

        # Before we start learning, make sure constraints are enforced
        self.enforce_constraints()