def test_partition_function_factorize_h(self): sys.stdout.write( 'RBM Estimator -> Performing partition_function_factorize_v test ...' ) sys.stdout.flush() LogZ = Estimator.partition_function_factorize_h( self.bbrbm, beta=None, batchsize_exponent='AUTO', status=False) assert numx.all(numx.abs(LogZ - self.bbrbmTruelogZ) < self.epsilon) LogZ = Estimator.partition_function_factorize_h(self.bbrbm, beta=None, batchsize_exponent=0, status=False) assert numx.all(numx.abs(LogZ - self.bbrbmTruelogZ) < self.epsilon) LogZ = Estimator.partition_function_factorize_h(self.bbrbm, beta=None, batchsize_exponent=3, status=False) assert numx.all(numx.abs(LogZ - self.bbrbmTruelogZ) < self.epsilon) LogZ = Estimator.partition_function_factorize_h(self.bbrbm, beta=None, batchsize_exponent=555, status=False) assert numx.all(numx.abs(LogZ - self.bbrbmTruelogZ) < self.epsilon) print(' successfully passed!') sys.stdout.flush()
def partition_function_exact(model, batchsize_exponent='AUTO'): ''' Computes the true partition function for the given model by factoring over the visible and hidden2 units. :Parameters: model: The model -type: Valid DBM model batchsize_exponent: 2^batchsize_exponent will be the batch size. -type: int :Returns: Log Partition function for the model. -type: float ''' # We transform the DBM to an RBM with restricted connections. rbm = RBM_MODEL.BinaryBinaryRBM( number_visibles=model.input_dim + model.hidden2_dim, number_hiddens=model.hidden1_dim, data=None, initial_weights=numx.vstack((model.W1, model.W2.T)), initial_visible_bias=numx.hstack((model.b1, model.b3)), initial_hidden_bias=model.b2, initial_visible_offsets=numx.hstack((model.o1, model.o3)), initial_hidden_offsets=model.o2) return RBM_ESTIMATOR.partition_function_factorize_h(rbm)
def _train(self, data, epsilon, k, momentum, reg_l1norm, reg_l2norm, reg_sparseness, desired_sparseness, update_visible_offsets, update_hidden_offsets, offset_typ, use_centered_gradient, restrict_gradient, restriction_norm, use_hidden_states): """ The training for one batch is performed using True Gradient (GD) for k Gibbs-sampling steps. :param data: The data used for training. :type data: numpy array [batch_size, input dimension] :param epsilon: The learning rate. :type epsilon: scalar or numpy array[num parameters] or numpy array[num parameters, parameter shape] :param k: NUmber of sampling steps. :type k: int :param momentum: The momentum term. :type momentum: scalar or numpy array[num parameters] or numpy array[num parameters, parameter shape] :param reg_l1norm: The parameter for the L1 regularization :type reg_l1norm: float :param reg_l2norm: The parameter for the L2 regularization also know as weight decay. :type reg_l2norm: float :param reg_sparseness: The parameter for the desired_sparseness regularization. :type reg_sparseness: None or float :param desired_sparseness: Desired average hidden activation or None for no regularization. :type desired_sparseness: None or float :param update_visible_offsets: The update step size for the models visible offsets. :type update_visible_offsets: float :param update_hidden_offsets: The update step size for the models hidden offsets. :type update_hidden_offsets: float :param offset_typ: | Different offsets can be used to center the gradient.<br /> | Example: 'DM' uses the positive phase visible mean and the negative phase hidden mean. | 'A0' uses the average of positive and negative phase mean for visible, zero for the | hiddens. Possible values are out of {A,D,M,0}x{A,D,M,0} :type offset_typ: string :param use_centered_gradient: Uses the centered gradient instead of centering. :type use_centered_gradient: bool :param restrict_gradient: If a scalar is given the norm of the weight gradient (along the input dim) is \ restricted to stay below this value. :type restrict_gradient: None, float :param restriction_norm: Restricts the column norm, row norm or Matrix norm. :type restriction_norm: string, 'Cols','Rows', 'Mat' :param use_hidden_states: If True, the hidden states are used for the gradient calculations, the hiddens \ probabilities otherwise. :type use_hidden_states: bool """ # Sample the first time hid_probs_pos = self.model.probability_h_given_v(data) if update_visible_offsets != 0.0: xmean_pos = numx.mean(data, axis=0).reshape(1, self.model.input_dim) hmean_pos = 0.0 if update_hidden_offsets != 0.0 or reg_sparseness != 0.0: if use_hidden_states: hid_states_pos = self.model.sample_h(hid_probs_pos) hmean_pos = numx.mean(hid_states_pos, axis=0).reshape(1, self.model.output_dim) else: hmean_pos = numx.mean(hid_probs_pos, axis=0).reshape(1, self.model.output_dim) # Calculate the partition function if self.model.input_dim < self.model.output_dim: batch_size = numx.min([self.model.input_dim, 12]) ln_z = estimator.partition_function_factorize_v(self.model, beta=1.0, batchsize_exponent=batch_size, status=False) else: batch_size = numx.min([self.model.output_dim, 12]) ln_z = estimator.partition_function_factorize_h(self.model, beta=1.0, batchsize_exponent=batch_size, status=False) # empty negative phase parts neg_gradients = [numx.zeros(self.model.w.shape), numx.zeros(self.model.bv.shape), numx.zeros(self.model.bh.shape)] # Calculate gradient stepwise in batches bit_length = self.model.input_dim batchsize = numx.power(2, batch_size) num_combinations = numx.power(2, bit_length) num_batches = num_combinations / batchsize for batch in range(0, num_batches): # Generate current batch bit_combinations = numxext.generate_binary_code(bit_length, batch_size, batch) # P(x) prob_x = numx.exp( self.model.log_probability_v(ln_z, bit_combinations)) # P(h|x) prob_h_x = self.model.probability_h_given_v(bit_combinations) # Calculate gradient neg_gradients[1] += numx.sum(numx.tile(prob_x, (1, self.model.input_dim)) * (bit_combinations - self.model.ov), axis=0) prob_x = (numx.tile(prob_x, (1, self.model.output_dim)) * (prob_h_x - self.model.oh)) neg_gradients[0] += numx.dot((bit_combinations - self.model.ov).T, prob_x) neg_gradients[2] += numx.sum(prob_x, axis=0) if update_visible_offsets != 0.0 and (offset_typ[0] is 'A' or offset_typ[0] is 'M'): bit_combinations = numxext.generate_binary_code(self.model.input_dim, None, 0) prob_x = numx.exp(self.model.log_probability_v(ln_z, bit_combinations)) xmean_neg = numx.sum(prob_x * bit_combinations, axis=0).reshape(1, self.model.input_dim) if update_hidden_offsets != 0.0 and (offset_typ[1] is 'A' or offset_typ[1] is 'M'): bit_combinations = numxext.generate_binary_code(self.model.output_dim, None, 0) prob_h = numx.exp(self.model.log_probability_h(ln_z, bit_combinations)) hmean_neg = numx.sum(prob_h * bit_combinations, axis=0).reshape(1, self.model.output_dim) new_visible_offsets = 0.0 if update_visible_offsets != 0.0: if offset_typ[0] is 'A': new_visible_offsets = (xmean_pos + xmean_neg) * 0.5 if offset_typ[0] is 'D': new_visible_offsets = xmean_pos if offset_typ[0] is 'M': new_visible_offsets = xmean_neg if offset_typ[0] is '0': new_visible_offsets = 0.0 * xmean_pos new_hidden_offsets = 0.0 if update_hidden_offsets != 0.0: if offset_typ[1] is 'A': new_hidden_offsets = (hmean_pos + hmean_neg) * 0.5 if offset_typ[1] is 'D': new_hidden_offsets = hmean_pos if offset_typ[1] is 'M': new_hidden_offsets = hmean_neg if offset_typ[1] is '0': new_hidden_offsets = 0.0 * hmean_pos if use_centered_gradient is False: # update the centers self.model.update_offsets(new_visible_offsets, new_hidden_offsets, update_visible_offsets, update_hidden_offsets) self.visible_offsets = 0.0 self.hidden_offsets = 0.0 else: self.hidden_offsets = ((1.0 - update_hidden_offsets) * self.hidden_offsets + update_hidden_offsets * new_hidden_offsets) self.visible_offsets = ((1.0 - update_visible_offsets) * self.visible_offsets + update_visible_offsets * new_visible_offsets) # Calculate positive phase gradient using states or probabilities if use_hidden_states: pos_gradients = self.model.calculate_gradients(data, hid_states_pos) else: pos_gradients = self.model.calculate_gradients(data, hid_probs_pos) # Times batch size since adpat gradient devides by batchsize neg_gradients[0] *= data.shape[0] neg_gradients[1] *= data.shape[0] neg_gradients[2] *= data.shape[0] # Adapt the gradients by weight decay momentum and learning rate self._adapt_gradient(pos_gradients=pos_gradients, neg_gradients=neg_gradients, batch_size=data.shape[0], epsilon=epsilon, momentum=momentum, reg_l1norm=reg_l1norm, reg_l2norm=reg_l2norm, reg_sparseness=reg_sparseness, desired_sparseness=desired_sparseness, mean_hidden_activity=hmean_pos, visible_offsets=self.visible_offsets, hidden_offsets=self.hidden_offsets, use_centered_gradient=use_centered_gradient, restrict_gradient=restrict_gradient, restriction_norm=restriction_norm) # update the parameters with the calculated gradient self.model.update_parameters(self.parameter_updates)
print( 'Epoch\tRecon. Error\tLog likelihood train\tLog likelihood test\tExpected End-Time' ) for epoch in range(epochs): # Loop over all batches for b in range(0, train_data.shape[0], batch_size): batch = train_data[b:b + batch_size, :] trainer_pcd.train(data=batch, epsilon=0.01, update_visible_offsets=update_offsets, update_hidden_offsets=update_offsets) # Calculate Log-Likelihood, reconstruction error and expected end time every 5th epoch if (epoch == 0 or (epoch + 1) % 5 == 0): logZ = estimator.partition_function_factorize_h(rbm) ll_train = numx.mean(estimator.log_likelihood_v(rbm, logZ, train_data)) ll_test = numx.mean(estimator.log_likelihood_v(rbm, logZ, test_data)) re = numx.mean(estimator.reconstruction_error(rbm, train_data)) print('{}\t\t{:.4f}\t\t\t{:.4f}\t\t\t\t{:.4f}\t\t\t{}'.format( epoch + 1, re, ll_train, ll_test, measurer.get_expected_end_time(epoch + 1, epochs))) else: print(epoch + 1) measurer.end() # Print end/training time print("End-time: \t{}".format(measurer.get_end_time())) print("Training time:\t{}".format(measurer.get_interval()))
momentum=0.0, reg_l1norm=0.0, reg_l2norm=0.0, reg_sparseness=0.0, desired_sparseness=0.0, update_visible_offsets=0.0, update_hidden_offsets=0.0, restrict_gradient=False, restriction_norm='Cols', use_hidden_states=False, use_centered_gradient=False) # Calculate Log likelihood and reconstruction error RE_train = numx.mean(estimator.reconstruction_error(rbm, train_data)) RE_test = numx.mean(estimator.reconstruction_error(rbm, test_data)) logZ = estimator.partition_function_factorize_h(rbm, batchsize_exponent=h1) LL_train = numx.mean(estimator.log_likelihood_v(rbm, logZ, train_data)) LL_test = numx.mean(estimator.log_likelihood_v(rbm, logZ, test_data)) print '%5d \t%0.5f \t%0.5f \t%0.5f \t%0.5f' % (epoch, RE_train, RE_test, LL_train, LL_test) # Calculate partition function and its AIS approximation logZ = estimator.partition_function_factorize_h(rbm, batchsize_exponent=h1) logZ_AIS = estimator.annealed_importance_sampling(rbm, num_chains=100, k=1, betas=1000, status=False)[0] logZ_rAIS = estimator.reverse_annealed_importance_sampling(rbm, num_chains=100, k=1,
# Measuring time measurer = MEASURE.Stopwatch() # Train model print "Training" print "Epoch\tRecon. Error\tLog likelihood \tExpected End-Time" for epoch in range(1, epochs + 1): train_data = numx.random.permutation(train_data) for b in range(0, train_data.shape[0], batch_size): batch = train_data[b : b + batch_size, :] trainer.train(data=batch, epsilon=0.1, regL2Norm=0.001) # Calculate Log-Likelihood, reconstruction error and expected end time every 10th epoch if epoch % 10 == 0: Z = ESTIMATOR.partition_function_factorize_h(rbm) LL = numx.mean(ESTIMATOR.log_likelihood_v(rbm, Z, train_data)) RE = numx.mean(ESTIMATOR.reconstruction_error(rbm, train_data)) print "%d\t\t%8.6f\t\t%8.4f\t\t" % (epoch, RE, LL), print measurer.get_expected_end_time(epoch, epochs), print measurer.end() # Print end time print print "End-time: \t", measurer.get_end_time() print "Training time:\t", measurer.get_interval() # Calculate and approximate partition function Z = ESTIMATOR.partition_function_factorize_h(rbm, batchsize_exponent=h1, status=False)