def partition_function_exact(model, batchsize_exponent='AUTO'): ''' Computes the true partition function for the given model by factoring over the visible and hidden2 units. :Parameters: model: The model -type: Valid DBM model batchsize_exponent: 2^batchsize_exponent will be the batch size. -type: int :Returns: Log Partition function for the model. -type: float ''' # We transform the DBM to an RBM with restricted connections. rbm = RBM_MODEL.BinaryBinaryRBM( number_visibles=model.input_dim + model.hidden2_dim, number_hiddens=model.hidden1_dim, data=None, initial_weights=numx.vstack((model.W1, model.W2.T)), initial_visible_bias=numx.hstack((model.b1, model.b3)), initial_hidden_bias=model.b2, initial_visible_offsets=numx.hstack((model.o1, model.o3)), initial_hidden_offsets=model.o2) return RBM_ESTIMATOR.partition_function_factorize_h(rbm)
def partition_function_AIS(model, num_chains=100, k=1, betas=10000, status=False): ''' Approximates the partition function for the given model using annealed importance sampling. :Parameters: model: The model. -type: Valid RBM model. num_chains: Number of AIS runs. -type: int k: Number of Gibbs sampling steps. -type: int beta: Number or a list of inverse temperatures to sample from. -type: int, numpy array [num_betas] status: If true prints the progress on console. -type: bool :Returns: Mean estimated log partition function. -type: float Mean +3std estimated log partition function. -type: float Mean -3std estimated log partition function. -type: float ''' # We transform the DBM to an RBM with restricted connections. rbm = RBM_MODEL.BinaryBinaryRBM( number_visibles=model.input_dim + model.hidden2_dim, number_hiddens=model.hidden1_dim, data=None, initial_weights=numx.vstack((model.W1, model.W2.T)), initial_visible_bias=numx.hstack((model.b1, model.b3)), initial_hidden_bias=model.b2, initial_visible_offsets=numx.hstack((model.o1, model.o3)), initial_hidden_offsets=model.o2) # Run AIS for the transformed DBM return RBM_ESTIMATOR.annealed_importance_sampling(model=rbm, num_chains=num_chains, k=k, betas=betas, status=status)
def __init__(self, model, batch_size): # Set batch size self.batch_size = batch_size # Store model self.model = model self.rbm = RBM_MODEL.BinaryBinaryRBM( number_visibles=model.input_dim + model.hidden2_dim, number_hiddens=model.hidden1_dim, data=None, initial_weights=numx.vstack((model.W1, model.W2.T)), initial_visible_bias=numx.hstack((model.b1, model.b3)), initial_hidden_bias=model.b2, initial_visible_offsets=numx.hstack((model.o1, model.o3)), initial_hidden_offsets=model.o2) self.sampler = RBM_SAMPLER.GibbsSampler(self.rbm)
def __init__(self, model, batch_size): # Set batch size self.batch_size = batch_size # Store model self.model = model rbm = RBM_MODEL.BinaryBinaryRBM( number_visibles=model.input_dim + model.hidden2_dim, number_hiddens=model.hidden1_dim, data=None, initial_weights=numx.vstack((model.W1, model.W2.T)), initial_visible_bias=numx.hstack((model.b1, model.b3)), initial_hidden_bias=model.b2, initial_visible_offsets=numx.hstack((model.o1, model.o3)), initial_hidden_offsets=model.o2) # Initializee Markov chains self.m1 = model.o1 + numx.zeros((batch_size, model.input_dim)) self.m2 = model.o2 + numx.zeros((batch_size, model.hidden1_dim)) self.m3 = model.o3 + numx.zeros((batch_size, model.hidden2_dim))
class TestSampler(unittest.TestCase): bbrbmData = generate_bars_and_stripes_complete(2) bbrbmData = numx.vstack((bbrbmData[0], bbrbmData, bbrbmData[5])) bbrbmw = numx.array([[0.12179488, 2.95950177, 0.33513356, 35.05380642], [0.20318085, -28.62372894, 26.52611278, 28.41793445], [-0.19105386, -28.58530584, -26.52747507, 28.78447320], [0.08953740, -59.82556859, -0.06665933, -27.71723459]]) bbrbmbv = numx.array([[-19.24399659, -13.26258696, 13.25909850, 43.74408543]]) bbrbmbh = numx.array([[-0.11155958, 57.02097584, -0.13331758, -32.25991501]]) bbrbm = Model.BinaryBinaryRBM(4, 4, bbrbmData, bbrbmw, bbrbmbv, bbrbmbh, 0.0, 0.0) epsilon = 0.05 num_samples = 2000.0 @classmethod def execute_sampler(cls, sampler, num_samples): dictC = {'[ 0. 0. 0. 0.]': 0, # 2 '[ 1. 1. 1. 1.]': 0, # 2 '[ 0. 0. 1. 1.]': 0, # 1 '[ 1. 1. 0. 0.]': 0, # 1 '[ 1. 0. 1. 0.]': 0, # 1 '[ 0. 1. 0. 1.]': 0, # 1 '[ 0. 1. 1. 0.]': 0, '[ 1. 0. 0. 1.]': 0, '[ 0. 0. 0. 1.]': 0, '[ 0. 0. 1. 0.]': 0, '[ 0. 1. 0. 0.]': 0, '[ 1. 0. 0. 0.]': 0, '[ 0. 1. 1. 1.]': 0, '[ 1. 1. 1. 0.]': 0, '[ 1. 0. 1. 1.]': 0, '[ 1. 1. 0. 1.]': 0} for _ in range(numx.int32(num_samples)): if isinstance(sampler, Sampler.GibbsSampler): # Start form random since model is rather deterministic samples = sampler.sample(numx.random.rand(1, 4), 1, ret_states=True) else: if isinstance(sampler, Sampler.PersistentGibbsSampler): # Start form random since model is rather deterministic sampler.chains = numx.random.rand(1, 4) samples = sampler.sample(1, 1, ret_states=True) else: samples = sampler.sample(1, 1, ret_states=True) dictC[str(samples[0])] += 1 probCD1 = dictC['[ 0. 0. 0. 0.]'] / num_samples probCD2 = dictC['[ 1. 1. 1. 1.]'] / num_samples probCS1 = dictC['[ 0. 0. 1. 1.]'] / num_samples probCS2 = dictC['[ 1. 1. 0. 0.]'] / num_samples probCS3 = dictC['[ 1. 0. 1. 0.]'] / num_samples probCS4 = dictC['[ 0. 1. 0. 1.]'] / num_samples sumProbs = probCD1 + probCD2 + probCS1 + probCS2 + probCS3 + probCS4 return [probCD1, probCD2, probCS1, probCS2, probCS3, probCS4, sumProbs] def test_Gibbs_sampler(self): sys.stdout.write('RBM Sampler -> Performing GibbsSampler test ... ') sys.stdout.flush() numx.random.seed(42) sampler = Sampler.GibbsSampler(self.bbrbm) probCD1, probCD2, probCS1, probCS2, probCS3, probCS4, sumProbs = self.execute_sampler(sampler, self.num_samples) assert numx.all(numx.abs(1.0 / 4.0 - probCD1) < self.epsilon) assert numx.all(numx.abs(1.0 / 4.0 - probCD2) < self.epsilon) assert numx.all(numx.abs(1.0 / 8.0 - probCS1) < self.epsilon) assert numx.all(numx.abs(1.0 / 8.0 - probCS2) < self.epsilon) assert numx.all(numx.abs(1.0 / 8.0 - probCS3) < self.epsilon) assert numx.all(numx.abs(1.0 / 8.0 - probCS4) < self.epsilon) assert numx.all(numx.abs(1.0 - sumProbs) < self.epsilon) print('successfully passed!') sys.stdout.flush() def test_Persistent_Gibbs_sampler(self): sys.stdout.write('RBM Sampler -> Performing PersistentGibbsSampler test ... ') sys.stdout.flush() numx.random.seed(42) sampler = Sampler.PersistentGibbsSampler(self.bbrbm, 1) probCD1, probCD2, probCS1, probCS2, probCS3, probCS4, sumProbs = self.execute_sampler(sampler, self.num_samples) assert numx.all(numx.abs(1.0 / 4.0 - probCD1) < self.epsilon) assert numx.all(numx.abs(1.0 / 4.0 - probCD2) < self.epsilon) assert numx.all(numx.abs(1.0 / 8.0 - probCS1) < self.epsilon) assert numx.all(numx.abs(1.0 / 8.0 - probCS2) < self.epsilon) assert numx.all(numx.abs(1.0 / 8.0 - probCS3) < self.epsilon) assert numx.all(numx.abs(1.0 / 8.0 - probCS4) < self.epsilon) assert numx.all(numx.abs(1.0 - sumProbs) < self.epsilon) print('successfully passed!') sys.stdout.flush() def test_Parallel_Tempering_sampler(self): sys.stdout.write('RBM Sampler -> Performing ParallelTemperingSampler test ... ') sys.stdout.flush() numx.random.seed(42) sampler = Sampler.ParallelTemperingSampler(self.bbrbm, 10) probCD1, probCD2, probCS1, probCS2, probCS3, probCS4, sumProbs = self.execute_sampler(sampler, self.num_samples) assert numx.all(numx.abs(1.0 / 4.0 - probCD1) < self.epsilon) assert numx.all(numx.abs(1.0 / 4.0 - probCD2) < self.epsilon) assert numx.all(numx.abs(1.0 / 8.0 - probCS1) < self.epsilon) assert numx.all(numx.abs(1.0 / 8.0 - probCS2) < self.epsilon) assert numx.all(numx.abs(1.0 / 8.0 - probCS3) < self.epsilon) assert numx.all(numx.abs(1.0 / 8.0 - probCS4) < self.epsilon) assert numx.all(numx.abs(1.0 - sumProbs) < self.epsilon) print('successfully passed!') sys.stdout.flush() def test_Independent_Parallel_Tempering_sampler(self): sys.stdout.write('RBM Sampler -> Performing IndependentParallelTemperingSampler test ... ') sys.stdout.flush() numx.random.seed(42) sampler = Sampler.IndependentParallelTemperingSampler(self.bbrbm, 10, 10) probCD1, probCD2, probCS1, probCS2, probCS3, probCS4, sumProbs = self.execute_sampler(sampler, self.num_samples) assert numx.all(numx.abs(1.0 / 4.0 - probCD1) < self.epsilon) assert numx.all(numx.abs(1.0 / 4.0 - probCD2) < self.epsilon) assert numx.all(numx.abs(1.0 / 8.0 - probCS1) < self.epsilon) assert numx.all(numx.abs(1.0 / 8.0 - probCS2) < self.epsilon) assert numx.all(numx.abs(1.0 / 8.0 - probCS3) < self.epsilon) assert numx.all(numx.abs(1.0 / 8.0 - probCS4) < self.epsilon) assert numx.all(numx.abs(1.0 - sumProbs) < self.epsilon) print('successfully passed!') sys.stdout.flush()
class TestBinaryBinaryRBM(unittest.TestCase): # Known RBM bbrbmData = generate_bars_and_stripes_complete(2) bbrbmData = numx.vstack((bbrbmData[0], bbrbmData, bbrbmData[5])) bbrbmw = numx.array( [[0.12179488, 2.95950177, 0.33513356, 35.05380642], [0.20318085, -28.62372894, 26.52611278, 28.41793445], [-0.19105386, -28.58530584, -26.52747507, 28.78447320], [0.08953740, -59.82556859, -0.06665933, -27.71723459]]) bbrbmbv = numx.array( [[-19.24399659, -13.26258696, 13.25909850, 43.74408543]]) bbrbmbh = numx.array( [[-0.11155958, 57.02097584, -0.13331758, -32.25991501]]) bbrbm = Model.BinaryBinaryRBM(4, 4, bbrbmData, bbrbmw, bbrbmbv, bbrbmbh, 0.0, 0.0) bbrbmTruelogZ = 59.6749019726 bbrbmTrueLL = -1.7328699078 bbrbmBestLLPossible = -1.732867951 epsilon = 0.00001 def test___init__(self): sys.stdout.write('BinaryBinaryRBM -> Performing init test ...') sys.stdout.flush() assert numx.all(self.bbrbm.bv_base == numx.array([[0, 0, 0, 0]])) print(' successfully passed!') sys.stdout.flush() def test__add_visible_units(self): sys.stdout.write( 'BinaryBinaryRBM -> Performing add_visible_units test ...') sys.stdout.flush() localmodel = copy.deepcopy(self.bbrbm) localmodel._add_visible_units(2, 3) assert numx.all(localmodel.bv_base == numx.array([[0, 0, 0, 0, 0, 0]])) print(' successfully passed!') sys.stdout.flush() def test__remove_visible_units(self): sys.stdout.write( 'BinaryBinaryRBM -> Performing remove_visible_units test ...') sys.stdout.flush() localmodel = copy.deepcopy(self.bbrbm) localmodel._remove_visible_units([0, 2]) assert numx.all(localmodel.bv_base == numx.array([[0, 0]])) print(' successfully passed!') sys.stdout.flush() def test__calculate_weight_gradient(self): sys.stdout.write( 'BinaryBinaryRBM -> Performing calculate_weight_gradient test ...') sys.stdout.flush() deltaW = self.bbrbm._calculate_weight_gradient( numx.array([[1, 1, 1, 0], [0, 1, 0, 1]]), numx.array([[0, 1, 0, 1], [0, 1, 1, 0]])) target = numx.array([[0., 1., 0., 1.], [0., 2., 1., 1.], [0., 1., 0., 1.], [0., 1., 1., 0.]]) assert numx.all(target == deltaW) print(' successfully passed!') sys.stdout.flush() def test__calculate_visible_bias_gradient(self): sys.stdout.write( 'BinaryBinaryRBM -> Performing calculate_visible_bias_gradient test ...' ) sys.stdout.flush() deltaBv = self.bbrbm._calculate_visible_bias_gradient( numx.array([[1, 1, 1, 0], [0, 1, 0, 1]])) target = numx.array([[1., 2., 1., 1.]]) assert numx.all(target == deltaBv) print(' successfully passed!') sys.stdout.flush() def test__calculate_hidden_bias_gradient(self): sys.stdout.write( 'BinaryBinaryRBM -> Performing calculate_hidden_bias_gradient test ...' ) sys.stdout.flush() deltaBh = self.bbrbm._calculate_hidden_bias_gradient( numx.array([[0, 1, 0, 1], [0, 1, 1, 0]])) target = numx.array([[0., 2., 1., 1.]]) assert numx.all(target == deltaBh) print(' successfully passed!') sys.stdout.flush() def test_calculate_gradients(self): sys.stdout.write( 'BinaryBinaryRBM -> Performing calculate_gradients test ...') sys.stdout.flush() deltaW = self.bbrbm._calculate_weight_gradient( numx.array([[1, 1, 1, 0], [0, 1, 0, 1]]), numx.array([[0, 1, 0, 1], [0, 1, 1, 0]])) deltaBv = self.bbrbm._calculate_visible_bias_gradient( numx.array([[1, 1, 1, 0], [0, 1, 0, 1]])) deltaBh = self.bbrbm._calculate_hidden_bias_gradient( numx.array([[0, 1, 0, 1], [0, 1, 1, 0]])) deltas = self.bbrbm.calculate_gradients( numx.array([[1, 1, 1, 0], [0, 1, 0, 1]]), numx.array([[0, 1, 0, 1], [0, 1, 1, 0]])) assert numx.all(deltaW == deltas[0]) assert numx.all(deltaBv == deltas[1]) assert numx.all(deltaBh == deltas[2]) print(' successfully passed!') sys.stdout.flush() def test_sample_v(self): sys.stdout.write('BinaryBinaryRBM -> Performing sample_v test ...') sys.stdout.flush() assert numx.all(self.bbrbm.sample_v(numx.ones((10000, 4))) == 1.0) assert numx.all(self.bbrbm.sample_v(numx.zeros((10000, 4))) == 0.0) numx.random.seed(42) samples = self.bbrbm.sample_v(numx.ones((10000, 4)) * 0.5) assert numx.sum(samples != 0.0) + numx.sum(samples != 1.0) == 40000 assert numx.abs(numx.sum(samples) / 40000.0 - 0.5) < 0.01 print(' successfully passed!') sys.stdout.flush() def test_sample_h(self): sys.stdout.write('BinaryBinaryRBM -> Performing sample_h test ...') sys.stdout.flush() assert numx.all(self.bbrbm.sample_h(numx.ones((10000, 4))) == 1.0) assert numx.all(self.bbrbm.sample_h(numx.zeros((10000, 4))) == 0.0) numx.random.seed(42) samples = self.bbrbm.sample_h(numx.ones((10000, 4)) * 0.5) assert numx.sum(samples != 0.0) + numx.sum(samples != 1.0) == 40000 assert numx.abs(numx.sum(samples) / 40000.0 - 0.5) < 0.01 print(' successfully passed!') sys.stdout.flush() def test_probability_v_given_h(self): sys.stdout.write( 'BinaryBinaryRBM -> Performing probability_v_given_h test ...') sys.stdout.flush() probs = self.bbrbm.probability_v_given_h(self.bbrbmData) target = numx.array( [[4.38973669e-09, 1.73832475e-06, 9.99998256e-01, 1.00000000e+00], [4.38973669e-09, 1.73832475e-06, 9.99998256e-01, 1.00000000e+00], [6.93234181e-09, 9.99998583e-01, 1.42771991e-06, 1.00000000e+00], [9.99999993e-01, 1.41499740e-06, 9.99998571e-01, 0.00000000e+00], [9.56375764e-08, 0.00000000e+00, 1.82363957e-07, 1.13445207e-07], [9.99999903e-01, 1.00000000e+00, 9.99999817e-01, 9.99999883e-01], [9.99999996e-01, 9.99998259e-01, 1.74236911e-06, 0.00000000e+00], [9.99999996e-01, 9.99998259e-01, 1.74236911e-06, 0.00000000e+00]]) assert numx.all(numx.abs(probs - target) < self.epsilon) print(' successfully passed!') sys.stdout.flush() def test_probability_h_given_v(self): sys.stdout.write( 'BinaryBinaryRBM -> Performing probability_h_given_v test ...') sys.stdout.flush() probs = self.bbrbm.probability_h_given_v(self.bbrbmData) target = numx.array( [[4.72138994e-01, 1.00000000e+00, 4.66719883e-01, 9.76996262e-15], [4.72138994e-01, 1.00000000e+00, 4.66719883e-01, 9.76996262e-15], [4.54918124e-01, 1.00000000e+00, 3.68904907e-12, 1.00000000e+00], [5.45166211e-01, 2.24265051e-14, 1.00000000e+00, 1.97064587e-14], [5.53152448e-01, 1.00000000e+00, 1.00000000e+00, 1.00000000e+00], [4.46931620e-01, 2.33146835e-14, 2.46841436e-12, 2.83661983e-14], [5.27945768e-01, 0.00000000e+00, 5.33398782e-01, 1.00000000e+00], [5.27945768e-01, 0.00000000e+00, 5.33398782e-01, 1.00000000e+00]]) assert numx.all(numx.abs(probs - target) < self.epsilon) print(' successfully passed!') sys.stdout.flush() def test_energy(self): sys.stdout.write('BinaryBinaryRBM -> Performing energy test ...') sys.stdout.flush() energies = self.bbrbm.energy(self.bbrbmData, self.bbrbmData) target = numx.array([[0.], [0.], [32.49137574], [32.50603837], [0.93641873], [0.91694445], [0.03276686], [0.03276686]]) assert numx.all(numx.abs(energies - target) < self.epsilon) print(' successfully passed!') sys.stdout.flush() def test_unnormalized_log_probability_v(self): sys.stdout.write( 'BinaryBinaryRBM -> Performing unnormalized_log_probability_v test ...' ) sys.stdout.flush() probs = self.bbrbm.unnormalized_log_probability_v(self.bbrbmData) target = numx.array([[58.28860656], [58.28860656], [57.59545755], [57.59545757], [57.59545753], [57.59545756], [58.28860656], [58.28860656]]) assert numx.all(numx.abs(probs - target) < self.epsilon) print(' successfully passed!') sys.stdout.flush() def test_unnormalized_log_probability_h(self): sys.stdout.write( 'BinaryBinaryRBM -> Performing unnormalized_log_probability_h test ...' ) sys.stdout.flush() probs = self.bbrbm.unnormalized_log_probability_h(self.bbrbmData) target = numx.array([[57.00318742], [57.00318742], [56.98879586], [56.98864114], [56.90941665], [56.90945961], [57.00333938], [57.00333938]]) assert numx.all(numx.abs(probs - target) < self.epsilon) print(' successfully passed!') sys.stdout.flush() def test_log_probability_v(self): sys.stdout.write( 'BinaryBinaryRBM -> Performing log_probability_v test ...') sys.stdout.flush() probs = self.bbrbm.log_probability_v(self.bbrbmTruelogZ, self.bbrbmData) target = numx.array([[-1.38629541], [-1.38629541], [-2.07944442], [-2.07944441], [-2.07944444], [-2.07944441], [-1.38629541], [-1.38629541]]) assert numx.all(numx.abs(probs - target) < self.epsilon) print(' successfully passed!') sys.stdout.flush() def test_log_probability_h(self): sys.stdout.write( 'BinaryBinaryRBM -> Performing log_probability_h test ...') sys.stdout.flush() probs = self.bbrbm.log_probability_h(self.bbrbmTruelogZ, self.bbrbmData) target = numx.array([[-2.67171456], [-2.67171456], [-2.68610611], [-2.68626083], [-2.76548532], [-2.76544237], [-2.67156259], [-2.67156259]]) assert numx.all(numx.abs(probs - target) < self.epsilon) print(' successfully passed!') sys.stdout.flush() def test_log_probability_v_h(self): sys.stdout.write( 'BinaryBinaryRBM -> Performing log_probability_v_h test ...') sys.stdout.flush() h = self.bbrbm.probability_h_given_v(self.bbrbmData) probs = self.bbrbm.log_probability_v_h(self.bbrbmTruelogZ, self.bbrbmData, h) target = numx.array([[-2.76881973], [-2.76881973], [-2.76852132], [-2.76850605], [-2.76693057], [-2.76694846], [-2.76879441], [-2.76879441]]) assert numx.all(numx.abs(probs - target) < self.epsilon) print(' successfully passed!') sys.stdout.flush() def test__base_log_partition(self): sys.stdout.write( 'BinaryBinaryRBM -> Performing base_log_partition test ...') sys.stdout.flush() localmodel = copy.deepcopy(self.bbrbm) localmodel.bv_base += 1.0 assert numx.abs(localmodel._base_log_partition(False) - 5.54517744448) < self.epsilon assert numx.abs(localmodel._base_log_partition(True) - 8.02563547231) < self.epsilon print(' successfully passed!') sys.stdout.flush() def test__getbasebias(self): sys.stdout.write('BinaryBinaryRBM -> Performing getbasebias test ...') sys.stdout.flush() # zero base bias when data mean is 0.5 localmodel = copy.deepcopy(self.bbrbm) localmodel._data_mean = localmodel._data_mean / 2.0 assert numx.all( numx.abs(localmodel._getbasebias() + 1.09861229) < self.epsilon) print(' successfully passed!') sys.stdout.flush()
class TestEstimator(unittest.TestCase): # Known model bbrbmData = generate_bars_and_stripes_complete(2) bbrbmData = numx.vstack((bbrbmData[0], bbrbmData, bbrbmData[5])) bbrbmw = numx.array( [[0.12179488, 2.95950177, 0.33513356, 35.05380642], [0.20318085, -28.62372894, 26.52611278, 28.41793445], [-0.19105386, -28.58530584, -26.52747507, 28.78447320], [0.08953740, -59.82556859, -0.06665933, -27.71723459]]) bbrbmbv = numx.array( [[-19.24399659, -13.26258696, 13.25909850, 43.74408543]]) bbrbmbh = numx.array( [[-0.11155958, 57.02097584, -0.13331758, -32.25991501]]) bbrbm = Model.BinaryBinaryRBM(4, 4, bbrbmData, bbrbmw, bbrbmbv, bbrbmbh, 0.0, 0.0) bbrbmTruelogZ = 59.6749019726 bbrbmTrueLL = -1.7328699078 bbrbmBestLLPossible = -1.732867951 epsilon = 0.00001 def test_reconstruction_error(self): sys.stdout.write( 'RBM Estimator -> Performing reconstruction_error test ...') sys.stdout.flush() numx.random.seed(42) rec = Estimator.reconstruction_error(self.bbrbm, self.bbrbmData, k=1, beta=1.0, use_states=True, absolut_error=False) assert numx.all(numx.abs(rec) < self.epsilon) rec = Estimator.reconstruction_error(self.bbrbm, self.bbrbmData, k=1, beta=1.0, use_states=False, absolut_error=False) assert numx.all(numx.abs(rec) < self.epsilon) rec = Estimator.reconstruction_error(self.bbrbm, self.bbrbmData, k=1, beta=1.0, use_states=True, absolut_error=True) assert numx.all(numx.abs(rec) < self.epsilon) rec = Estimator.reconstruction_error(self.bbrbm, self.bbrbmData, k=1, beta=1.0, use_states=False, absolut_error=True) assert numx.all(numx.abs(rec) < self.epsilon) rec = Estimator.reconstruction_error(self.bbrbm, self.bbrbmData, k=10, beta=1.0, use_states=False, absolut_error=False) assert numx.all(numx.abs(rec) < self.epsilon) # Test List testList = [] for i in range(self.bbrbmData.shape[0]): testList.append(self.bbrbmData[i].reshape(1, 4)) rec = Estimator.reconstruction_error(self.bbrbm, testList, k=10, beta=1.0, use_states=False, absolut_error=False) assert numx.all(numx.abs(rec) < self.epsilon) print(' successfully passed!') sys.stdout.flush() def test_log_likelihood_v(self): sys.stdout.write( 'RBM Estimator -> Performing log_likelihood_v test ...') sys.stdout.flush() numx.random.seed(42) ll = numx.mean( Estimator.log_likelihood_v(self.bbrbm, self.bbrbmTruelogZ, self.bbrbmData, 1.0)) assert numx.all(numx.abs(ll - self.bbrbmTrueLL) < self.epsilon) # Test List testList = [] for i in range(self.bbrbmData.shape[0]): testList.append(self.bbrbmData[i].reshape(1, 4)) ll = numx.mean( Estimator.log_likelihood_v(self.bbrbm, self.bbrbmTruelogZ, testList, 1.0)) assert numx.all(numx.abs(ll - self.bbrbmTrueLL) < self.epsilon) print(' successfully passed!') sys.stdout.flush() def test_log_likelihood_h(self): sys.stdout.write( 'RBM Estimator -> Performing log_likelihood_h test ...') sys.stdout.flush() numx.random.seed(42) hdata = numx.float64( self.bbrbm.probability_h_given_v(self.bbrbmData) < 0.5) ll = numx.mean( Estimator.log_likelihood_h(self.bbrbm, self.bbrbmTruelogZ, hdata, 1.0)) assert numx.all(numx.abs(ll + 9.55929166739) < self.epsilon) # Test List testList = [] for i in range(hdata.shape[0]): testList.append(hdata[i].reshape(1, 4)) ll = numx.mean( Estimator.log_likelihood_v(self.bbrbm, self.bbrbmTruelogZ, testList, 1.0)) assert numx.all(numx.abs(ll + 9.55929166739) < self.epsilon) print(' successfully passed!') sys.stdout.flush() def test_partition_function_factorize_v(self): sys.stdout.write( 'RBM Estimator -> Performing partition_function_factorize_v test ...' ) sys.stdout.flush() LogZ = Estimator.partition_function_factorize_v( self.bbrbm, beta=None, batchsize_exponent='AUTO', status=False) assert numx.all(numx.abs(LogZ - self.bbrbmTruelogZ) < self.epsilon) LogZ = Estimator.partition_function_factorize_v(self.bbrbm, beta=None, batchsize_exponent=0, status=False) assert numx.all(numx.abs(LogZ - self.bbrbmTruelogZ) < self.epsilon) LogZ = Estimator.partition_function_factorize_v(self.bbrbm, beta=None, batchsize_exponent=3, status=False) assert numx.all(numx.abs(LogZ - self.bbrbmTruelogZ) < self.epsilon) LogZ = Estimator.partition_function_factorize_v(self.bbrbm, beta=None, batchsize_exponent=555, status=False) assert numx.all(numx.abs(LogZ - self.bbrbmTruelogZ) < self.epsilon) print(' successfully passed!') sys.stdout.flush() def test_partition_function_factorize_h(self): sys.stdout.write( 'RBM Estimator -> Performing partition_function_factorize_v test ...' ) sys.stdout.flush() LogZ = Estimator.partition_function_factorize_h( self.bbrbm, beta=None, batchsize_exponent='AUTO', status=False) assert numx.all(numx.abs(LogZ - self.bbrbmTruelogZ) < self.epsilon) LogZ = Estimator.partition_function_factorize_h(self.bbrbm, beta=None, batchsize_exponent=0, status=False) assert numx.all(numx.abs(LogZ - self.bbrbmTruelogZ) < self.epsilon) LogZ = Estimator.partition_function_factorize_h(self.bbrbm, beta=None, batchsize_exponent=3, status=False) assert numx.all(numx.abs(LogZ - self.bbrbmTruelogZ) < self.epsilon) LogZ = Estimator.partition_function_factorize_h(self.bbrbm, beta=None, batchsize_exponent=555, status=False) assert numx.all(numx.abs(LogZ - self.bbrbmTruelogZ) < self.epsilon) print(' successfully passed!') sys.stdout.flush() def test_annealed_importance_sampling(self): sys.stdout.write( 'RBM Estimator -> Performing annealed_importance_sampling test ...' ) sys.stdout.flush() numx.random.seed(42) LogZ = Estimator.annealed_importance_sampling(self.bbrbm, num_chains=100, k=1, betas=100, status=False) assert numx.all(numx.abs(LogZ[0] - self.bbrbmTruelogZ) < 0.5) LogZ = Estimator.annealed_importance_sampling(self.bbrbm, num_chains=100, k=1, betas=1000, status=False) assert numx.all(numx.abs(LogZ[0] - self.bbrbmTruelogZ) < 0.05) LogZ = Estimator.annealed_importance_sampling(self.bbrbm, num_chains=100, k=1, betas=10000, status=False) assert numx.all(numx.abs(LogZ[0] - self.bbrbmTruelogZ) < 0.005) print(' successfully passed!') sys.stdout.flush() def test_reverse_annealed_importance_sampling(self): sys.stdout.write( 'RBM Estimator -> Performing reverse_annealed_importance_sampling test ...' ) sys.stdout.flush() numx.random.seed(42) LogZ = Estimator.reverse_annealed_importance_sampling(self.bbrbm, num_chains=100, k=1, betas=100, status=False) assert numx.all(numx.abs(LogZ[0] - self.bbrbmTruelogZ) < 0.5) LogZ = Estimator.reverse_annealed_importance_sampling(self.bbrbm, num_chains=100, k=1, betas=1000, status=False) assert numx.all(numx.abs(LogZ[0] - self.bbrbmTruelogZ) < 0.05) LogZ = Estimator.reverse_annealed_importance_sampling(self.bbrbm, num_chains=100, k=1, betas=10000, status=False) assert numx.all(numx.abs(LogZ[0] - self.bbrbmTruelogZ) < 0.005) print(' successfully passed!') sys.stdout.flush()
if flipped: train_data = 1 - train_data test_data = 1 - test_data print("Flipped MNIST") else: print("Normal MNIST") # Training parameters batch_size = 100 epochs = 50 # Create centered or normal model if update_offsets <= 0.0: rbm = model.BinaryBinaryRBM(number_visibles=v1 * v2, number_hiddens=h1 * h2, data=train_data, initial_visible_offsets=0.0, initial_hidden_offsets=0.0) print("Normal RBM") else: rbm = model.BinaryBinaryRBM(number_visibles=v1 * v2, number_hiddens=h1 * h2, data=train_data, initial_visible_offsets='AUTO', initial_hidden_offsets='AUTO') print("Centered RBM") # Create trainer trainer_pcd = trainer.PCD(rbm, num_chains=batch_size) # Measuring time
def train(self, data, epsilon, k=[3, 1], offset_typ='DDD', meanfield=False): #positive phase id1 = numx.dot(data - self.model.o1, self.model.W1) d3 = numx.copy(self.model.o3) d2 = numx.copy(self.model.o2) #for _ in range(k[0]): if meanfield == False: for _ in range(k[0]): d2 = Sigmoid.f(id1 + numx.dot(d3 - self.model.o3, self.model.W2.T) + self.model.b2) d2 = self.model.dtype(d2 > numx.random.random(d2.shape)) d3 = Sigmoid.f( numx.dot(d2 - self.model.o2, self.model.W2) + self.model.b3) d3 = self.model.dtype(d3 > numx.random.random(d3.shape)) else: if meanfield == True: for _ in range(k[0]): d2 = Sigmoid.f(id1 + numx.dot(d3 - self.model.o3, self.model.W2.T) + self.model.b2) d3 = Sigmoid.f( numx.dot(d2 - self.model.o2, self.model.W2) + self.model.b3) else: d2_new = Sigmoid.f(id1 + numx.dot(d3 - self.model.o3, self.model.W2.T) + self.model.b2) d3_new = Sigmoid.f( numx.dot(d2_new - self.model.o2, self.model.W2) + self.model.b3) while numx.max(numx.abs(d2_new - d2)) > meanfield or numx.max( numx.abs(d3_new - d3)) > meanfield: d2 = d2_new d3 = d3_new d2_new = Sigmoid.f( id1 + numx.dot(d3_new - self.model.o3, self.model.W2.T) + self.model.b2) d3_new = Sigmoid.f( numx.dot(d2_new - self.model.o2, self.model.W2) + self.model.b3) d2 = d2_new d3 = d3_new self.sampler.model = RBM_MODEL.BinaryBinaryRBM( number_visibles=self.model.input_dim + self.model.hidden2_dim, number_hiddens=self.model.hidden1_dim, data=None, initial_weights=numx.vstack((self.model.W1, self.model.W2.T)), initial_visible_bias=numx.hstack((self.model.b1, self.model.b3)), initial_hidden_bias=self.model.b2, initial_visible_offsets=numx.hstack( (self.model.o1, self.model.o3)), initial_hidden_offsets=self.model.o2) if isinstance(self.sampler, RBM_SAMPLER.GibbsSampler): sample = self.sampler.sample(numx.hstack((data, d3))) else: sample = self.sampler.sample(self.batch_size, k[1]) self.m2 = self.sampler.model.probability_h_given_v(sample) self.m1 = sample[:, 0:self.model.input_dim] self.m3 = sample[:, self.model.input_dim:] # Estimate new means new_o1 = 0 if offset_typ[0] is 'D': new_o1 = data.mean(axis=0) if offset_typ[0] is 'A': new_o1 = (self.m1.mean(axis=0) + data.mean(axis=0)) / 2.0 if offset_typ[0] is 'M': new_o1 = self.m1.mean(axis=0) new_o2 = 0 if offset_typ[1] is 'D': new_o2 = d2.mean(axis=0) if offset_typ[1] is 'A': new_o2 = (self.m2.mean(axis=0) + d2.mean(axis=0)) / 2.0 if offset_typ[1] is 'M': new_o2 = self.m2.mean(axis=0) new_o3 = 0 if offset_typ[2] is 'D': new_o3 = d3.mean(axis=0) if offset_typ[2] is 'A': new_o3 = (self.m3.mean(axis=0) + d3.mean(axis=0)) / 2.0 if offset_typ[2] is 'M': new_o3 = self.m3.mean(axis=0) # Reparameterize self.model.b1 += epsilon[6] * numx.dot(new_o2 - self.model.o2, self.model.W1.T) self.model.b2 += epsilon[5] * numx.dot( new_o1 - self.model.o1, self.model.W1) + epsilon[7] * numx.dot( new_o3 - self.model.o3, self.model.W2.T) self.model.b3 += epsilon[7] * numx.dot(new_o2 - self.model.o2, self.model.W2) # Shift means self.model.o1 = (1.0 - epsilon[5]) * self.model.o1 + epsilon[5] * new_o1 self.model.o2 = (1.0 - epsilon[6]) * self.model.o2 + epsilon[6] * new_o2 self.model.o3 = (1.0 - epsilon[7]) * self.model.o3 + epsilon[7] * new_o3 # Calculate gradients dW1 = (numx.dot( (data - self.model.o1).T, d2 - self.model.o2) - numx.dot( (self.m1 - self.model.o1).T, self.m2 - self.model.o2)) dW2 = (numx.dot((d2 - self.model.o2).T, d3 - self.model.o3) - numx.dot( (self.m2 - self.model.o2).T, self.m3 - self.model.o3)) db1 = (numx.sum(data - self.m1, axis=0)).reshape(1, self.model.input_dim) db2 = (numx.sum(d2 - self.m2, axis=0)).reshape(1, self.model.hidden1_dim) db3 = (numx.sum(d3 - self.m3, axis=0)).reshape(1, self.model.hidden2_dim) # Update Model self.model.W1 += epsilon[0] / self.batch_size * dW1 self.model.W2 += epsilon[1] / self.batch_size * dW2 self.model.b1 += epsilon[2] / self.batch_size * db1 self.model.b2 += epsilon[3] / self.batch_size * db2 self.model.b3 += epsilon[4] / self.batch_size * db3
def test_forward_backward_reconstruct_sampling(self): print( 'Deep Believe Network -> Performing forward backward prop test ...' ) sys.stdout.flush() numx.random.seed(42) rbm1 = MODEL.BinaryBinaryRBM(number_visibles=2, number_hiddens=4) rbm2 = MODEL.BinaryBinaryRBM(number_visibles=4, number_hiddens=2) stack = STACK.DBN([rbm1, rbm2]) forward_target = numx.array([[0.54640997, 0.8437009], [0.45359003, 0.1562991]]) backward_target = numx.array([[0.30266536, 0.52656316], [0.69733464, 0.47343684]]) rec_target = numx.array([[0.57055553, 0.23073692], [0.42944447, 0.76926308]]) assert numx.sum( numx.abs( stack.forward_propagate(numx.array([[1, 0], [0, 1]])) - forward_target) < 0.000001) assert numx.sum( numx.abs( stack.backward_propagate(numx.array([[1, 0], [0, 1]])) - backward_target) < 0.000001) assert numx.sum( numx.abs( stack.backward_propagate( stack.forward_propagate(numx.array([[1, 0], [0, 1]]))) - rec_target) < 0.000001) assert numx.sum( numx.abs( stack.reconstruct(numx.array([[1, 0], [0, 1]])) - rec_target) < 0.000001) forward_target = numx.array([[1, 1], [1, 0]]) backward_target = numx.array([[1, 1], [0, 0]]) rec_target = numx.array([[1, 0], [0, 0]]) assert numx.sum( numx.abs( stack.forward_propagate(numx.array([[1, 0], [0, 1]]), True) - forward_target) < 0.000001) assert numx.sum( numx.abs( stack.backward_propagate(numx.array([[1, 0], [0, 1]]), True) - backward_target) < 0.000001) assert numx.sum( numx.abs( stack.backward_propagate( stack.forward_propagate(numx.array([[1, 0], [0, 1]])), True) - rec_target) < 0.000001) assert numx.sum( numx.abs( stack.reconstruct(numx.array([[1, 0], [0, 1]]), True) - rec_target) < 0.000001) assert numx.sum( numx.abs( stack.sample_top_layer(1, sample=False) - numx.array([[0.81216745, 0.80223488], [0.94668029, 0.05014349] ])) < 0.000001) numx.random.seed(42) assert numx.sum( numx.abs( stack.sample_top_layer(100, sample=True) - numx.array([[0, 0], [0, 1]])) < 0.000001) numx.random.seed(42) assert numx.sum( numx.abs( stack.sample_top_layer( 100, initial_state=numx.array([[1, 1], [1, 1]]), sample=True) - numx.array([[0, 1], [1, 0]])) < 0.000001) numx.random.seed(42) assert numx.sum( numx.abs( stack.reconstruct_sample_top_layer( input_data=numx.array([[1, 0], [0, 1]]), sampling_steps=10, sample_forward_backward=False) - numx.array([[0.69733464, 0.47343684], [0.69733464, 0.47343684] ])) < 0.000001) numx.random.seed(42) assert numx.sum( numx.abs( stack.reconstruct_sample_top_layer( input_data=numx.array([[1, 1], [1, 1]]), sampling_steps=10, sample_forward_backward=True) - numx.array([[0, 0], [0, 0]])) < 0.000001) print('successfully passed!') sys.stdout.flush()