def test_allpenelties_gradients(self): ''' Checks all possible combinations (act fct., penalties, etc.) by finite differences. ''' sys.stdout.write('Auto encoder -> Performing finite differences check of all possible contractive+sparse+slow auto encoder models ...') sys.stdout.flush() data = generate_bars_and_stripes_complete(5)[0:2] data_next = generate_bars_and_stripes_complete(5)[2:4] self.check_all(data=data, epsilon=0.0001, contractive=0.1, sparseness=0.1, desired_sparseness=0.1, data_next=data_next, slowness_penalty=0.1) print(' successfully passed!') sys.stdout.flush()
class TestSampler(unittest.TestCase): bbrbmData = generate_bars_and_stripes_complete(2) bbrbmData = numx.vstack((bbrbmData[0], bbrbmData, bbrbmData[5])) bbrbmw = numx.array([[0.12179488, 2.95950177, 0.33513356, 35.05380642], [0.20318085, -28.62372894, 26.52611278, 28.41793445], [-0.19105386, -28.58530584, -26.52747507, 28.78447320], [0.08953740, -59.82556859, -0.06665933, -27.71723459]]) bbrbmbv = numx.array([[-19.24399659, -13.26258696, 13.25909850, 43.74408543]]) bbrbmbh = numx.array([[-0.11155958, 57.02097584, -0.13331758, -32.25991501]]) bbrbm = Model.BinaryBinaryRBM(4, 4, bbrbmData, bbrbmw, bbrbmbv, bbrbmbh, 0.0, 0.0) epsilon = 0.05 num_samples = 2000.0 @classmethod def execute_sampler(cls, sampler, num_samples): dictC = {'[ 0. 0. 0. 0.]': 0, # 2 '[ 1. 1. 1. 1.]': 0, # 2 '[ 0. 0. 1. 1.]': 0, # 1 '[ 1. 1. 0. 0.]': 0, # 1 '[ 1. 0. 1. 0.]': 0, # 1 '[ 0. 1. 0. 1.]': 0, # 1 '[ 0. 1. 1. 0.]': 0, '[ 1. 0. 0. 1.]': 0, '[ 0. 0. 0. 1.]': 0, '[ 0. 0. 1. 0.]': 0, '[ 0. 1. 0. 0.]': 0, '[ 1. 0. 0. 0.]': 0, '[ 0. 1. 1. 1.]': 0, '[ 1. 1. 1. 0.]': 0, '[ 1. 0. 1. 1.]': 0, '[ 1. 1. 0. 1.]': 0} for _ in range(numx.int32(num_samples)): if isinstance(sampler, Sampler.GibbsSampler): # Start form random since model is rather deterministic samples = sampler.sample(numx.random.rand(1, 4), 1, ret_states=True) else: if isinstance(sampler, Sampler.PersistentGibbsSampler): # Start form random since model is rather deterministic sampler.chains = numx.random.rand(1, 4) samples = sampler.sample(1, 1, ret_states=True) else: samples = sampler.sample(1, 1, ret_states=True) dictC[str(samples[0])] += 1 probCD1 = dictC['[ 0. 0. 0. 0.]'] / num_samples probCD2 = dictC['[ 1. 1. 1. 1.]'] / num_samples probCS1 = dictC['[ 0. 0. 1. 1.]'] / num_samples probCS2 = dictC['[ 1. 1. 0. 0.]'] / num_samples probCS3 = dictC['[ 1. 0. 1. 0.]'] / num_samples probCS4 = dictC['[ 0. 1. 0. 1.]'] / num_samples sumProbs = probCD1 + probCD2 + probCS1 + probCS2 + probCS3 + probCS4 return [probCD1, probCD2, probCS1, probCS2, probCS3, probCS4, sumProbs] def test_Gibbs_sampler(self): sys.stdout.write('RBM Sampler -> Performing GibbsSampler test ... ') sys.stdout.flush() numx.random.seed(42) sampler = Sampler.GibbsSampler(self.bbrbm) probCD1, probCD2, probCS1, probCS2, probCS3, probCS4, sumProbs = self.execute_sampler(sampler, self.num_samples) assert numx.all(numx.abs(1.0 / 4.0 - probCD1) < self.epsilon) assert numx.all(numx.abs(1.0 / 4.0 - probCD2) < self.epsilon) assert numx.all(numx.abs(1.0 / 8.0 - probCS1) < self.epsilon) assert numx.all(numx.abs(1.0 / 8.0 - probCS2) < self.epsilon) assert numx.all(numx.abs(1.0 / 8.0 - probCS3) < self.epsilon) assert numx.all(numx.abs(1.0 / 8.0 - probCS4) < self.epsilon) assert numx.all(numx.abs(1.0 - sumProbs) < self.epsilon) print('successfully passed!') sys.stdout.flush() def test_Persistent_Gibbs_sampler(self): sys.stdout.write('RBM Sampler -> Performing PersistentGibbsSampler test ... ') sys.stdout.flush() numx.random.seed(42) sampler = Sampler.PersistentGibbsSampler(self.bbrbm, 1) probCD1, probCD2, probCS1, probCS2, probCS3, probCS4, sumProbs = self.execute_sampler(sampler, self.num_samples) assert numx.all(numx.abs(1.0 / 4.0 - probCD1) < self.epsilon) assert numx.all(numx.abs(1.0 / 4.0 - probCD2) < self.epsilon) assert numx.all(numx.abs(1.0 / 8.0 - probCS1) < self.epsilon) assert numx.all(numx.abs(1.0 / 8.0 - probCS2) < self.epsilon) assert numx.all(numx.abs(1.0 / 8.0 - probCS3) < self.epsilon) assert numx.all(numx.abs(1.0 / 8.0 - probCS4) < self.epsilon) assert numx.all(numx.abs(1.0 - sumProbs) < self.epsilon) print('successfully passed!') sys.stdout.flush() def test_Parallel_Tempering_sampler(self): sys.stdout.write('RBM Sampler -> Performing ParallelTemperingSampler test ... ') sys.stdout.flush() numx.random.seed(42) sampler = Sampler.ParallelTemperingSampler(self.bbrbm, 10) probCD1, probCD2, probCS1, probCS2, probCS3, probCS4, sumProbs = self.execute_sampler(sampler, self.num_samples) assert numx.all(numx.abs(1.0 / 4.0 - probCD1) < self.epsilon) assert numx.all(numx.abs(1.0 / 4.0 - probCD2) < self.epsilon) assert numx.all(numx.abs(1.0 / 8.0 - probCS1) < self.epsilon) assert numx.all(numx.abs(1.0 / 8.0 - probCS2) < self.epsilon) assert numx.all(numx.abs(1.0 / 8.0 - probCS3) < self.epsilon) assert numx.all(numx.abs(1.0 / 8.0 - probCS4) < self.epsilon) assert numx.all(numx.abs(1.0 - sumProbs) < self.epsilon) print('successfully passed!') sys.stdout.flush() def test_Independent_Parallel_Tempering_sampler(self): sys.stdout.write('RBM Sampler -> Performing IndependentParallelTemperingSampler test ... ') sys.stdout.flush() numx.random.seed(42) sampler = Sampler.IndependentParallelTemperingSampler(self.bbrbm, 10, 10) probCD1, probCD2, probCS1, probCS2, probCS3, probCS4, sumProbs = self.execute_sampler(sampler, self.num_samples) assert numx.all(numx.abs(1.0 / 4.0 - probCD1) < self.epsilon) assert numx.all(numx.abs(1.0 / 4.0 - probCD2) < self.epsilon) assert numx.all(numx.abs(1.0 / 8.0 - probCS1) < self.epsilon) assert numx.all(numx.abs(1.0 / 8.0 - probCS2) < self.epsilon) assert numx.all(numx.abs(1.0 / 8.0 - probCS3) < self.epsilon) assert numx.all(numx.abs(1.0 / 8.0 - probCS4) < self.epsilon) assert numx.all(numx.abs(1.0 - sumProbs) < self.epsilon) print('successfully passed!') sys.stdout.flush()
class TestBinaryBinaryRBM(unittest.TestCase): # Known RBM bbrbmData = generate_bars_and_stripes_complete(2) bbrbmData = numx.vstack((bbrbmData[0], bbrbmData, bbrbmData[5])) bbrbmw = numx.array( [[0.12179488, 2.95950177, 0.33513356, 35.05380642], [0.20318085, -28.62372894, 26.52611278, 28.41793445], [-0.19105386, -28.58530584, -26.52747507, 28.78447320], [0.08953740, -59.82556859, -0.06665933, -27.71723459]]) bbrbmbv = numx.array( [[-19.24399659, -13.26258696, 13.25909850, 43.74408543]]) bbrbmbh = numx.array( [[-0.11155958, 57.02097584, -0.13331758, -32.25991501]]) bbrbm = Model.BinaryBinaryRBM(4, 4, bbrbmData, bbrbmw, bbrbmbv, bbrbmbh, 0.0, 0.0) bbrbmTruelogZ = 59.6749019726 bbrbmTrueLL = -1.7328699078 bbrbmBestLLPossible = -1.732867951 epsilon = 0.00001 def test___init__(self): sys.stdout.write('BinaryBinaryRBM -> Performing init test ...') sys.stdout.flush() assert numx.all(self.bbrbm.bv_base == numx.array([[0, 0, 0, 0]])) print(' successfully passed!') sys.stdout.flush() def test__add_visible_units(self): sys.stdout.write( 'BinaryBinaryRBM -> Performing add_visible_units test ...') sys.stdout.flush() localmodel = copy.deepcopy(self.bbrbm) localmodel._add_visible_units(2, 3) assert numx.all(localmodel.bv_base == numx.array([[0, 0, 0, 0, 0, 0]])) print(' successfully passed!') sys.stdout.flush() def test__remove_visible_units(self): sys.stdout.write( 'BinaryBinaryRBM -> Performing remove_visible_units test ...') sys.stdout.flush() localmodel = copy.deepcopy(self.bbrbm) localmodel._remove_visible_units([0, 2]) assert numx.all(localmodel.bv_base == numx.array([[0, 0]])) print(' successfully passed!') sys.stdout.flush() def test__calculate_weight_gradient(self): sys.stdout.write( 'BinaryBinaryRBM -> Performing calculate_weight_gradient test ...') sys.stdout.flush() deltaW = self.bbrbm._calculate_weight_gradient( numx.array([[1, 1, 1, 0], [0, 1, 0, 1]]), numx.array([[0, 1, 0, 1], [0, 1, 1, 0]])) target = numx.array([[0., 1., 0., 1.], [0., 2., 1., 1.], [0., 1., 0., 1.], [0., 1., 1., 0.]]) assert numx.all(target == deltaW) print(' successfully passed!') sys.stdout.flush() def test__calculate_visible_bias_gradient(self): sys.stdout.write( 'BinaryBinaryRBM -> Performing calculate_visible_bias_gradient test ...' ) sys.stdout.flush() deltaBv = self.bbrbm._calculate_visible_bias_gradient( numx.array([[1, 1, 1, 0], [0, 1, 0, 1]])) target = numx.array([[1., 2., 1., 1.]]) assert numx.all(target == deltaBv) print(' successfully passed!') sys.stdout.flush() def test__calculate_hidden_bias_gradient(self): sys.stdout.write( 'BinaryBinaryRBM -> Performing calculate_hidden_bias_gradient test ...' ) sys.stdout.flush() deltaBh = self.bbrbm._calculate_hidden_bias_gradient( numx.array([[0, 1, 0, 1], [0, 1, 1, 0]])) target = numx.array([[0., 2., 1., 1.]]) assert numx.all(target == deltaBh) print(' successfully passed!') sys.stdout.flush() def test_calculate_gradients(self): sys.stdout.write( 'BinaryBinaryRBM -> Performing calculate_gradients test ...') sys.stdout.flush() deltaW = self.bbrbm._calculate_weight_gradient( numx.array([[1, 1, 1, 0], [0, 1, 0, 1]]), numx.array([[0, 1, 0, 1], [0, 1, 1, 0]])) deltaBv = self.bbrbm._calculate_visible_bias_gradient( numx.array([[1, 1, 1, 0], [0, 1, 0, 1]])) deltaBh = self.bbrbm._calculate_hidden_bias_gradient( numx.array([[0, 1, 0, 1], [0, 1, 1, 0]])) deltas = self.bbrbm.calculate_gradients( numx.array([[1, 1, 1, 0], [0, 1, 0, 1]]), numx.array([[0, 1, 0, 1], [0, 1, 1, 0]])) assert numx.all(deltaW == deltas[0]) assert numx.all(deltaBv == deltas[1]) assert numx.all(deltaBh == deltas[2]) print(' successfully passed!') sys.stdout.flush() def test_sample_v(self): sys.stdout.write('BinaryBinaryRBM -> Performing sample_v test ...') sys.stdout.flush() assert numx.all(self.bbrbm.sample_v(numx.ones((10000, 4))) == 1.0) assert numx.all(self.bbrbm.sample_v(numx.zeros((10000, 4))) == 0.0) numx.random.seed(42) samples = self.bbrbm.sample_v(numx.ones((10000, 4)) * 0.5) assert numx.sum(samples != 0.0) + numx.sum(samples != 1.0) == 40000 assert numx.abs(numx.sum(samples) / 40000.0 - 0.5) < 0.01 print(' successfully passed!') sys.stdout.flush() def test_sample_h(self): sys.stdout.write('BinaryBinaryRBM -> Performing sample_h test ...') sys.stdout.flush() assert numx.all(self.bbrbm.sample_h(numx.ones((10000, 4))) == 1.0) assert numx.all(self.bbrbm.sample_h(numx.zeros((10000, 4))) == 0.0) numx.random.seed(42) samples = self.bbrbm.sample_h(numx.ones((10000, 4)) * 0.5) assert numx.sum(samples != 0.0) + numx.sum(samples != 1.0) == 40000 assert numx.abs(numx.sum(samples) / 40000.0 - 0.5) < 0.01 print(' successfully passed!') sys.stdout.flush() def test_probability_v_given_h(self): sys.stdout.write( 'BinaryBinaryRBM -> Performing probability_v_given_h test ...') sys.stdout.flush() probs = self.bbrbm.probability_v_given_h(self.bbrbmData) target = numx.array( [[4.38973669e-09, 1.73832475e-06, 9.99998256e-01, 1.00000000e+00], [4.38973669e-09, 1.73832475e-06, 9.99998256e-01, 1.00000000e+00], [6.93234181e-09, 9.99998583e-01, 1.42771991e-06, 1.00000000e+00], [9.99999993e-01, 1.41499740e-06, 9.99998571e-01, 0.00000000e+00], [9.56375764e-08, 0.00000000e+00, 1.82363957e-07, 1.13445207e-07], [9.99999903e-01, 1.00000000e+00, 9.99999817e-01, 9.99999883e-01], [9.99999996e-01, 9.99998259e-01, 1.74236911e-06, 0.00000000e+00], [9.99999996e-01, 9.99998259e-01, 1.74236911e-06, 0.00000000e+00]]) assert numx.all(numx.abs(probs - target) < self.epsilon) print(' successfully passed!') sys.stdout.flush() def test_probability_h_given_v(self): sys.stdout.write( 'BinaryBinaryRBM -> Performing probability_h_given_v test ...') sys.stdout.flush() probs = self.bbrbm.probability_h_given_v(self.bbrbmData) target = numx.array( [[4.72138994e-01, 1.00000000e+00, 4.66719883e-01, 9.76996262e-15], [4.72138994e-01, 1.00000000e+00, 4.66719883e-01, 9.76996262e-15], [4.54918124e-01, 1.00000000e+00, 3.68904907e-12, 1.00000000e+00], [5.45166211e-01, 2.24265051e-14, 1.00000000e+00, 1.97064587e-14], [5.53152448e-01, 1.00000000e+00, 1.00000000e+00, 1.00000000e+00], [4.46931620e-01, 2.33146835e-14, 2.46841436e-12, 2.83661983e-14], [5.27945768e-01, 0.00000000e+00, 5.33398782e-01, 1.00000000e+00], [5.27945768e-01, 0.00000000e+00, 5.33398782e-01, 1.00000000e+00]]) assert numx.all(numx.abs(probs - target) < self.epsilon) print(' successfully passed!') sys.stdout.flush() def test_energy(self): sys.stdout.write('BinaryBinaryRBM -> Performing energy test ...') sys.stdout.flush() energies = self.bbrbm.energy(self.bbrbmData, self.bbrbmData) target = numx.array([[0.], [0.], [32.49137574], [32.50603837], [0.93641873], [0.91694445], [0.03276686], [0.03276686]]) assert numx.all(numx.abs(energies - target) < self.epsilon) print(' successfully passed!') sys.stdout.flush() def test_unnormalized_log_probability_v(self): sys.stdout.write( 'BinaryBinaryRBM -> Performing unnormalized_log_probability_v test ...' ) sys.stdout.flush() probs = self.bbrbm.unnormalized_log_probability_v(self.bbrbmData) target = numx.array([[58.28860656], [58.28860656], [57.59545755], [57.59545757], [57.59545753], [57.59545756], [58.28860656], [58.28860656]]) assert numx.all(numx.abs(probs - target) < self.epsilon) print(' successfully passed!') sys.stdout.flush() def test_unnormalized_log_probability_h(self): sys.stdout.write( 'BinaryBinaryRBM -> Performing unnormalized_log_probability_h test ...' ) sys.stdout.flush() probs = self.bbrbm.unnormalized_log_probability_h(self.bbrbmData) target = numx.array([[57.00318742], [57.00318742], [56.98879586], [56.98864114], [56.90941665], [56.90945961], [57.00333938], [57.00333938]]) assert numx.all(numx.abs(probs - target) < self.epsilon) print(' successfully passed!') sys.stdout.flush() def test_log_probability_v(self): sys.stdout.write( 'BinaryBinaryRBM -> Performing log_probability_v test ...') sys.stdout.flush() probs = self.bbrbm.log_probability_v(self.bbrbmTruelogZ, self.bbrbmData) target = numx.array([[-1.38629541], [-1.38629541], [-2.07944442], [-2.07944441], [-2.07944444], [-2.07944441], [-1.38629541], [-1.38629541]]) assert numx.all(numx.abs(probs - target) < self.epsilon) print(' successfully passed!') sys.stdout.flush() def test_log_probability_h(self): sys.stdout.write( 'BinaryBinaryRBM -> Performing log_probability_h test ...') sys.stdout.flush() probs = self.bbrbm.log_probability_h(self.bbrbmTruelogZ, self.bbrbmData) target = numx.array([[-2.67171456], [-2.67171456], [-2.68610611], [-2.68626083], [-2.76548532], [-2.76544237], [-2.67156259], [-2.67156259]]) assert numx.all(numx.abs(probs - target) < self.epsilon) print(' successfully passed!') sys.stdout.flush() def test_log_probability_v_h(self): sys.stdout.write( 'BinaryBinaryRBM -> Performing log_probability_v_h test ...') sys.stdout.flush() h = self.bbrbm.probability_h_given_v(self.bbrbmData) probs = self.bbrbm.log_probability_v_h(self.bbrbmTruelogZ, self.bbrbmData, h) target = numx.array([[-2.76881973], [-2.76881973], [-2.76852132], [-2.76850605], [-2.76693057], [-2.76694846], [-2.76879441], [-2.76879441]]) assert numx.all(numx.abs(probs - target) < self.epsilon) print(' successfully passed!') sys.stdout.flush() def test__base_log_partition(self): sys.stdout.write( 'BinaryBinaryRBM -> Performing base_log_partition test ...') sys.stdout.flush() localmodel = copy.deepcopy(self.bbrbm) localmodel.bv_base += 1.0 assert numx.abs(localmodel._base_log_partition(False) - 5.54517744448) < self.epsilon assert numx.abs(localmodel._base_log_partition(True) - 8.02563547231) < self.epsilon print(' successfully passed!') sys.stdout.flush() def test__getbasebias(self): sys.stdout.write('BinaryBinaryRBM -> Performing getbasebias test ...') sys.stdout.flush() # zero base bias when data mean is 0.5 localmodel = copy.deepcopy(self.bbrbm) localmodel._data_mean = localmodel._data_mean / 2.0 assert numx.all( numx.abs(localmodel._getbasebias() + 1.09861229) < self.epsilon) print(' successfully passed!') sys.stdout.flush()
class TestEstimator(unittest.TestCase): # Known model bbrbmData = generate_bars_and_stripes_complete(2) bbrbmData = numx.vstack((bbrbmData[0], bbrbmData, bbrbmData[5])) bbrbmw = numx.array( [[0.12179488, 2.95950177, 0.33513356, 35.05380642], [0.20318085, -28.62372894, 26.52611278, 28.41793445], [-0.19105386, -28.58530584, -26.52747507, 28.78447320], [0.08953740, -59.82556859, -0.06665933, -27.71723459]]) bbrbmbv = numx.array( [[-19.24399659, -13.26258696, 13.25909850, 43.74408543]]) bbrbmbh = numx.array( [[-0.11155958, 57.02097584, -0.13331758, -32.25991501]]) bbrbm = Model.BinaryBinaryRBM(4, 4, bbrbmData, bbrbmw, bbrbmbv, bbrbmbh, 0.0, 0.0) bbrbmTruelogZ = 59.6749019726 bbrbmTrueLL = -1.7328699078 bbrbmBestLLPossible = -1.732867951 epsilon = 0.00001 def test_reconstruction_error(self): sys.stdout.write( 'RBM Estimator -> Performing reconstruction_error test ...') sys.stdout.flush() numx.random.seed(42) rec = Estimator.reconstruction_error(self.bbrbm, self.bbrbmData, k=1, beta=1.0, use_states=True, absolut_error=False) assert numx.all(numx.abs(rec) < self.epsilon) rec = Estimator.reconstruction_error(self.bbrbm, self.bbrbmData, k=1, beta=1.0, use_states=False, absolut_error=False) assert numx.all(numx.abs(rec) < self.epsilon) rec = Estimator.reconstruction_error(self.bbrbm, self.bbrbmData, k=1, beta=1.0, use_states=True, absolut_error=True) assert numx.all(numx.abs(rec) < self.epsilon) rec = Estimator.reconstruction_error(self.bbrbm, self.bbrbmData, k=1, beta=1.0, use_states=False, absolut_error=True) assert numx.all(numx.abs(rec) < self.epsilon) rec = Estimator.reconstruction_error(self.bbrbm, self.bbrbmData, k=10, beta=1.0, use_states=False, absolut_error=False) assert numx.all(numx.abs(rec) < self.epsilon) # Test List testList = [] for i in range(self.bbrbmData.shape[0]): testList.append(self.bbrbmData[i].reshape(1, 4)) rec = Estimator.reconstruction_error(self.bbrbm, testList, k=10, beta=1.0, use_states=False, absolut_error=False) assert numx.all(numx.abs(rec) < self.epsilon) print(' successfully passed!') sys.stdout.flush() def test_log_likelihood_v(self): sys.stdout.write( 'RBM Estimator -> Performing log_likelihood_v test ...') sys.stdout.flush() numx.random.seed(42) ll = numx.mean( Estimator.log_likelihood_v(self.bbrbm, self.bbrbmTruelogZ, self.bbrbmData, 1.0)) assert numx.all(numx.abs(ll - self.bbrbmTrueLL) < self.epsilon) # Test List testList = [] for i in range(self.bbrbmData.shape[0]): testList.append(self.bbrbmData[i].reshape(1, 4)) ll = numx.mean( Estimator.log_likelihood_v(self.bbrbm, self.bbrbmTruelogZ, testList, 1.0)) assert numx.all(numx.abs(ll - self.bbrbmTrueLL) < self.epsilon) print(' successfully passed!') sys.stdout.flush() def test_log_likelihood_h(self): sys.stdout.write( 'RBM Estimator -> Performing log_likelihood_h test ...') sys.stdout.flush() numx.random.seed(42) hdata = numx.float64( self.bbrbm.probability_h_given_v(self.bbrbmData) < 0.5) ll = numx.mean( Estimator.log_likelihood_h(self.bbrbm, self.bbrbmTruelogZ, hdata, 1.0)) assert numx.all(numx.abs(ll + 9.55929166739) < self.epsilon) # Test List testList = [] for i in range(hdata.shape[0]): testList.append(hdata[i].reshape(1, 4)) ll = numx.mean( Estimator.log_likelihood_v(self.bbrbm, self.bbrbmTruelogZ, testList, 1.0)) assert numx.all(numx.abs(ll + 9.55929166739) < self.epsilon) print(' successfully passed!') sys.stdout.flush() def test_partition_function_factorize_v(self): sys.stdout.write( 'RBM Estimator -> Performing partition_function_factorize_v test ...' ) sys.stdout.flush() LogZ = Estimator.partition_function_factorize_v( self.bbrbm, beta=None, batchsize_exponent='AUTO', status=False) assert numx.all(numx.abs(LogZ - self.bbrbmTruelogZ) < self.epsilon) LogZ = Estimator.partition_function_factorize_v(self.bbrbm, beta=None, batchsize_exponent=0, status=False) assert numx.all(numx.abs(LogZ - self.bbrbmTruelogZ) < self.epsilon) LogZ = Estimator.partition_function_factorize_v(self.bbrbm, beta=None, batchsize_exponent=3, status=False) assert numx.all(numx.abs(LogZ - self.bbrbmTruelogZ) < self.epsilon) LogZ = Estimator.partition_function_factorize_v(self.bbrbm, beta=None, batchsize_exponent=555, status=False) assert numx.all(numx.abs(LogZ - self.bbrbmTruelogZ) < self.epsilon) print(' successfully passed!') sys.stdout.flush() def test_partition_function_factorize_h(self): sys.stdout.write( 'RBM Estimator -> Performing partition_function_factorize_v test ...' ) sys.stdout.flush() LogZ = Estimator.partition_function_factorize_h( self.bbrbm, beta=None, batchsize_exponent='AUTO', status=False) assert numx.all(numx.abs(LogZ - self.bbrbmTruelogZ) < self.epsilon) LogZ = Estimator.partition_function_factorize_h(self.bbrbm, beta=None, batchsize_exponent=0, status=False) assert numx.all(numx.abs(LogZ - self.bbrbmTruelogZ) < self.epsilon) LogZ = Estimator.partition_function_factorize_h(self.bbrbm, beta=None, batchsize_exponent=3, status=False) assert numx.all(numx.abs(LogZ - self.bbrbmTruelogZ) < self.epsilon) LogZ = Estimator.partition_function_factorize_h(self.bbrbm, beta=None, batchsize_exponent=555, status=False) assert numx.all(numx.abs(LogZ - self.bbrbmTruelogZ) < self.epsilon) print(' successfully passed!') sys.stdout.flush() def test_annealed_importance_sampling(self): sys.stdout.write( 'RBM Estimator -> Performing annealed_importance_sampling test ...' ) sys.stdout.flush() numx.random.seed(42) LogZ = Estimator.annealed_importance_sampling(self.bbrbm, num_chains=100, k=1, betas=100, status=False) assert numx.all(numx.abs(LogZ[0] - self.bbrbmTruelogZ) < 0.5) LogZ = Estimator.annealed_importance_sampling(self.bbrbm, num_chains=100, k=1, betas=1000, status=False) assert numx.all(numx.abs(LogZ[0] - self.bbrbmTruelogZ) < 0.05) LogZ = Estimator.annealed_importance_sampling(self.bbrbm, num_chains=100, k=1, betas=10000, status=False) assert numx.all(numx.abs(LogZ[0] - self.bbrbmTruelogZ) < 0.005) print(' successfully passed!') sys.stdout.flush() def test_reverse_annealed_importance_sampling(self): sys.stdout.write( 'RBM Estimator -> Performing reverse_annealed_importance_sampling test ...' ) sys.stdout.flush() numx.random.seed(42) LogZ = Estimator.reverse_annealed_importance_sampling(self.bbrbm, num_chains=100, k=1, betas=100, status=False) assert numx.all(numx.abs(LogZ[0] - self.bbrbmTruelogZ) < 0.5) LogZ = Estimator.reverse_annealed_importance_sampling(self.bbrbm, num_chains=100, k=1, betas=1000, status=False) assert numx.all(numx.abs(LogZ[0] - self.bbrbmTruelogZ) < 0.05) LogZ = Estimator.reverse_annealed_importance_sampling(self.bbrbm, num_chains=100, k=1, betas=10000, status=False) assert numx.all(numx.abs(LogZ[0] - self.bbrbmTruelogZ) < 0.005) print(' successfully passed!') sys.stdout.flush()
import pydeep.dbm.binary3Layer.trainer as TRAINER import pydeep.dbm.binary3Layer.estimator as ESTIMATOR from pydeep.base.activationfunction import Sigmoid import pydeep.misc.toyproblems as TOY import pydeep.misc.visualization as VIS # Set the same seed value for all algorithms numx.random.seed(42) # Set dimensions v11 = v12 = 2 v21 = v22 = 4 v31 = v32 = 2 # Generate data train_set = TOY.generate_bars_and_stripes_complete(v11) N = v11 * v12 M = v21 * v22 O = v31 * v32 # Training parameters batch_size = train_set.shape[0] epochs = 100000 k_pos = 3 k_neg = 5 epsilon = 0.005 * numx.array([1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0]) offset_typ = 'DDD' dbm = MODEL.BinaryBinaryDBM(N, M, O, offset_typ, train_set)
def test_trainer(self): ''' Checks if Auto encoder converges in terms of rec error. ''' sys.stdout.write( 'Auto encoder -> Performing trainer convergences check ...') sys.stdout.flush() data = generate_bars_and_stripes_complete(4) data_next = numx.random.permutation( generate_bars_and_stripes_complete(4)) for act_out in [ AFct.Identity, AFct.SoftSign, AFct.Rectifier, AFct.SoftPlus, AFct.Sigmoid, AFct.HyperbolicTangent ]: for act_in in [AFct.Sigmoid]: ae = MODEL.AutoEncoder(number_visibles=16, number_hiddens=20, data=data, visible_activation_function=act_in, hidden_activation_function=act_out, cost_function=CFct.CrossEntropyError, initial_weights='AUTO', initial_visible_bias='AUTO', initial_hidden_bias='AUTO', initial_visible_offsets='AUTO', initial_hidden_offsets='AUTO') self.perform_training(ae=ae, data=data, epsilon=0.01, momentum=0.0, update_visible_offsets=0.0, update_hidden_offsets=0.0, corruptor=None, reg_L1Norm=0.0, reg_L2Norm=0.0, reg_sparseness=0.0, desired_sparseness=0.0, reg_contractive=0.0, reg_slowness=0.0, data_next=None, restrict_gradient=0.0, restriction_norm='Cols') for act_out in [ AFct.Identity, AFct.SoftSign, AFct.Rectifier, AFct.SoftPlus, AFct.Sigmoid, AFct.HyperbolicTangent ]: for act_in in [ AFct.Identity, AFct.SoftSign, AFct.Rectifier, AFct.SoftPlus, AFct.Sigmoid, AFct.HyperbolicTangent ]: ae = MODEL.AutoEncoder(number_visibles=16, number_hiddens=20, data=data, visible_activation_function=act_in, hidden_activation_function=act_out, cost_function=CFct.SquaredError, initial_weights='AUTO', initial_visible_bias='AUTO', initial_hidden_bias='AUTO', initial_visible_offsets='AUTO', initial_hidden_offsets='AUTO') self.perform_training(ae=ae, data=data, epsilon=0.01, momentum=0.0, update_visible_offsets=0.0, update_hidden_offsets=0.0, corruptor=None, reg_L1Norm=0.0, reg_L2Norm=0.0, reg_sparseness=0.0, desired_sparseness=0.0, reg_contractive=0.0, reg_slowness=0.0, data_next=None, restrict_gradient=0.0, restriction_norm='Cols') for act_out in [ AFct.Identity, AFct.SoftSign, AFct.Rectifier, AFct.SoftPlus, AFct.Sigmoid, AFct.HyperbolicTangent ]: for act_in in [ AFct.Identity, AFct.SoftSign, AFct.Rectifier, AFct.SoftPlus, AFct.Sigmoid, AFct.HyperbolicTangent ]: ae = MODEL.AutoEncoder(number_visibles=16, number_hiddens=20, data=data, visible_activation_function=act_in, hidden_activation_function=act_out, cost_function=CFct.AbsoluteError, initial_weights='AUTO', initial_visible_bias='AUTO', initial_hidden_bias='AUTO', initial_visible_offsets='AUTO', initial_hidden_offsets='AUTO') self.perform_training(ae=ae, data=data, epsilon=0.005, momentum=0.0, update_visible_offsets=0.0, update_hidden_offsets=0.0, corruptor=None, reg_L1Norm=0.0, reg_L2Norm=0.0, reg_sparseness=0.0, desired_sparseness=0.0, reg_contractive=0.0, reg_slowness=0.0, data_next=None, restrict_gradient=0.0, restriction_norm='Cols') # Normal ae = MODEL.AutoEncoder(number_visibles=16, number_hiddens=20, data=None, visible_activation_function=AFct.Sigmoid, hidden_activation_function=AFct.Sigmoid, cost_function=CFct.CrossEntropyError, initial_weights='AUTO', initial_visible_bias='AUTO', initial_hidden_bias='AUTO', initial_visible_offsets=0, initial_hidden_offsets=0, dtype=numx.float64) self.perform_training(ae=ae, data=data, epsilon=0.01, momentum=0.0, update_visible_offsets=0.0, update_hidden_offsets=0.0, corruptor=None, reg_L1Norm=0.000, reg_L2Norm=0.000, reg_sparseness=0.0, desired_sparseness=0.1, reg_contractive=0.0, reg_slowness=0.0, data_next=data_next, restrict_gradient=None, restriction_norm='Mat') # Centered ae = MODEL.AutoEncoder(number_visibles=16, number_hiddens=20, data=data, visible_activation_function=AFct.Sigmoid, hidden_activation_function=AFct.Sigmoid, cost_function=CFct.CrossEntropyError, initial_weights='AUTO', initial_visible_bias='AUTO', initial_hidden_bias='AUTO', initial_visible_offsets='AUTO', initial_hidden_offsets='AUTO', dtype=numx.float64) self.perform_training(ae=ae, data=data, epsilon=0.01, momentum=0.0, update_visible_offsets=0.01, update_hidden_offsets=0.01, corruptor=None, reg_L1Norm=0.000, reg_L2Norm=0.000, reg_sparseness=0.0, desired_sparseness=0.1, reg_contractive=0.0, reg_slowness=0.0, data_next=data_next, restrict_gradient=None, restriction_norm='Mat') # Momentum ae = MODEL.AutoEncoder(number_visibles=16, number_hiddens=20, data=None, visible_activation_function=AFct.Sigmoid, hidden_activation_function=AFct.Sigmoid, cost_function=CFct.CrossEntropyError, initial_weights='AUTO', initial_visible_bias='AUTO', initial_hidden_bias='AUTO', initial_visible_offsets='AUTO', initial_hidden_offsets='AUTO', dtype=numx.float64) self.perform_training(ae=ae, data=data, epsilon=0.01, momentum=0.9, update_visible_offsets=0.01, update_hidden_offsets=0.01, corruptor=None, reg_L1Norm=0.000, reg_L2Norm=0.000, reg_sparseness=0.0, desired_sparseness=0.1, reg_contractive=0.0, reg_slowness=0.0, data_next=data_next, restrict_gradient=None, restriction_norm='Mat') # L1 L2 Norm ae = MODEL.AutoEncoder(number_visibles=16, number_hiddens=20, data=data, visible_activation_function=AFct.Sigmoid, hidden_activation_function=AFct.Sigmoid, cost_function=CFct.CrossEntropyError, initial_weights='AUTO', initial_visible_bias='AUTO', initial_hidden_bias='AUTO', initial_visible_offsets='AUTO', initial_hidden_offsets='AUTO', dtype=numx.float64) self.perform_training(ae=ae, data=data, epsilon=0.01, momentum=0.0, update_visible_offsets=0.01, update_hidden_offsets=0.01, corruptor=None, reg_L1Norm=0.0002, reg_L2Norm=0.0002, reg_sparseness=0.0, desired_sparseness=0.1, reg_contractive=0.0, reg_slowness=0.0, data_next=data_next, restrict_gradient=None, restriction_norm='Mat') # Sparse ae = MODEL.AutoEncoder(number_visibles=16, number_hiddens=20, data=data, visible_activation_function=AFct.Sigmoid, hidden_activation_function=AFct.Sigmoid, cost_function=CFct.CrossEntropyError, initial_weights='AUTO', initial_visible_bias='AUTO', initial_hidden_bias='AUTO', initial_visible_offsets='AUTO', initial_hidden_offsets='AUTO', dtype=numx.float64) self.perform_training(ae=ae, data=data, epsilon=0.01, momentum=0.0, update_visible_offsets=0.01, update_hidden_offsets=0.01, corruptor=None, reg_L1Norm=0.0, reg_L2Norm=0.0, reg_sparseness=0.1, desired_sparseness=0.1, reg_contractive=0.0, reg_slowness=0.0, data_next=data_next, restrict_gradient=None, restriction_norm='Mat') # Contractive ae = MODEL.AutoEncoder(number_visibles=16, number_hiddens=20, data=data, visible_activation_function=AFct.Sigmoid, hidden_activation_function=AFct.Sigmoid, cost_function=CFct.CrossEntropyError, initial_weights='AUTO', initial_visible_bias='AUTO', initial_hidden_bias='AUTO', initial_visible_offsets='AUTO', initial_hidden_offsets='AUTO', dtype=numx.float64) self.perform_training(ae=ae, data=data, epsilon=0.01, momentum=0.0, update_visible_offsets=0.01, update_hidden_offsets=0.01, corruptor=None, reg_L1Norm=0.0, reg_L2Norm=0.0, reg_sparseness=0.0, desired_sparseness=0.1, reg_contractive=0.1, reg_slowness=0.0, data_next=data_next, restrict_gradient=None, restriction_norm='Mat') # Slowness ae = MODEL.AutoEncoder(number_visibles=16, number_hiddens=20, data=data, visible_activation_function=AFct.Sigmoid, hidden_activation_function=AFct.Sigmoid, cost_function=CFct.CrossEntropyError, initial_weights='AUTO', initial_visible_bias='AUTO', initial_hidden_bias='AUTO', initial_visible_offsets='AUTO', initial_hidden_offsets='AUTO', dtype=numx.float64) self.perform_training(ae=ae, data=data, epsilon=0.01, momentum=0.0, update_visible_offsets=0.01, update_hidden_offsets=0.01, corruptor=None, reg_L1Norm=0.0, reg_L2Norm=0.0, reg_sparseness=0.0, desired_sparseness=0.0, reg_contractive=0.0, reg_slowness=0.1, data_next=data_next, restrict_gradient=None, restriction_norm='Mat') # Restrict Mat ae = MODEL.AutoEncoder(number_visibles=16, number_hiddens=20, data=data, visible_activation_function=AFct.Sigmoid, hidden_activation_function=AFct.Sigmoid, cost_function=CFct.CrossEntropyError, initial_weights='AUTO', initial_visible_bias='AUTO', initial_hidden_bias='AUTO', initial_visible_offsets='AUTO', initial_hidden_offsets='AUTO', dtype=numx.float64) self.perform_training(ae=ae, data=data, epsilon=0.01, momentum=0.0, update_visible_offsets=0.01, update_hidden_offsets=0.01, corruptor=None, reg_L1Norm=0.0, reg_L2Norm=0.0, reg_sparseness=0.0, desired_sparseness=0.0, reg_contractive=0.0, reg_slowness=0.0, data_next=data_next, restrict_gradient=0.1, restriction_norm='Mat') # Restrict rows ae = MODEL.AutoEncoder(number_visibles=16, number_hiddens=20, data=data, visible_activation_function=AFct.Sigmoid, hidden_activation_function=AFct.Sigmoid, cost_function=CFct.CrossEntropyError, initial_weights='AUTO', initial_visible_bias='AUTO', initial_hidden_bias='AUTO', initial_visible_offsets='AUTO', initial_hidden_offsets='AUTO', dtype=numx.float64) self.perform_training(ae=ae, data=data, epsilon=0.01, momentum=0.0, update_visible_offsets=0.01, update_hidden_offsets=0.01, corruptor=None, reg_L1Norm=0.0, reg_L2Norm=0.0, reg_sparseness=0.0, desired_sparseness=0.0, reg_contractive=0.0, reg_slowness=0.0, data_next=data_next, restrict_gradient=0.1, restriction_norm='Rows') # Restrict Cols ae = MODEL.AutoEncoder(number_visibles=16, number_hiddens=20, data=data, visible_activation_function=AFct.Sigmoid, hidden_activation_function=AFct.Sigmoid, cost_function=CFct.CrossEntropyError, initial_weights='AUTO', initial_visible_bias='AUTO', initial_hidden_bias='AUTO', initial_visible_offsets='AUTO', initial_hidden_offsets='AUTO', dtype=numx.float64) self.perform_training(ae=ae, data=data, epsilon=0.01, momentum=0.0, update_visible_offsets=0.01, update_hidden_offsets=0.01, corruptor=None, reg_L1Norm=0.0, reg_L2Norm=0.0, reg_sparseness=0.0, desired_sparseness=0.0, reg_contractive=0.0, reg_slowness=0.0, data_next=data_next, restrict_gradient=0.1, restriction_norm='Cols') print(' successfully passed!') sys.stdout.flush()
import pydeep.rbm.estimator as ESTIMATOR import pydeep.misc.statistics as STATISTICS import pydeep.misc.toyproblems as TOY_DATA import pydeep.misc.visualization as VISUALIZATION import pydeep.misc.measuring as MEASURE # Set random seed (optional) numx.random.seed(42) # Input and hidden dimensionality v1 = v2 = 3 h1 = h2 = 4 # Load data , get it from 'deeplearning.net/data/mnist/mnist.pkl.gz' train_data = TOY_DATA.generate_bars_and_stripes_complete(v1) # Training paramters batch_size = train_data.shape[0] epochs = 20000 # Create trainer and model rbm = MODEL.BinaryBinaryRBM(number_visibles=v1 * v2, number_hiddens=h1 * h2, data=train_data) trainer = TRAINER.PCD(rbm, batch_size) # Measuring time measurer = MEASURE.Stopwatch() # Train model