def partition_function_exact(model, batchsize_exponent='AUTO'):
''' Computes the true partition function for the given model by factoring
over the visible and hidden2 units.
:Parameters:
model: The model
-type: Valid DBM model
batchsize_exponent: 2^batchsize_exponent will be the batch size.
-type: int
:Returns:
Log Partition function for the model.
-type: float
'''
# We transform the DBM to an RBM with restricted connections.
rbm = RBM_MODEL.BinaryBinaryRBM(
number_visibles=model.input_dim + model.hidden2_dim,
number_hiddens=model.hidden1_dim,
data=None,
initial_weights=numx.vstack((model.W1, model.W2.T)),
initial_visible_bias=numx.hstack((model.b1, model.b3)),
initial_hidden_bias=model.b2,
initial_visible_offsets=numx.hstack((model.o1, model.o3)),
initial_hidden_offsets=model.o2)
return RBM_ESTIMATOR.partition_function_factorize_h(rbm)
def partition_function_AIS(model,
num_chains=100,
k=1,
betas=10000,
status=False):
''' Approximates the partition function for the given model using annealed
importance sampling.
:Parameters:
model: The model.
-type: Valid RBM model.
num_chains: Number of AIS runs.
-type: int
k: Number of Gibbs sampling steps.
-type: int
beta: Number or a list of inverse temperatures to sample from.
-type: int, numpy array [num_betas]
status: If true prints the progress on console.
-type: bool
:Returns:
Mean estimated log partition function.
-type: float
Mean +3std estimated log partition function.
-type: float
Mean -3std estimated log partition function.
-type: float
'''
# We transform the DBM to an RBM with restricted connections.
rbm = RBM_MODEL.BinaryBinaryRBM(
number_visibles=model.input_dim + model.hidden2_dim,
number_hiddens=model.hidden1_dim,
data=None,
initial_weights=numx.vstack((model.W1, model.W2.T)),
initial_visible_bias=numx.hstack((model.b1, model.b3)),
initial_hidden_bias=model.b2,
initial_visible_offsets=numx.hstack((model.o1, model.o3)),
initial_hidden_offsets=model.o2)
# Run AIS for the transformed DBM
return RBM_ESTIMATOR.annealed_importance_sampling(model=rbm,
num_chains=num_chains,
k=k,
betas=betas,
status=status)
def __init__(self, model, batch_size):
# Set batch size
self.batch_size = batch_size
# Store model
self.model = model
self.rbm = RBM_MODEL.BinaryBinaryRBM(
number_visibles=model.input_dim + model.hidden2_dim,
number_hiddens=model.hidden1_dim,
data=None,
initial_weights=numx.vstack((model.W1, model.W2.T)),
initial_visible_bias=numx.hstack((model.b1, model.b3)),
initial_hidden_bias=model.b2,
initial_visible_offsets=numx.hstack((model.o1, model.o3)),
initial_hidden_offsets=model.o2)
self.sampler = RBM_SAMPLER.GibbsSampler(self.rbm)
def __init__(self, model, batch_size):
# Set batch size
self.batch_size = batch_size
# Store model
self.model = model
rbm = RBM_MODEL.BinaryBinaryRBM(
number_visibles=model.input_dim + model.hidden2_dim,
number_hiddens=model.hidden1_dim,
data=None,
initial_weights=numx.vstack((model.W1, model.W2.T)),
initial_visible_bias=numx.hstack((model.b1, model.b3)),
initial_hidden_bias=model.b2,
initial_visible_offsets=numx.hstack((model.o1, model.o3)),
initial_hidden_offsets=model.o2)
# Initializee Markov chains
self.m1 = model.o1 + numx.zeros((batch_size, model.input_dim))
self.m2 = model.o2 + numx.zeros((batch_size, model.hidden1_dim))
self.m3 = model.o3 + numx.zeros((batch_size, model.hidden2_dim))
class TestSampler(unittest.TestCase):
bbrbmData = generate_bars_and_stripes_complete(2)
bbrbmData = numx.vstack((bbrbmData[0], bbrbmData, bbrbmData[5]))
bbrbmw = numx.array([[0.12179488, 2.95950177, 0.33513356, 35.05380642],
[0.20318085, -28.62372894, 26.52611278, 28.41793445],
[-0.19105386, -28.58530584, -26.52747507, 28.78447320],
[0.08953740, -59.82556859, -0.06665933, -27.71723459]])
bbrbmbv = numx.array([[-19.24399659, -13.26258696, 13.25909850, 43.74408543]])
bbrbmbh = numx.array([[-0.11155958, 57.02097584, -0.13331758, -32.25991501]])
bbrbm = Model.BinaryBinaryRBM(4, 4, bbrbmData, bbrbmw, bbrbmbv, bbrbmbh, 0.0, 0.0)
epsilon = 0.05
num_samples = 2000.0
@classmethod
def execute_sampler(cls, sampler, num_samples):
dictC = {'[ 0. 0. 0. 0.]': 0, # 2
'[ 1. 1. 1. 1.]': 0, # 2
'[ 0. 0. 1. 1.]': 0, # 1
'[ 1. 1. 0. 0.]': 0, # 1
'[ 1. 0. 1. 0.]': 0, # 1
'[ 0. 1. 0. 1.]': 0, # 1
'[ 0. 1. 1. 0.]': 0,
'[ 1. 0. 0. 1.]': 0,
'[ 0. 0. 0. 1.]': 0,
'[ 0. 0. 1. 0.]': 0,
'[ 0. 1. 0. 0.]': 0,
'[ 1. 0. 0. 0.]': 0,
'[ 0. 1. 1. 1.]': 0,
'[ 1. 1. 1. 0.]': 0,
'[ 1. 0. 1. 1.]': 0,
'[ 1. 1. 0. 1.]': 0}
for _ in range(numx.int32(num_samples)):
if isinstance(sampler, Sampler.GibbsSampler):
# Start form random since model is rather deterministic
samples = sampler.sample(numx.random.rand(1, 4), 1, ret_states=True)
else:
if isinstance(sampler, Sampler.PersistentGibbsSampler):
# Start form random since model is rather deterministic
sampler.chains = numx.random.rand(1, 4)
samples = sampler.sample(1, 1, ret_states=True)
else:
samples = sampler.sample(1, 1, ret_states=True)
dictC[str(samples[0])] += 1
probCD1 = dictC['[ 0. 0. 0. 0.]'] / num_samples
probCD2 = dictC['[ 1. 1. 1. 1.]'] / num_samples
probCS1 = dictC['[ 0. 0. 1. 1.]'] / num_samples
probCS2 = dictC['[ 1. 1. 0. 0.]'] / num_samples
probCS3 = dictC['[ 1. 0. 1. 0.]'] / num_samples
probCS4 = dictC['[ 0. 1. 0. 1.]'] / num_samples
sumProbs = probCD1 + probCD2 + probCS1 + probCS2 + probCS3 + probCS4
return [probCD1, probCD2, probCS1, probCS2, probCS3, probCS4, sumProbs]
def test_Gibbs_sampler(self):
sys.stdout.write('RBM Sampler -> Performing GibbsSampler test ... ')
sys.stdout.flush()
numx.random.seed(42)
sampler = Sampler.GibbsSampler(self.bbrbm)
probCD1, probCD2, probCS1, probCS2, probCS3, probCS4, sumProbs = self.execute_sampler(sampler, self.num_samples)
assert numx.all(numx.abs(1.0 / 4.0 - probCD1) < self.epsilon)
assert numx.all(numx.abs(1.0 / 4.0 - probCD2) < self.epsilon)
assert numx.all(numx.abs(1.0 / 8.0 - probCS1) < self.epsilon)
assert numx.all(numx.abs(1.0 / 8.0 - probCS2) < self.epsilon)
assert numx.all(numx.abs(1.0 / 8.0 - probCS3) < self.epsilon)
assert numx.all(numx.abs(1.0 / 8.0 - probCS4) < self.epsilon)
assert numx.all(numx.abs(1.0 - sumProbs) < self.epsilon)
print('successfully passed!')
sys.stdout.flush()
def test_Persistent_Gibbs_sampler(self):
sys.stdout.write('RBM Sampler -> Performing PersistentGibbsSampler test ... ')
sys.stdout.flush()
numx.random.seed(42)
sampler = Sampler.PersistentGibbsSampler(self.bbrbm, 1)
probCD1, probCD2, probCS1, probCS2, probCS3, probCS4, sumProbs = self.execute_sampler(sampler, self.num_samples)
assert numx.all(numx.abs(1.0 / 4.0 - probCD1) < self.epsilon)
assert numx.all(numx.abs(1.0 / 4.0 - probCD2) < self.epsilon)
assert numx.all(numx.abs(1.0 / 8.0 - probCS1) < self.epsilon)
assert numx.all(numx.abs(1.0 / 8.0 - probCS2) < self.epsilon)
assert numx.all(numx.abs(1.0 / 8.0 - probCS3) < self.epsilon)
assert numx.all(numx.abs(1.0 / 8.0 - probCS4) < self.epsilon)
assert numx.all(numx.abs(1.0 - sumProbs) < self.epsilon)
print('successfully passed!')
sys.stdout.flush()
def test_Parallel_Tempering_sampler(self):
sys.stdout.write('RBM Sampler -> Performing ParallelTemperingSampler test ... ')
sys.stdout.flush()
numx.random.seed(42)
sampler = Sampler.ParallelTemperingSampler(self.bbrbm, 10)
probCD1, probCD2, probCS1, probCS2, probCS3, probCS4, sumProbs = self.execute_sampler(sampler, self.num_samples)
assert numx.all(numx.abs(1.0 / 4.0 - probCD1) < self.epsilon)
assert numx.all(numx.abs(1.0 / 4.0 - probCD2) < self.epsilon)
assert numx.all(numx.abs(1.0 / 8.0 - probCS1) < self.epsilon)
assert numx.all(numx.abs(1.0 / 8.0 - probCS2) < self.epsilon)
assert numx.all(numx.abs(1.0 / 8.0 - probCS3) < self.epsilon)
assert numx.all(numx.abs(1.0 / 8.0 - probCS4) < self.epsilon)
assert numx.all(numx.abs(1.0 - sumProbs) < self.epsilon)
print('successfully passed!')
sys.stdout.flush()
def test_Independent_Parallel_Tempering_sampler(self):
sys.stdout.write('RBM Sampler -> Performing IndependentParallelTemperingSampler test ... ')
sys.stdout.flush()
numx.random.seed(42)
sampler = Sampler.IndependentParallelTemperingSampler(self.bbrbm, 10, 10)
probCD1, probCD2, probCS1, probCS2, probCS3, probCS4, sumProbs = self.execute_sampler(sampler, self.num_samples)
assert numx.all(numx.abs(1.0 / 4.0 - probCD1) < self.epsilon)
assert numx.all(numx.abs(1.0 / 4.0 - probCD2) < self.epsilon)
assert numx.all(numx.abs(1.0 / 8.0 - probCS1) < self.epsilon)
assert numx.all(numx.abs(1.0 / 8.0 - probCS2) < self.epsilon)
assert numx.all(numx.abs(1.0 / 8.0 - probCS3) < self.epsilon)
assert numx.all(numx.abs(1.0 / 8.0 - probCS4) < self.epsilon)
assert numx.all(numx.abs(1.0 - sumProbs) < self.epsilon)
print('successfully passed!')
sys.stdout.flush()
class TestBinaryBinaryRBM(unittest.TestCase):
# Known RBM
bbrbmData = generate_bars_and_stripes_complete(2)
bbrbmData = numx.vstack((bbrbmData[0], bbrbmData, bbrbmData[5]))
bbrbmw = numx.array(
[[0.12179488, 2.95950177, 0.33513356, 35.05380642],
[0.20318085, -28.62372894, 26.52611278, 28.41793445],
[-0.19105386, -28.58530584, -26.52747507, 28.78447320],
[0.08953740, -59.82556859, -0.06665933, -27.71723459]])
bbrbmbv = numx.array(
[[-19.24399659, -13.26258696, 13.25909850, 43.74408543]])
bbrbmbh = numx.array(
[[-0.11155958, 57.02097584, -0.13331758, -32.25991501]])
bbrbm = Model.BinaryBinaryRBM(4, 4, bbrbmData, bbrbmw, bbrbmbv, bbrbmbh,
0.0, 0.0)
bbrbmTruelogZ = 59.6749019726
bbrbmTrueLL = -1.7328699078
bbrbmBestLLPossible = -1.732867951
epsilon = 0.00001
def test___init__(self):
sys.stdout.write('BinaryBinaryRBM -> Performing init test ...')
sys.stdout.flush()
assert numx.all(self.bbrbm.bv_base == numx.array([[0, 0, 0, 0]]))
print(' successfully passed!')
sys.stdout.flush()
def test__add_visible_units(self):
sys.stdout.write(
'BinaryBinaryRBM -> Performing add_visible_units test ...')
sys.stdout.flush()
localmodel = copy.deepcopy(self.bbrbm)
localmodel._add_visible_units(2, 3)
assert numx.all(localmodel.bv_base == numx.array([[0, 0, 0, 0, 0, 0]]))
print(' successfully passed!')
sys.stdout.flush()
def test__remove_visible_units(self):
sys.stdout.write(
'BinaryBinaryRBM -> Performing remove_visible_units test ...')
sys.stdout.flush()
localmodel = copy.deepcopy(self.bbrbm)
localmodel._remove_visible_units([0, 2])
assert numx.all(localmodel.bv_base == numx.array([[0, 0]]))
print(' successfully passed!')
sys.stdout.flush()
def test__calculate_weight_gradient(self):
sys.stdout.write(
'BinaryBinaryRBM -> Performing calculate_weight_gradient test ...')
sys.stdout.flush()
deltaW = self.bbrbm._calculate_weight_gradient(
numx.array([[1, 1, 1, 0], [0, 1, 0, 1]]),
numx.array([[0, 1, 0, 1], [0, 1, 1, 0]]))
target = numx.array([[0., 1., 0., 1.], [0., 2., 1., 1.],
[0., 1., 0., 1.], [0., 1., 1., 0.]])
assert numx.all(target == deltaW)
print(' successfully passed!')
sys.stdout.flush()
def test__calculate_visible_bias_gradient(self):
sys.stdout.write(
'BinaryBinaryRBM -> Performing calculate_visible_bias_gradient test ...'
)
sys.stdout.flush()
deltaBv = self.bbrbm._calculate_visible_bias_gradient(
numx.array([[1, 1, 1, 0], [0, 1, 0, 1]]))
target = numx.array([[1., 2., 1., 1.]])
assert numx.all(target == deltaBv)
print(' successfully passed!')
sys.stdout.flush()
def test__calculate_hidden_bias_gradient(self):
sys.stdout.write(
'BinaryBinaryRBM -> Performing calculate_hidden_bias_gradient test ...'
)
sys.stdout.flush()
deltaBh = self.bbrbm._calculate_hidden_bias_gradient(
numx.array([[0, 1, 0, 1], [0, 1, 1, 0]]))
target = numx.array([[0., 2., 1., 1.]])
assert numx.all(target == deltaBh)
print(' successfully passed!')
sys.stdout.flush()
def test_calculate_gradients(self):
sys.stdout.write(
'BinaryBinaryRBM -> Performing calculate_gradients test ...')
sys.stdout.flush()
deltaW = self.bbrbm._calculate_weight_gradient(
numx.array([[1, 1, 1, 0], [0, 1, 0, 1]]),
numx.array([[0, 1, 0, 1], [0, 1, 1, 0]]))
deltaBv = self.bbrbm._calculate_visible_bias_gradient(
numx.array([[1, 1, 1, 0], [0, 1, 0, 1]]))
deltaBh = self.bbrbm._calculate_hidden_bias_gradient(
numx.array([[0, 1, 0, 1], [0, 1, 1, 0]]))
deltas = self.bbrbm.calculate_gradients(
numx.array([[1, 1, 1, 0], [0, 1, 0, 1]]),
numx.array([[0, 1, 0, 1], [0, 1, 1, 0]]))
assert numx.all(deltaW == deltas[0])
assert numx.all(deltaBv == deltas[1])
assert numx.all(deltaBh == deltas[2])
print(' successfully passed!')
sys.stdout.flush()
def test_sample_v(self):
sys.stdout.write('BinaryBinaryRBM -> Performing sample_v test ...')
sys.stdout.flush()
assert numx.all(self.bbrbm.sample_v(numx.ones((10000, 4))) == 1.0)
assert numx.all(self.bbrbm.sample_v(numx.zeros((10000, 4))) == 0.0)
numx.random.seed(42)
samples = self.bbrbm.sample_v(numx.ones((10000, 4)) * 0.5)
assert numx.sum(samples != 0.0) + numx.sum(samples != 1.0) == 40000
assert numx.abs(numx.sum(samples) / 40000.0 - 0.5) < 0.01
print(' successfully passed!')
sys.stdout.flush()
def test_sample_h(self):
sys.stdout.write('BinaryBinaryRBM -> Performing sample_h test ...')
sys.stdout.flush()
assert numx.all(self.bbrbm.sample_h(numx.ones((10000, 4))) == 1.0)
assert numx.all(self.bbrbm.sample_h(numx.zeros((10000, 4))) == 0.0)
numx.random.seed(42)
samples = self.bbrbm.sample_h(numx.ones((10000, 4)) * 0.5)
assert numx.sum(samples != 0.0) + numx.sum(samples != 1.0) == 40000
assert numx.abs(numx.sum(samples) / 40000.0 - 0.5) < 0.01
print(' successfully passed!')
sys.stdout.flush()
def test_probability_v_given_h(self):
sys.stdout.write(
'BinaryBinaryRBM -> Performing probability_v_given_h test ...')
sys.stdout.flush()
probs = self.bbrbm.probability_v_given_h(self.bbrbmData)
target = numx.array(
[[4.38973669e-09, 1.73832475e-06, 9.99998256e-01, 1.00000000e+00],
[4.38973669e-09, 1.73832475e-06, 9.99998256e-01, 1.00000000e+00],
[6.93234181e-09, 9.99998583e-01, 1.42771991e-06, 1.00000000e+00],
[9.99999993e-01, 1.41499740e-06, 9.99998571e-01, 0.00000000e+00],
[9.56375764e-08, 0.00000000e+00, 1.82363957e-07, 1.13445207e-07],
[9.99999903e-01, 1.00000000e+00, 9.99999817e-01, 9.99999883e-01],
[9.99999996e-01, 9.99998259e-01, 1.74236911e-06, 0.00000000e+00],
[9.99999996e-01, 9.99998259e-01, 1.74236911e-06, 0.00000000e+00]])
assert numx.all(numx.abs(probs - target) < self.epsilon)
print(' successfully passed!')
sys.stdout.flush()
def test_probability_h_given_v(self):
sys.stdout.write(
'BinaryBinaryRBM -> Performing probability_h_given_v test ...')
sys.stdout.flush()
probs = self.bbrbm.probability_h_given_v(self.bbrbmData)
target = numx.array(
[[4.72138994e-01, 1.00000000e+00, 4.66719883e-01, 9.76996262e-15],
[4.72138994e-01, 1.00000000e+00, 4.66719883e-01, 9.76996262e-15],
[4.54918124e-01, 1.00000000e+00, 3.68904907e-12, 1.00000000e+00],
[5.45166211e-01, 2.24265051e-14, 1.00000000e+00, 1.97064587e-14],
[5.53152448e-01, 1.00000000e+00, 1.00000000e+00, 1.00000000e+00],
[4.46931620e-01, 2.33146835e-14, 2.46841436e-12, 2.83661983e-14],
[5.27945768e-01, 0.00000000e+00, 5.33398782e-01, 1.00000000e+00],
[5.27945768e-01, 0.00000000e+00, 5.33398782e-01, 1.00000000e+00]])
assert numx.all(numx.abs(probs - target) < self.epsilon)
print(' successfully passed!')
sys.stdout.flush()
def test_energy(self):
sys.stdout.write('BinaryBinaryRBM -> Performing energy test ...')
sys.stdout.flush()
energies = self.bbrbm.energy(self.bbrbmData, self.bbrbmData)
target = numx.array([[0.], [0.], [32.49137574], [32.50603837],
[0.93641873], [0.91694445], [0.03276686],
[0.03276686]])
assert numx.all(numx.abs(energies - target) < self.epsilon)
print(' successfully passed!')
sys.stdout.flush()
def test_unnormalized_log_probability_v(self):
sys.stdout.write(
'BinaryBinaryRBM -> Performing unnormalized_log_probability_v test ...'
)
sys.stdout.flush()
probs = self.bbrbm.unnormalized_log_probability_v(self.bbrbmData)
target = numx.array([[58.28860656], [58.28860656], [57.59545755],
[57.59545757], [57.59545753], [57.59545756],
[58.28860656], [58.28860656]])
assert numx.all(numx.abs(probs - target) < self.epsilon)
print(' successfully passed!')
sys.stdout.flush()
def test_unnormalized_log_probability_h(self):
sys.stdout.write(
'BinaryBinaryRBM -> Performing unnormalized_log_probability_h test ...'
)
sys.stdout.flush()
probs = self.bbrbm.unnormalized_log_probability_h(self.bbrbmData)
target = numx.array([[57.00318742], [57.00318742], [56.98879586],
[56.98864114], [56.90941665], [56.90945961],
[57.00333938], [57.00333938]])
assert numx.all(numx.abs(probs - target) < self.epsilon)
print(' successfully passed!')
sys.stdout.flush()
def test_log_probability_v(self):
sys.stdout.write(
'BinaryBinaryRBM -> Performing log_probability_v test ...')
sys.stdout.flush()
probs = self.bbrbm.log_probability_v(self.bbrbmTruelogZ,
self.bbrbmData)
target = numx.array([[-1.38629541], [-1.38629541], [-2.07944442],
[-2.07944441], [-2.07944444], [-2.07944441],
[-1.38629541], [-1.38629541]])
assert numx.all(numx.abs(probs - target) < self.epsilon)
print(' successfully passed!')
sys.stdout.flush()
def test_log_probability_h(self):
sys.stdout.write(
'BinaryBinaryRBM -> Performing log_probability_h test ...')
sys.stdout.flush()
probs = self.bbrbm.log_probability_h(self.bbrbmTruelogZ,
self.bbrbmData)
target = numx.array([[-2.67171456], [-2.67171456], [-2.68610611],
[-2.68626083], [-2.76548532], [-2.76544237],
[-2.67156259], [-2.67156259]])
assert numx.all(numx.abs(probs - target) < self.epsilon)
print(' successfully passed!')
sys.stdout.flush()
def test_log_probability_v_h(self):
sys.stdout.write(
'BinaryBinaryRBM -> Performing log_probability_v_h test ...')
sys.stdout.flush()
h = self.bbrbm.probability_h_given_v(self.bbrbmData)
probs = self.bbrbm.log_probability_v_h(self.bbrbmTruelogZ,
self.bbrbmData, h)
target = numx.array([[-2.76881973], [-2.76881973], [-2.76852132],
[-2.76850605], [-2.76693057], [-2.76694846],
[-2.76879441], [-2.76879441]])
assert numx.all(numx.abs(probs - target) < self.epsilon)
print(' successfully passed!')
sys.stdout.flush()
def test__base_log_partition(self):
sys.stdout.write(
'BinaryBinaryRBM -> Performing base_log_partition test ...')
sys.stdout.flush()
localmodel = copy.deepcopy(self.bbrbm)
localmodel.bv_base += 1.0
assert numx.abs(localmodel._base_log_partition(False) -
5.54517744448) < self.epsilon
assert numx.abs(localmodel._base_log_partition(True) -
8.02563547231) < self.epsilon
print(' successfully passed!')
sys.stdout.flush()
def test__getbasebias(self):
sys.stdout.write('BinaryBinaryRBM -> Performing getbasebias test ...')
sys.stdout.flush()
# zero base bias when data mean is 0.5
localmodel = copy.deepcopy(self.bbrbm)
localmodel._data_mean = localmodel._data_mean / 2.0
assert numx.all(
numx.abs(localmodel._getbasebias() + 1.09861229) < self.epsilon)
print(' successfully passed!')
sys.stdout.flush()
class TestEstimator(unittest.TestCase):
# Known model
bbrbmData = generate_bars_and_stripes_complete(2)
bbrbmData = numx.vstack((bbrbmData[0], bbrbmData, bbrbmData[5]))
bbrbmw = numx.array(
[[0.12179488, 2.95950177, 0.33513356, 35.05380642],
[0.20318085, -28.62372894, 26.52611278, 28.41793445],
[-0.19105386, -28.58530584, -26.52747507, 28.78447320],
[0.08953740, -59.82556859, -0.06665933, -27.71723459]])
bbrbmbv = numx.array(
[[-19.24399659, -13.26258696, 13.25909850, 43.74408543]])
bbrbmbh = numx.array(
[[-0.11155958, 57.02097584, -0.13331758, -32.25991501]])
bbrbm = Model.BinaryBinaryRBM(4, 4, bbrbmData, bbrbmw, bbrbmbv, bbrbmbh,
0.0, 0.0)
bbrbmTruelogZ = 59.6749019726
bbrbmTrueLL = -1.7328699078
bbrbmBestLLPossible = -1.732867951
epsilon = 0.00001
def test_reconstruction_error(self):
sys.stdout.write(
'RBM Estimator -> Performing reconstruction_error test ...')
sys.stdout.flush()
numx.random.seed(42)
rec = Estimator.reconstruction_error(self.bbrbm,
self.bbrbmData,
k=1,
beta=1.0,
use_states=True,
absolut_error=False)
assert numx.all(numx.abs(rec) < self.epsilon)
rec = Estimator.reconstruction_error(self.bbrbm,
self.bbrbmData,
k=1,
beta=1.0,
use_states=False,
absolut_error=False)
assert numx.all(numx.abs(rec) < self.epsilon)
rec = Estimator.reconstruction_error(self.bbrbm,
self.bbrbmData,
k=1,
beta=1.0,
use_states=True,
absolut_error=True)
assert numx.all(numx.abs(rec) < self.epsilon)
rec = Estimator.reconstruction_error(self.bbrbm,
self.bbrbmData,
k=1,
beta=1.0,
use_states=False,
absolut_error=True)
assert numx.all(numx.abs(rec) < self.epsilon)
rec = Estimator.reconstruction_error(self.bbrbm,
self.bbrbmData,
k=10,
beta=1.0,
use_states=False,
absolut_error=False)
assert numx.all(numx.abs(rec) < self.epsilon)
# Test List
testList = []
for i in range(self.bbrbmData.shape[0]):
testList.append(self.bbrbmData[i].reshape(1, 4))
rec = Estimator.reconstruction_error(self.bbrbm,
testList,
k=10,
beta=1.0,
use_states=False,
absolut_error=False)
assert numx.all(numx.abs(rec) < self.epsilon)
print(' successfully passed!')
sys.stdout.flush()
def test_log_likelihood_v(self):
sys.stdout.write(
'RBM Estimator -> Performing log_likelihood_v test ...')
sys.stdout.flush()
numx.random.seed(42)
ll = numx.mean(
Estimator.log_likelihood_v(self.bbrbm, self.bbrbmTruelogZ,
self.bbrbmData, 1.0))
assert numx.all(numx.abs(ll - self.bbrbmTrueLL) < self.epsilon)
# Test List
testList = []
for i in range(self.bbrbmData.shape[0]):
testList.append(self.bbrbmData[i].reshape(1, 4))
ll = numx.mean(
Estimator.log_likelihood_v(self.bbrbm, self.bbrbmTruelogZ,
testList, 1.0))
assert numx.all(numx.abs(ll - self.bbrbmTrueLL) < self.epsilon)
print(' successfully passed!')
sys.stdout.flush()
def test_log_likelihood_h(self):
sys.stdout.write(
'RBM Estimator -> Performing log_likelihood_h test ...')
sys.stdout.flush()
numx.random.seed(42)
hdata = numx.float64(
self.bbrbm.probability_h_given_v(self.bbrbmData) < 0.5)
ll = numx.mean(
Estimator.log_likelihood_h(self.bbrbm, self.bbrbmTruelogZ, hdata,
1.0))
assert numx.all(numx.abs(ll + 9.55929166739) < self.epsilon)
# Test List
testList = []
for i in range(hdata.shape[0]):
testList.append(hdata[i].reshape(1, 4))
ll = numx.mean(
Estimator.log_likelihood_v(self.bbrbm, self.bbrbmTruelogZ,
testList, 1.0))
assert numx.all(numx.abs(ll + 9.55929166739) < self.epsilon)
print(' successfully passed!')
sys.stdout.flush()
def test_partition_function_factorize_v(self):
sys.stdout.write(
'RBM Estimator -> Performing partition_function_factorize_v test ...'
)
sys.stdout.flush()
LogZ = Estimator.partition_function_factorize_v(
self.bbrbm, beta=None, batchsize_exponent='AUTO', status=False)
assert numx.all(numx.abs(LogZ - self.bbrbmTruelogZ) < self.epsilon)
LogZ = Estimator.partition_function_factorize_v(self.bbrbm,
beta=None,
batchsize_exponent=0,
status=False)
assert numx.all(numx.abs(LogZ - self.bbrbmTruelogZ) < self.epsilon)
LogZ = Estimator.partition_function_factorize_v(self.bbrbm,
beta=None,
batchsize_exponent=3,
status=False)
assert numx.all(numx.abs(LogZ - self.bbrbmTruelogZ) < self.epsilon)
LogZ = Estimator.partition_function_factorize_v(self.bbrbm,
beta=None,
batchsize_exponent=555,
status=False)
assert numx.all(numx.abs(LogZ - self.bbrbmTruelogZ) < self.epsilon)
print(' successfully passed!')
sys.stdout.flush()
def test_partition_function_factorize_h(self):
sys.stdout.write(
'RBM Estimator -> Performing partition_function_factorize_v test ...'
)
sys.stdout.flush()
LogZ = Estimator.partition_function_factorize_h(
self.bbrbm, beta=None, batchsize_exponent='AUTO', status=False)
assert numx.all(numx.abs(LogZ - self.bbrbmTruelogZ) < self.epsilon)
LogZ = Estimator.partition_function_factorize_h(self.bbrbm,
beta=None,
batchsize_exponent=0,
status=False)
assert numx.all(numx.abs(LogZ - self.bbrbmTruelogZ) < self.epsilon)
LogZ = Estimator.partition_function_factorize_h(self.bbrbm,
beta=None,
batchsize_exponent=3,
status=False)
assert numx.all(numx.abs(LogZ - self.bbrbmTruelogZ) < self.epsilon)
LogZ = Estimator.partition_function_factorize_h(self.bbrbm,
beta=None,
batchsize_exponent=555,
status=False)
assert numx.all(numx.abs(LogZ - self.bbrbmTruelogZ) < self.epsilon)
print(' successfully passed!')
sys.stdout.flush()
def test_annealed_importance_sampling(self):
sys.stdout.write(
'RBM Estimator -> Performing annealed_importance_sampling test ...'
)
sys.stdout.flush()
numx.random.seed(42)
LogZ = Estimator.annealed_importance_sampling(self.bbrbm,
num_chains=100,
k=1,
betas=100,
status=False)
assert numx.all(numx.abs(LogZ[0] - self.bbrbmTruelogZ) < 0.5)
LogZ = Estimator.annealed_importance_sampling(self.bbrbm,
num_chains=100,
k=1,
betas=1000,
status=False)
assert numx.all(numx.abs(LogZ[0] - self.bbrbmTruelogZ) < 0.05)
LogZ = Estimator.annealed_importance_sampling(self.bbrbm,
num_chains=100,
k=1,
betas=10000,
status=False)
assert numx.all(numx.abs(LogZ[0] - self.bbrbmTruelogZ) < 0.005)
print(' successfully passed!')
sys.stdout.flush()
def test_reverse_annealed_importance_sampling(self):
sys.stdout.write(
'RBM Estimator -> Performing reverse_annealed_importance_sampling test ...'
)
sys.stdout.flush()
numx.random.seed(42)
LogZ = Estimator.reverse_annealed_importance_sampling(self.bbrbm,
num_chains=100,
k=1,
betas=100,
status=False)
assert numx.all(numx.abs(LogZ[0] - self.bbrbmTruelogZ) < 0.5)
LogZ = Estimator.reverse_annealed_importance_sampling(self.bbrbm,
num_chains=100,
k=1,
betas=1000,
status=False)
assert numx.all(numx.abs(LogZ[0] - self.bbrbmTruelogZ) < 0.05)
LogZ = Estimator.reverse_annealed_importance_sampling(self.bbrbm,
num_chains=100,
k=1,
betas=10000,
status=False)
assert numx.all(numx.abs(LogZ[0] - self.bbrbmTruelogZ) < 0.005)
print(' successfully passed!')
sys.stdout.flush()
if flipped:
train_data = 1 - train_data
test_data = 1 - test_data
print("Flipped MNIST")
else:
print("Normal MNIST")
# Training parameters
batch_size = 100
epochs = 50
# Create centered or normal model
if update_offsets <= 0.0:
rbm = model.BinaryBinaryRBM(number_visibles=v1 * v2,
number_hiddens=h1 * h2,
data=train_data,
initial_visible_offsets=0.0,
initial_hidden_offsets=0.0)
print("Normal RBM")
else:
rbm = model.BinaryBinaryRBM(number_visibles=v1 * v2,
number_hiddens=h1 * h2,
data=train_data,
initial_visible_offsets='AUTO',
initial_hidden_offsets='AUTO')
print("Centered RBM")
# Create trainer
trainer_pcd = trainer.PCD(rbm, num_chains=batch_size)
# Measuring time
def train(self,
data,
epsilon,
k=[3, 1],
offset_typ='DDD',
meanfield=False):
#positive phase
id1 = numx.dot(data - self.model.o1, self.model.W1)
d3 = numx.copy(self.model.o3)
d2 = numx.copy(self.model.o2)
#for _ in range(k[0]):
if meanfield == False:
for _ in range(k[0]):
d2 = Sigmoid.f(id1 +
numx.dot(d3 - self.model.o3, self.model.W2.T) +
self.model.b2)
d2 = self.model.dtype(d2 > numx.random.random(d2.shape))
d3 = Sigmoid.f(
numx.dot(d2 - self.model.o2, self.model.W2) +
self.model.b3)
d3 = self.model.dtype(d3 > numx.random.random(d3.shape))
else:
if meanfield == True:
for _ in range(k[0]):
d2 = Sigmoid.f(id1 +
numx.dot(d3 -
self.model.o3, self.model.W2.T) +
self.model.b2)
d3 = Sigmoid.f(
numx.dot(d2 - self.model.o2, self.model.W2) +
self.model.b3)
else:
d2_new = Sigmoid.f(id1 +
numx.dot(d3 -
self.model.o3, self.model.W2.T) +
self.model.b2)
d3_new = Sigmoid.f(
numx.dot(d2_new - self.model.o2, self.model.W2) +
self.model.b3)
while numx.max(numx.abs(d2_new - d2)) > meanfield or numx.max(
numx.abs(d3_new - d3)) > meanfield:
d2 = d2_new
d3 = d3_new
d2_new = Sigmoid.f(
id1 +
numx.dot(d3_new - self.model.o3, self.model.W2.T) +
self.model.b2)
d3_new = Sigmoid.f(
numx.dot(d2_new - self.model.o2, self.model.W2) +
self.model.b3)
d2 = d2_new
d3 = d3_new
self.sampler.model = RBM_MODEL.BinaryBinaryRBM(
number_visibles=self.model.input_dim + self.model.hidden2_dim,
number_hiddens=self.model.hidden1_dim,
data=None,
initial_weights=numx.vstack((self.model.W1, self.model.W2.T)),
initial_visible_bias=numx.hstack((self.model.b1, self.model.b3)),
initial_hidden_bias=self.model.b2,
initial_visible_offsets=numx.hstack(
(self.model.o1, self.model.o3)),
initial_hidden_offsets=self.model.o2)
if isinstance(self.sampler, RBM_SAMPLER.GibbsSampler):
sample = self.sampler.sample(numx.hstack((data, d3)))
else:
sample = self.sampler.sample(self.batch_size, k[1])
self.m2 = self.sampler.model.probability_h_given_v(sample)
self.m1 = sample[:, 0:self.model.input_dim]
self.m3 = sample[:, self.model.input_dim:]
# Estimate new means
new_o1 = 0
if offset_typ[0] is 'D':
new_o1 = data.mean(axis=0)
if offset_typ[0] is 'A':
new_o1 = (self.m1.mean(axis=0) + data.mean(axis=0)) / 2.0
if offset_typ[0] is 'M':
new_o1 = self.m1.mean(axis=0)
new_o2 = 0
if offset_typ[1] is 'D':
new_o2 = d2.mean(axis=0)
if offset_typ[1] is 'A':
new_o2 = (self.m2.mean(axis=0) + d2.mean(axis=0)) / 2.0
if offset_typ[1] is 'M':
new_o2 = self.m2.mean(axis=0)
new_o3 = 0
if offset_typ[2] is 'D':
new_o3 = d3.mean(axis=0)
if offset_typ[2] is 'A':
new_o3 = (self.m3.mean(axis=0) + d3.mean(axis=0)) / 2.0
if offset_typ[2] is 'M':
new_o3 = self.m3.mean(axis=0)
# Reparameterize
self.model.b1 += epsilon[6] * numx.dot(new_o2 - self.model.o2,
self.model.W1.T)
self.model.b2 += epsilon[5] * numx.dot(
new_o1 - self.model.o1, self.model.W1) + epsilon[7] * numx.dot(
new_o3 - self.model.o3, self.model.W2.T)
self.model.b3 += epsilon[7] * numx.dot(new_o2 - self.model.o2,
self.model.W2)
# Shift means
self.model.o1 = (1.0 -
epsilon[5]) * self.model.o1 + epsilon[5] * new_o1
self.model.o2 = (1.0 -
epsilon[6]) * self.model.o2 + epsilon[6] * new_o2
self.model.o3 = (1.0 -
epsilon[7]) * self.model.o3 + epsilon[7] * new_o3
# Calculate gradients
dW1 = (numx.dot(
(data - self.model.o1).T, d2 - self.model.o2) - numx.dot(
(self.m1 - self.model.o1).T, self.m2 - self.model.o2))
dW2 = (numx.dot((d2 - self.model.o2).T, d3 - self.model.o3) - numx.dot(
(self.m2 - self.model.o2).T, self.m3 - self.model.o3))
db1 = (numx.sum(data - self.m1,
axis=0)).reshape(1, self.model.input_dim)
db2 = (numx.sum(d2 - self.m2, axis=0)).reshape(1,
self.model.hidden1_dim)
db3 = (numx.sum(d3 - self.m3, axis=0)).reshape(1,
self.model.hidden2_dim)
# Update Model
self.model.W1 += epsilon[0] / self.batch_size * dW1
self.model.W2 += epsilon[1] / self.batch_size * dW2
self.model.b1 += epsilon[2] / self.batch_size * db1
self.model.b2 += epsilon[3] / self.batch_size * db2
self.model.b3 += epsilon[4] / self.batch_size * db3
def test_forward_backward_reconstruct_sampling(self):
print(
'Deep Believe Network -> Performing forward backward prop test ...'
)
sys.stdout.flush()
numx.random.seed(42)
rbm1 = MODEL.BinaryBinaryRBM(number_visibles=2, number_hiddens=4)
rbm2 = MODEL.BinaryBinaryRBM(number_visibles=4, number_hiddens=2)
stack = STACK.DBN([rbm1, rbm2])
forward_target = numx.array([[0.54640997, 0.8437009],
[0.45359003, 0.1562991]])
backward_target = numx.array([[0.30266536, 0.52656316],
[0.69733464, 0.47343684]])
rec_target = numx.array([[0.57055553, 0.23073692],
[0.42944447, 0.76926308]])
assert numx.sum(
numx.abs(
stack.forward_propagate(numx.array([[1, 0], [0, 1]])) -
forward_target) < 0.000001)
assert numx.sum(
numx.abs(
stack.backward_propagate(numx.array([[1, 0], [0, 1]])) -
backward_target) < 0.000001)
assert numx.sum(
numx.abs(
stack.backward_propagate(
stack.forward_propagate(numx.array([[1, 0], [0, 1]]))) -
rec_target) < 0.000001)
assert numx.sum(
numx.abs(
stack.reconstruct(numx.array([[1, 0], [0, 1]])) -
rec_target) < 0.000001)
forward_target = numx.array([[1, 1], [1, 0]])
backward_target = numx.array([[1, 1], [0, 0]])
rec_target = numx.array([[1, 0], [0, 0]])
assert numx.sum(
numx.abs(
stack.forward_propagate(numx.array([[1, 0], [0, 1]]), True) -
forward_target) < 0.000001)
assert numx.sum(
numx.abs(
stack.backward_propagate(numx.array([[1, 0], [0, 1]]), True) -
backward_target) < 0.000001)
assert numx.sum(
numx.abs(
stack.backward_propagate(
stack.forward_propagate(numx.array([[1, 0], [0, 1]])),
True) - rec_target) < 0.000001)
assert numx.sum(
numx.abs(
stack.reconstruct(numx.array([[1, 0], [0, 1]]), True) -
rec_target) < 0.000001)
assert numx.sum(
numx.abs(
stack.sample_top_layer(1, sample=False) -
numx.array([[0.81216745, 0.80223488], [0.94668029, 0.05014349]
])) < 0.000001)
numx.random.seed(42)
assert numx.sum(
numx.abs(
stack.sample_top_layer(100, sample=True) -
numx.array([[0, 0], [0, 1]])) < 0.000001)
numx.random.seed(42)
assert numx.sum(
numx.abs(
stack.sample_top_layer(
100,
initial_state=numx.array([[1, 1], [1, 1]]),
sample=True) - numx.array([[0, 1], [1, 0]])) < 0.000001)
numx.random.seed(42)
assert numx.sum(
numx.abs(
stack.reconstruct_sample_top_layer(
input_data=numx.array([[1, 0], [0, 1]]),
sampling_steps=10,
sample_forward_backward=False) -
numx.array([[0.69733464, 0.47343684], [0.69733464, 0.47343684]
])) < 0.000001)
numx.random.seed(42)
assert numx.sum(
numx.abs(
stack.reconstruct_sample_top_layer(
input_data=numx.array([[1, 1], [1, 1]]),
sampling_steps=10,
sample_forward_backward=True) -
numx.array([[0, 0], [0, 0]])) < 0.000001)
print('successfully passed!')
sys.stdout.flush()
