def call(self, x):
if self.mode == MODE_VISIBLE_BERNOULLI:
return K.cast(
K.less(
K.random_uniform(shape=(self.hps['batch_size'],
x.shape[1])) #?
,
K.sigmoid(K.dot(x, self.rbm_weight) + self.hidden_bias)))
elif self.mode == MODE_VISIBLE_GAUSSIAN:
return K.cast(
K.less(
K.random_uniform(shape=(self.hps['batch_size'],
x.shape[1])),
K.relu(K.dot(x, self.rbm_weight) + self.hidden_bias))) #?
def call(self, inputs):
rank = inputs.shape.rank
if rank is not None and rank > 2:
h_prob = standard_ops.sigmoid(
standard_ops.tensordot(inputs, self.h, [[rank - 1], [0]]))
h_state = tf.nn.relu(
tf.sign(h_prob - backend.random_uniform(tf.shape(h_prob))))
else:
inputs = math_ops.cast(inputs, self._compute_dtype)
if K.is_sparse(inputs):
h_prob = sparse_ops.sigmoid(
sparse_ops.sparse_tensor_dense_matmul(inputs, self.h))
h_state = tf.nn.relu(
tf.sign(h_prob - backend.random_uniform(tf.shape(h_prob))))
else:
h_prob = gen_math_ops.sigmoid(
gen_math_ops.mat_mul(inputs, self.h))
h_state = tf.nn.relu(
tf.sign(h_prob - backend.random_uniform(tf.shape(h_prob))))
return h_state
def _merge_function(self, inputs):
# Check exception.
x = inputs
if isinstance(x, list) != True or len(x) != 2:
raise ValueError('Input must be a list of two tensors.')
d1 = x[0] # Disentangled latent 1.
d2 = x[1] # Disentangled latent 1.
# Mixing style according to mixing probability.
num_layers = K.int_shape(d1)[1]
if self.mixing_prob is not None:
cutoff = Ke.cond(
K.random_uniform([], 0.0, 1.0) < self.mixing_prob,
lambda: K.random_uniform([], 1, num_layers, dtype=np.int32),
lambda: num_layers) #?
d = Ke.where(Ke.broadcast_to(np.arange(num_layers)[np.newaxis, :, np.newaxis] \
< cutoff, K.shape(d1)), d1, d2) #?
else:
d = d1
return d
def dropped_inputs(inputs=inputs, rate=self.rate, seed=self.seed): # pylint: disable=missing-docstring
alpha = 1.6732632423543772848170429916717
scale = 1.0507009873554804934193349852946
alpha_p = -alpha * scale
kept_idx = math_ops.greater_equal(
K.random_uniform(noise_shape, seed=seed), rate)
kept_idx = math_ops.cast(kept_idx, inputs.dtype)
# Get affine transformation params
a = ((1 - rate) * (1 + rate * alpha_p**2))**-0.5
b = -a * alpha_p * rate
# Apply mask
x = inputs * kept_idx + alpha_p * (1 - kept_idx)
# Do affine transformation
return a * x + b
def dropped_inputs(inputs=inputs, rate=self.rate, seed=self.seed): # pylint: disable=missing-docstring
alpha = 1.6732632423543772848170429916717
scale = 1.0507009873554804934193349852946
alpha_p = -alpha * scale
kept_idx = math_ops.greater_equal(
K.random_uniform(noise_shape, seed=seed), rate)
kept_idx = math_ops.cast(kept_idx, K.floatx())
# Get affine transformation params
a = ((1 - rate) * (1 + rate * alpha_p**2))**-0.5
b = -a * alpha_p * rate
# Apply mask
x = inputs * kept_idx + alpha_p * (1 - kept_idx)
# Do affine transformation
return a * x + b
def _merge_function(self, inputs):
alpha = K.random_uniform((self.batch_size, 1, 1, 1))
return (alpha * inputs[0]) + ((1 - alpha) * inputs[1])
def bernoulli_sample(prob):
return tf.nn.relu(tf.sign(prob - backend.random_uniform(tf.shape(prob))))
def noised():
eps = K.random_uniform(shape=[1], maxval=self.alpha)
return inputs + K.random_normal(
shape=K.shape(inputs), mean=0., stddev=eps)
def _merge_function(self, inputs):
alpha = K.random_uniform((90, 2, 235, 1))
return (alpha * inputs[0]) + ((1 - alpha) * inputs[1])
def build(self, input_shape):
self.rbm_weight = self.add_weight(
name='rbm_weight',
shape=(input_shape[1], self.output_dim),
initializer='uniform' # Which initializer is optimal?
,
trainable=True)
self.hidden_bias = self.add_weight(name='rbm_hidden_bias',
shape=(self.output_dim, ),
initializer='uniform',
trainable=True)
self.visible_bias = K.variable(initializers.get('uniform')(
(input_shape[1], )),
dtype=K.floatx(),
name='rbm_visible_bias')
# Make symbolic computation objects.
if self.mode == MODE_VISIBLE_BERNOULLI:
# Transform visible units.
self.input_visible = K.placeholder(shape=(None, input_shape[1]),
name='input_visible')
self.transform = K.cast(
K.less(
K.random_uniform(shape=(self.hps['batch_size'],
input_shape[1])),
K.sigmoid(
K.dot(self.input_visible, self.rbm_weight) +
self.hidden_bias)))
self.transform_func = K.function([self.input_visible],
[self.transform])
# Transform hidden units.
self.input_hidden = K.placeholder(shape=(None, self.output_dim),
name='input_hidden')
self.inv_transform = K.cast(
K.less(
K.random_uniform(shape=(self.hps['batch_size'],
input_shape[1])),
K.sigmoid(
K.dot(self.input_hidden, K.transpose(self.rbm_weight))
+ self.visible_bias)))
self.inv_transform_func = K.function([self.input_hidden],
[self.inv_transform])
elif self.mode == MODE_VISIBLE_GAUSSIAN:
# Transform visible units.
self.input_visible = K.placeholder(shape=(None, input_shape[1]),
name='input_visible')
self.transform = K.cast(
K.less(
K.random_uniform(shape=(self.hps['batch_size'],
input_shape[1])),
K.relu(
K.dot(self.input_visible, self.rbm_weight) +
self.hidden_bias))) #?
self.transform_func = K.function([self.input_visible],
[self.transform])
# Transform hidden units.
self.input_hidden = K.placeholder(shape=(None, self.output_dim),
name='input_hidden')
self.inv_transform = Ke.multivariate_normal_diag(
loc=(K.dot(self.input_hidden, K.transpose(self.rbm_weight)) +
self.visible_bias),
scale_diag=np.ones(shape=(self.hps['batch_size'],
input_shape[1]))).sample()
self.inv_transform_func = K.function([self.input_hidden],
[self.inv_transform])
else:
# TODO
pass
# Calculate free energy. #?
self.free_energy = -1 * (K.squeeze(K.dot(self.input_visible, K.expand_dims(self.visible_bias, axis=-1)), -1) +\
K.sum(K.log(1 + K.exp(K.dot(self.input_visible, self.rbm_weight) +\
self.hidden_bias)), axis=-1))
self.free_energy_func = K.function([self.input_visible],
[self.free_energy])
super(RBM, self).build(input_shape)
def fit(self, V, verbose=1):
"""Train RBM with the data V.
Parameters
----------
V : 2d numpy array
Visible data (batch size x input_dim).
verbose : integer
Verbose mode (default, 1).
"""
num_step = V.shape[0] // self.hps['batch_size'] \
if V.shape[0] % self.hps['batch_size'] == 0 else V.shape[0] // self.hps['batch_size'] + 1 # Exception processing?
for k in range(self.hps['epochs']):
if verbose == 1:
print(k + 1, '/', self.hps['epochs'], ' epochs', end='\r')
if self.mode == MODE_VISIBLE_BERNOULLI:
# Contrastive divergence.
v_pos = self.input_visible
h_pos = self.transform
v_neg = K.cast(K.less(
K.random_uniform(shape=(self.hps['batch_size'],
V.shape[1])),
K.sigmoid(
K.dot(h_pos, K.transpose(self.rbm_weight)) +
self.visible_bias)),
dtype=np.float32)
h_neg = K.sigmoid(
K.dot(v_neg, self.rbm_weight) + self.hidden_bias)
update = K.transpose(K.transpose(K.dot(K.transpose(v_pos), h_pos)) \
- K.dot(K.transpose(h_neg), v_neg))
self.rbm_weight_update_func = K.function(
[self.input_visible],
[K.update_add(self.rbm_weight, self.hps['lr'] * update)])
self.hidden_bias_update_func = K.function([self.input_visible]
, [K.update_add(self.hidden_bias, self.hps['lr'] \
* (K.sum(h_pos, axis=0) - K.sum(h_neg, axis=0)))])
self.visible_bias_update_func = K.function([self.input_visible]
, [K.update_add(self.visible_bias, self.hps['lr'] \
* (K.sum(v_pos, axis=0) - K.sum(v_neg, axis=0)))])
# Create the fist visible nodes sampling object.
self.sample_first_visible = K.function([self.input_visible],
[v_neg])
elif self.mode == MODE_VISIBLE_GAUSSIAN:
# Contrastive divergence.
v_pos = self.input_visible
h_pos = self.transform
v_neg = Ke.multivariate_normal_diag(
loc=(K.dot(h_pos, K.transpose(self.rbm_weight)) +
self.visible_bias),
scale_diag=np.ones(shape=(self.hps['batch_size'],
V.shape[1]))).sample()
h_neg = K.sigmoid(
K.dot(v_neg, self.rbm_weight) + self.hidden_bias)
update = K.transpose(K.transpose(K.dot(K.transpose(v_pos), h_pos)) \
- K.dot(K.transpose(h_neg), v_neg))
self.rbm_weight_update_func = K.function(
[self.input_visible],
[K.update_add(self.rbm_weight, self.hps['lr'] * update)])
self.hidden_bias_update_func = K.function([self.input_visible]
, [K.update_add(self.hidden_bias, self.hps['lr'] \
* (K.sum(h_pos, axis=0) - K.sum(h_neg, axis=0)))])
self.visible_bias_update_func = K.function([self.input_visible]
, [K.update_add(self.visible_bias, self.hps['lr'] \
* (K.sum(v_pos, axis=0) - K.sum(v_neg, axis=0)))])
# Create the fist visible nodes sampling object.
self.sample_first_visible = K.function([self.input_visible],
[v_neg])
else:
pass
for i in range(num_step):
if i == (num_step - 1):
if self.mode == MODE_VISIBLE_BERNOULLI:
# Contrastive divergence.
v_pos = self.input_visible
h_pos = self.transform
v_neg = K.cast(K.less(
K.random_uniform(shape=(
V.shape[0] -
int(i * self.hps['batch_size'], V.shape[1]))),
K.sigmoid(
K.dot(h_pos, K.transpose(self.rbm_weight)) +
self.visible_bias)),
dtype=np.float32)
h_neg = K.sigmoid(
K.dot(v_neg, self.rbm_weight) + self.hidden_bias)
update = K.transpose(K.transpose(K.dot(K.transpose(v_pos), h_pos)) \
- K.dot(K.transpose(h_neg), v_neg))
self.rbm_weight_update_func = K.function(
[self.input_visible], [
K.update_add(self.rbm_weight,
self.hps['lr'] * update)
])
self.hidden_bias_update_func = K.function([self.input_visible]
, [K.update_add(self.hidden_bias, self.hps['lr'] \
* (K.sum(h_pos, axis=0) - K.sum(h_neg, axis=0)))])
self.visible_bias_update_func = K.function([self.input_visible]
, [K.update_add(self.visible_bias, self.hps['lr'] \
* (K.sum(v_pos, axis=0) - K.sum(v_neg, axis=0)))])
# Create the fist visible nodes sampling object.
self.sample_first_visible = K.function(
[self.input_visible], [v_neg])
elif self.mode == MODE_VISIBLE_GAUSSIAN:
# Contrastive divergence.
v_pos = self.input_visible
h_pos = self.transform
v_neg = Ke.multivariate_normal_diag(
loc=(K.dot(h_pos, K.transpose(self.rbm_weight)) +
self.visible_bias),
scale_diag=np.ones(shape=(
V.shape[0] -
int(i * self.hps['batch_size'], V.shape[1])
))).sample()
h_neg = K.sigmoid(
K.dot(v_neg, self.rbm_weight) + self.hidden_bias)
update = K.transpose(K.transpose(K.dot(K.transpose(v_pos), h_pos)) \
- K.dot(K.transpose(h_neg), v_neg))
self.rbm_weight_update_func = K.function(
[self.input_visible], [
K.update_add(self.rbm_weight,
self.hps['lr'] * update)
])
self.hidden_bias_update_func = K.function([self.input_visible]
, [K.update_add(self.hidden_bias, self.hps['lr'] \
* (K.sum(h_pos, axis=0) - K.sum(h_neg, axis=0)))])
self.visible_bias_update_func = K.function([self.input_visible]
, [K.update_add(self.visible_bias, self.hps['lr'] \
* (K.sum(v_pos, axis=0) - K.sum(v_neg, axis=0)))])
# Create the fist visible nodes sampling object.
self.sample_first_visible = K.function(
[self.input_visible], [v_neg])
else:
pass
V_batch = [V[int(i * self.hps['batch_size']):V.shape[0]]]
# Train.
self.rbm_weight_update_func(V_batch)
self.hidden_bias_update_func(V_batch)
self.visible_bias_update_func(V_batch)
else:
V_batch = [
V[int(i * self.hps['batch_size']):int(
(i + 1) * self.hps['batch_size'])]
]
# Train.
self.rbm_weight_update_func(V_batch)
self.hidden_bias_update_func(V_batch)
self.visible_bias_update_func(V_batch)
# Calculate a training score by each step.
# Free energy of the input visible nodes.
fe = self.cal_free_energy(V_batch)
# Free energy of the first sampled visible nodes.
V_p_batch = self.sample_first_visible(V_batch)
fe_p = self.cal_free_energy(V_p_batch)
score = np.mean(np.abs(fe[0] - fe_p[0])) # Scale?
print('\n{0:d}/{1:d}, score: {2:f}'.format(
i + 1, num_step, score))