def _initialize_weights(self):
self.x = tf.placeholder(tf.float32, [None, self.n_visible])
self.y = tf.placeholder(tf.float32, [None, self.n_hidden])
self.w = tf.Variable(tf_xavier_init(self.n_visible,
self.n_hidden,
const=1.0),
dtype=tf.float32)
self.vbias = tf.Variable(tf.zeros([self.n_visible]), dtype=tf.float32)
self.hbias = tf.Variable(tf.zeros([self.n_hidden]), dtype=tf.float32)
self.delta_w = tf.Variable(tf.zeros([self.n_visible, self.n_hidden]),
dtype=tf.float32)
self.delta_vbias = tf.Variable(tf.zeros([self.n_visible]),
dtype=tf.float32)
self.delta_hbias = tf.Variable(tf.zeros([self.n_hidden]),
dtype=tf.float32)
def def_network(self, lr):
next_layer_input = self.x
for i in range(len(self.layer_sizes)):
dim = self.layer_sizes[i]
input_dim = int(next_layer_input.get_shape()[1])
W = tf.Variable(tf_xavier_init(input_dim, dim, const=1.0),
dtype=tf.float32)
b = tf.Variable(tf.zeros([dim]), dtype=tf.float32)
self.encoding_matrices.append(W)
self.encoding_biases.append(b)
output = tf.nn.sigmoid(tf.matmul(next_layer_input, W) + b)
next_layer_input = output
self.encoded_x = next_layer_input
# FFNN
self.W = tf.Variable(tf.truncated_normal(
[self.layer_sizes[-1], self.FFNN_layer]),
name='Weight_FFNN',
dtype=tf.float32)
self.b = tf.Variable(tf.zeros([self.FFNN_layer], np.float32),
name='bias_FFNN',
dtype=tf.float32)
self.W_out = tf.Variable(tf.truncated_normal([self.FFNN_layer, 1]),
name='Weight_output',
dtype=tf.float32)
self.b_out = tf.Variable(tf.zeros([1], np.float32),
name='bias_output',
dtype=tf.float32)
# compute cost
# self.cost = tf.sqrt(tf.reduce_mean(tf.square(self.x - self.reconstructed_x)))
# self.cost = tf.sqrt(tf.reduce_mean(tf.square((tf.matmul(self.x, self.W) + self.b) - self.y)))
self.y_ = tf.matmul(self.encoded_x, self.W) + self.b
self.y_logits = tf.matmul(self.y_, self.W_out) + self.b_out
tf.add_to_collection('pred_network', self.y_logits)
self.cost = tf.reduce_mean(
tf.nn.sigmoid_cross_entropy_with_logits(logits=self.y_logits,
labels=self.y,
name='cross_entropy'))
self.optimizer = tf.train.GradientDescentOptimizer(lr).minimize(
self.cost)
self.predict_op = tf.nn.sigmoid(self.y_logits)
init = tf.global_variables_initializer()
self.sess = tf.Session()
self.sess.run(init)
def __init__(
self,
n_visible,
n_hidden,
learning_rate=0.01,
momentum=0.95,
xavier_const=1.0,
err_function='mse',
use_tqdm=False,
# DEPRECATED:
tqdm=None):
if not 0.0 <= momentum <= 1.0:
raise ValueError('momentum should be in range [0, 1]')
if err_function not in {'mse', 'cosine'}:
raise ValueError(
'err_function should be either \'mse\' or \'cosine\'')
self._use_tqdm = use_tqdm
self._tqdm = None
if use_tqdm or tqdm is not None:
from tqdm import tqdm
self._tqdm = tqdm
self.n_visible = n_visible
self.n_hidden = n_hidden
self.learning_rate = learning_rate
self.momentum = momentum
self.x = tf.placeholder(tf.float32, [None, self.n_visible])
self.y = tf.placeholder(tf.float32, [None, self.n_hidden])
self.w = tf.Variable(tf_xavier_init(self.n_visible,
self.n_hidden,
const=xavier_const),
dtype=tf.float32)
self.visible_bias = tf.Variable(tf.zeros([self.n_visible]),
dtype=tf.float32)
self.hidden_bias = tf.Variable(tf.zeros([self.n_hidden]),
dtype=tf.float32)
self.delta_w = tf.Variable(tf.zeros([self.n_visible, self.n_hidden]),
dtype=tf.float32)
self.delta_visible_bias = tf.Variable(tf.zeros([self.n_visible]),
dtype=tf.float32)
self.delta_hidden_bias = tf.Variable(tf.zeros([self.n_hidden]),
dtype=tf.float32)
self.update_weights = None
self.update_deltas = None
self.compute_hidden = None
self.compute_visible = None
self.compute_visible_from_hidden = None
self._initialize_vars()
assert self.update_weights is not None
assert self.update_deltas is not None
assert self.compute_hidden is not None
assert self.compute_visible is not None
assert self.compute_visible_from_hidden is not None
if err_function == 'cosine':
x1_norm = tf.nn.l2_normalize(self.x, 1)
x2_norm = tf.nn.l2_normalize(self.compute_visible, 1)
cos_val = tf.reduce_mean(tf.reduce_sum(tf.mul(x1_norm, x2_norm),
1))
self.compute_err = tf.acos(cos_val) / tf.constant(np.pi)
else:
self.compute_err = tf.reduce_mean(
tf.square(self.x - self.compute_visible))
init = tf.global_variables_initializer()
self.sess = tf.Session()
self.sess.run(init)
def __init__(self,
n_visible,
n_hidden,
momentum=0.95,
xavier_const=1.0,
err_function='mse',
use_tqdm=False,
logsdir = None,
# DEPRECATED:
tqdm=None):
if not 0.0 <= momentum <= 1.0:
raise ValueError('momentum should be in range [0, 1]')
if err_function not in {'mse', 'cosine', 'rmse'}:
raise ValueError('err_function should be either \'mse\', \'rmse\' or \'cosine\'')
self._use_tqdm = use_tqdm
self._tqdm = None
if use_tqdm or tqdm is not None:
from tqdm import tqdm
self._tqdm = tqdm
self.n_visible = n_visible
self.n_hidden = n_hidden
self.learning_rate_ph = tf.placeholder(tf.float32, ())#learning_rate
self.momentum = momentum
self.x = tf.placeholder(tf.float32, [None, self.n_visible])
self.y = tf.placeholder(tf.float32, [None, self.n_hidden])
# self.w = tf.Variable(tf.random_normal(mean=0.0, stddev=(1/self.n_visible), dtype = tf.float32, shape = [self.n_visible, self.n_hidden]))
# self.w = tf.Variable(tf.random_normal(mean=0.0, stddev=(0.1), dtype = tf.float32, shape = [self.n_visible, self.n_hidden]))
self.w = tf.Variable(tf_xavier_init(self.n_visible, self.n_hidden, const=xavier_const), dtype=tf.float32)
self.visible_bias = tf.Variable(1*tf.ones([self.n_visible]), dtype=tf.float32)
# self.hidden_bias = tf.Variable(tf.random_uniform(minval = 0, maxval = 1, dtype = tf.float32, shape = [self.n_hidden]))
self.hidden_bias = tf.Variable(1*tf.ones([self.n_hidden]), dtype=tf.float32)
self.delta_w = tf.Variable(tf.zeros([self.n_visible, self.n_hidden]), dtype=tf.float32)
self.delta_visible_bias = tf.Variable(tf.zeros([self.n_visible]), dtype=tf.float32)
self.delta_hidden_bias = tf.Variable(tf.zeros([self.n_hidden]), dtype=tf.float32)
self.update_weights = None
self.update_deltas = None
self.compute_hidden = None
self.compute_visible = None
self.compute_visible_from_hidden = None
self._initialize_vars()
assert self.update_weights is not None
assert self.update_deltas is not None
assert self.compute_hidden is not None
assert self.compute_visible is not None
assert self.compute_visible_from_hidden is not None
if err_function == 'cosine':
x1_norm = tf.nn.l2_normalize(self.x, 1)
x2_norm = tf.nn.l2_normalize(self.compute_visible, 1)
cos_val = tf.reduce_mean(tf.reduce_sum(tf.mul(x1_norm, x2_norm), 1))
self.compute_err = tf.acos(cos_val) / tf.constant(np.pi)
elif err_function == 'mse':
self.compute_err = tf.reduce_mean(tf.square(self.x - self.compute_visible))
else: #rmse
self.compute_err = tf.sqrt(tf.reduce_mean(tf.square(self.x - self.compute_visible)))
hist_bv = tf.summary.histogram('visible_bias', self.visible_bias)
hist_b = tf.summary.histogram('hidden_bias', self.hidden_bias)
hist_w = tf.summary.histogram("w", self.w)
self.hists_vars = tf.summary.merge([hist_w] + [hist_b] + [hist_bv])
self.loss_ph = tf.placeholder(tf.float32, shape = (), name="loss_placeholder")
self.summary_loss = tf.summary.scalar("loss", self.loss_ph)
self.hist_metrics = tf.summary.merge([self.summary_loss])
self.logs_dir = logsdir
if self.logs_dir is not None:
self.train_writer = tf.summary.FileWriter(self.logs_dir+"/logs/train")
# self.test_writer = tf.summary.FileWriter(self.logs_dir+"/logs/test")
self.hidden_bias_below_zero = tf.cast(tf.math.greater(tf.zeros_like(self.hidden_bias), self.hidden_bias), tf.float32)
self.bias_below_zero = tf.cast(self.hidden_bias.assign(tf.abs(self.hidden_bias)), tf.float32)
self.hidden_bias_correction = self.hidden_bias.assign(self.hidden_bias*self.hidden_bias_below_zero)
# self.hidden_bias_below_zero = tf.cast(tf.math.greater(tf.zeros_like(self.hidden_bias), self.hidden_bias), tf.float32)
# self.hidden_bias_correction = self.hidden_bias.assign_add(tf.abs(tf.reduce_mean(self.hidden_bias))*self.hidden_bias_below_zero)
self.visible_bias_below_zero = tf.cast(tf.math.greater(tf.zeros_like(self.visible_bias), self.visible_bias), tf.float32)
self.visible_bias_correction = self.visible_bias.assign_add(tf.abs(tf.reduce_mean(self.visible_bias))*self.visible_bias_below_zero)
init = tf.global_variables_initializer()
self.sess = tf.Session()
self.sess.run(init)