def _run_feg(self, X, X_val):
"""Calculate difference between average free energies of subsets
of validation and training sets to monitor overfitting,
as proposed in [2]. If the model is not overfitting at all, this
quantity should be close to zero. Once this value starts
growing, the model is overfitting and the value ("free energy gap")
represents the amount of overfitting.
"""
self._free_energy_op = tf.get_collection('free_energy_op')[0]
train_fes = []
for _, X_b in zip(range(self.metrics_config['n_batches_for_feg']),
batch_iter(X, batch_size=self.batch_size)):
train_fe = self._tf_session.run(
self._free_energy_op, feed_dict=self._make_tf_feed_dict(X_b))
train_fes.append(train_fe)
val_fes = []
for _, X_vb in zip(range(self.metrics_config['n_batches_for_feg']),
batch_iter(X_val, batch_size=self.batch_size)):
val_fe = self._tf_session.run(
self._free_energy_op, feed_dict=self._make_tf_feed_dict(X_vb))
val_fes.append(val_fe)
feg = np.mean(val_fes) - np.mean(train_fes)
summary_value = [
summary_pb2.Summary.Value(tag=self._metrics_names_map['feg'],
simple_value=feg)
]
feg_s = summary_pb2.Summary(value=summary_value)
self._tf_val_writer.add_summary(feg_s, self.iter_)
return feg
def _train_epoch(self, X):
results = [[] for _ in range(len(self._train_metrics_map))]
for X_batch in batch_iter(X, self.batch_size, verbose=self.verbose):
self.iter_ += 1
if self.iter_ % self.metrics_config[
'train_metrics_every_iter'] == 0:
run_ops = [
v for _, v in sorted(self._train_metrics_map.items())
]
run_ops += [self._tf_merged_summaries, self._train_op]
outputs = \
self._tf_session.run(run_ops,
feed_dict=self._make_tf_feed_dict(X_batch))
values = outputs[:len(self._train_metrics_map)]
for i, v in enumerate(values):
results[i].append(v)
train_s = outputs[len(self._train_metrics_map)]
self._tf_train_writer.add_summary(train_s, self.iter_)
else:
self._tf_session.run(
self._train_op, feed_dict=self._make_tf_feed_dict(X_batch))
# aggregate and return metrics values
results = map(lambda r: np.mean(r) if r else None, results)
return dict(zip(sorted(self._train_metrics_map), results))
def reconstruct(self, X):
"""Compute p(v|h_0=q, h...)=p(v|h_0=q), where q=p(h_0|v=x)"""
self._reconstruction = tf.get_collection('reconstruction')[0]
X_recon = np.zeros_like(X)
start = 0
for X_b in batch_iter(X,
batch_size=self.batch_size,
verbose=self.verbose,
desc='reconstruction'):
X_recon_b = self._reconstruction.eval(
feed_dict=self._make_tf_feed_dict(X_b))
X_recon[start:(start + self.batch_size)] = X_recon_b
start += self.batch_size
return X_recon
def transform(self, X, np_dtype=None):
"""Compute hidden units' activation probabilities."""
np_dtype = np_dtype or self._np_dtype
self._transform_op = tf.get_collection('transform_op')[0]
H = np.zeros((len(X), self.n_hidden), dtype=np_dtype)
start = 0
for X_b in batch_iter(X,
batch_size=self.batch_size,
verbose=self.verbose,
desc='transform'):
H_b = self._transform_op.eval(
feed_dict=self._make_tf_feed_dict(X_b))
H[start:(start + self.batch_size)] = H_b
start += self.batch_size
return H
def _run_val_metrics(self, X_val):
val_msres, val_n_mf_updates = [], []
for X_vb in batch_iter(X_val, batch_size=self.batch_size):
msre, n_mf_upds = self._tf_session.run(
[self._msre, self._n_mf_updates],
feed_dict=self._make_tf_feed_dict(X_vb))
val_msres.append(msre)
val_n_mf_updates.append(n_mf_upds)
mean_msre = np.mean(val_msres)
mean_n_mf_updates = np.mean(val_n_mf_updates)
s = summary_pb2.Summary(value=[
summary_pb2.Summary.Value(tag='mean_squared_recon_error',
simple_value=mean_msre),
summary_pb2.Summary.Value(tag='n_mf_updates',
simple_value=mean_n_mf_updates),
])
self._tf_val_writer.add_summary(s, self.iter_)
return mean_msre, mean_n_mf_updates
def log_proba(self, X_test, log_Z):
"""Estimate variational lower-bound on a test set, as in [5].
Currently implemented only for 2-layer binary BM.
"""
assert self.n_layers_ == 2
for L in [self._v_layer] + self._h_layers:
assert isinstance(L, BernoulliLayer)
self._log_proba = tf.get_collection('log_proba')[0]
P = np.zeros(len(X_test))
start = 0
for X_b in batch_iter(X_test,
batch_size=self.batch_size,
verbose=self.verbose):
P_b = self._log_proba.eval(feed_dict=self._make_tf_feed_dict(X_b))
P[start:(start + self.batch_size)] = P_b
start += self.batch_size
return P - log_Z
def _run_val_metrics(self, X_val):
results = [[] for _ in range(len(self._val_metrics_map))]
for X_vb in batch_iter(X_val, batch_size=self.batch_size):
run_ops = [v for _, v in sorted(self._val_metrics_map.items())]
values = \
self._tf_session.run(run_ops,
feed_dict=self._make_tf_feed_dict(X_vb))
for i, v in enumerate(values):
results[i].append(v)
for i, r in enumerate(results):
results[i] = np.mean(r) if r else None
summary_value = []
for i, m in enumerate(sorted(self._val_metrics_map)):
summary_value.append(
summary_pb2.Summary.Value(tag=self._metrics_names_map[m],
simple_value=results[i]))
val_s = summary_pb2.Summary(value=summary_value)
self._tf_val_writer.add_summary(val_s, self.iter_)
return dict(zip(sorted(self._val_metrics_map), results))
def _train_epoch(self, X):
train_msres, train_n_mf_updates = [], []
for X_batch in batch_iter(X, self.batch_size, verbose=self.verbose):
self.iter_ += 1
if self.iter_ % self.train_metrics_every_iter == 0:
msre, n_mf_upds, _, s = self._tf_session.run(
[
self._msre, self._n_mf_updates, self._train_op,
self._tf_merged_summaries
],
feed_dict=self._make_tf_feed_dict(X_batch))
train_msres.append(msre)
train_n_mf_updates.append(n_mf_upds)
self._tf_train_writer.add_summary(s, self.iter_)
else:
self._tf_session.run(
self._train_op, feed_dict=self._make_tf_feed_dict(X_batch))
return (np.mean(train_msres) if train_msres else None,
np.mean(train_n_mf_updates) if train_n_mf_updates else None)