def finalize(self,
optimizer,
optimizer_args=None,
suffix="",
*args,
**kwargs):
"""Finalizes the network.
Arguments:
optimizer: (tf.train.Optimizer) An optimizer class from those available at tf.train.Optimizer.
optimizer_args: (dict) A dictionary of arguments for the __init__ method of the chosen optimizer.
Returns: None
"""
if len(self.layers) == 0:
raise RuntimeError("Cannot finalize an empty network.")
if self.finalized:
raise RuntimeError("Can only finalize a network once.")
optimizer_args = {} if optimizer_args is None else optimizer_args
self.optimizer = optimizer(**optimizer_args)
# Construct all variables.
with self.sess.as_default():
with tf.variable_scope(self.name):
self.scaler = TensorStandardScaler(
self.layers[0].get_input_dim(), suffix)
for i, layer in enumerate(self.layers):
with tf.variable_scope(("Layer%i" + suffix) % i):
layer.construct_vars()
self.decays.extend(layer.get_decays())
self.optvars.extend(layer.get_vars())
self.nonoptvars.extend(self.scaler.get_vars())
# Setup training
with tf.variable_scope(self.name):
self.optimizer = optimizer(**optimizer_args)
self.sy_train_in = tf.placeholder(
dtype=tf.float32,
shape=[self.num_nets, None, self.layers[0].get_input_dim()],
name="training_inputs")
self.sy_train_targ = tf.placeholder(
dtype=tf.float32,
shape=[self.num_nets, None, self.layers[-1].get_output_dim()],
name="training_targets")
train_loss = tf.reduce_sum(
self._compile_losses(self.sy_train_in, self.sy_train_targ))
train_loss += tf.add_n(self.decays)
self.mse_loss = self._compile_losses(self.sy_train_in,
self.sy_train_targ)
self.train_op = self.optimizer.minimize(train_loss,
var_list=self.optvars)
# Initialize all variables
self.sess.run(
tf.variables_initializer(self.optvars + self.nonoptvars +
self.optimizer.variables()))
# Setup prediction
with tf.variable_scope(self.name):
self.sy_pred_in2d = tf.placeholder(
dtype=tf.float32,
shape=[None, self.layers[0].get_input_dim()],
name="2D_training_inputs")
self.sy_pred_mean2d_fac = self.create_prediction_tensors(
self.sy_pred_in2d, factored=True)[0]
self.sy_pred_mean2d = tf.reduce_mean(self.sy_pred_mean2d_fac,
axis=0)
self.sy_pred_var2d = tf.reduce_mean(
tf.square(self.sy_pred_mean2d_fac - self.sy_pred_mean2d),
axis=0)
self.sy_pred_in3d = tf.placeholder(
dtype=tf.float32,
shape=[self.num_nets, None, self.layers[0].get_input_dim()],
name="3D_training_inputs")
self.sy_pred_mean3d_fac = \
self.create_prediction_tensors(self.sy_pred_in3d, factored=True)[0]
# Load model if needed
if self.model_loaded:
with self.sess.as_default():
params_dict = loadmat(
os.path.join(self.model_dir, "%s.mat" % self.name))
all_vars = self.nonoptvars + self.optvars
for i, var in enumerate(all_vars):
var.load(params_dict[str(i)])
self.finalized = True
def finalize(self, optimizer, optimizer_args=None, *args, **kwargs):
"""Finalizes the network.
Arguments:
optimizer: (tf.train.Optimizer) An optimizer class from those available at tf.train.Optimizer.
optimizer_args: (dict) A dictionary of arguments for the __init__ method of the chosen optimizer.
Returns: None
"""
if len(self.layers) == 0:
raise RuntimeError("Cannot finalize an empty network.")
if self.finalized:
raise RuntimeError("Can only finalize a network once.")
optimizer_args = {} if optimizer_args is None else optimizer_args
self.optimizer = optimizer(**optimizer_args)
# Add variance output.
self.layers[-1].set_output_dim(2 * self.layers[-1].get_output_dim())
# Remove last activation to isolate variance from activation function.
self.end_act = self.layers[-1].get_activation()
self.end_act_name = self.layers[-1].get_activation(as_func=False)
self.layers[-1].unset_activation()
# Construct all variables.
with self.sess.as_default():
with tf.variable_scope(self.name):
self.scaler = TensorStandardScaler(
self.layers[0].get_input_dim())
self.max_logvar = tf.Variable(
np.ones([1, self.layers[-1].get_output_dim() // 2]) / 2.,
dtype=tf.float32,
name="max_log_var")
self.min_logvar = tf.Variable(
-np.ones([1, self.layers[-1].get_output_dim() // 2]) * 10.,
dtype=tf.float32,
name="min_log_var")
for i, layer in enumerate(self.layers):
with tf.variable_scope("Layer%i" % i):
layer.construct_vars()
self.decays.extend(layer.get_decays())
self.optvars.extend(layer.get_vars())
self.optvars.extend([self.max_logvar, self.min_logvar])
self.nonoptvars.extend(self.scaler.get_vars())
# Set up training
with tf.variable_scope(self.name):
self.optimizer = optimizer(**optimizer_args)
self.sy_train_in = tf.placeholder(
dtype=tf.float32,
shape=[self.num_nets, None, self.layers[0].get_input_dim()],
name="training_inputs")
self.sy_train_targ = tf.placeholder(
dtype=tf.float32,
shape=[
self.num_nets, None, self.layers[-1].get_output_dim() // 2
],
name="training_targets")
train_loss = tf.reduce_sum(
self._compile_losses(self.sy_train_in,
self.sy_train_targ,
inc_var_loss=True))
train_loss += tf.add_n(self.decays)
# regularization to ensure max_logvar not grow too much beyond training distribution
# and min_logvar not drop below training distribution
train_loss += 0.01 * tf.reduce_sum(
self.max_logvar) - 0.01 * tf.reduce_sum(self.min_logvar)
self.mse_loss = self._compile_losses(self.sy_train_in,
self.sy_train_targ,
inc_var_loss=False)
self.train_op = self.optimizer.minimize(train_loss,
var_list=self.optvars)
# Initialize all variables
self.sess.run(
tf.variables_initializer(self.optvars + self.nonoptvars +
self.optimizer.variables()))
# Set up prediction
with tf.variable_scope(self.name):
self.sy_pred_in2d = tf.placeholder(
dtype=tf.float32,
shape=[None, self.layers[0].get_input_dim()],
name="2D_training_inputs")
self.sy_pred_mean2d_fac, self.sy_pred_var2d_fac = \
self.create_prediction_tensors(self.sy_pred_in2d, factored=True)
self.sy_pred_mean2d = tf.reduce_mean(self.sy_pred_mean2d_fac,
axis=0)
self.sy_pred_var2d = tf.reduce_mean(self.sy_pred_var2d_fac, axis=0) + \
tf.reduce_mean(tf.square(self.sy_pred_mean2d_fac - self.sy_pred_mean2d), axis=0)
self.sy_pred_in3d = tf.placeholder(
dtype=tf.float32,
shape=[self.num_nets, None, self.layers[0].get_input_dim()],
name="3D_training_inputs")
self.sy_pred_mean3d_fac, self.sy_pred_var3d_fac = \
self.create_prediction_tensors(self.sy_pred_in3d, factored=True)
# Load model if needed
# self.optimizer.variables() no need to save and load
if self.model_loaded:
with self.sess.as_default():
params_dict = loadmat(
os.path.join(self.model_dir, "%s.mat" % self.name))
all_vars = self.nonoptvars + self.optvars
for i, var in enumerate(all_vars):
var.load(params_dict[str(i)])
self.finalized = True