def __init__(self,
images=None,
actions=None,
states=None,
sequence_length=None,
reuse_scope=None,
prefix=None):
if sequence_length is None:
sequence_length = FLAGS.sequence_length
if prefix is None:
prefix = tf.placeholder(tf.string, [])
self.prefix = prefix
self.iter_num = tf.placeholder(tf.float32, [])
summaries = []
# Split into timesteps.
actions = tf.split(axis=1, num_or_size_splits=int(actions.get_shape()[1]), value=actions)
actions = [tf.squeeze(act) for act in actions]
states = tf.split(axis=1, num_or_size_splits=int(states.get_shape()[1]), value=states)
states = [tf.squeeze(st) for st in states]
images = tf.split(axis=1, num_or_size_splits=int(images.get_shape()[1]), value=images)
images = [tf.squeeze(img) for img in images]
if reuse_scope is None:
gen_images, gen_states = construct_model(
images,
actions,
states,
iter_num=self.iter_num,
k=FLAGS.schedsamp_k,
use_state=FLAGS.use_state,
num_masks=FLAGS.num_masks,
cdna=FLAGS.model == 'CDNA',
dna=FLAGS.model == 'DNA',
stp=FLAGS.model == 'STP',
context_frames=FLAGS.context_frames)
else: # If it's a validation or test model.
with tf.variable_scope(reuse_scope, reuse=True):
gen_images, gen_states = construct_model(
images,
actions,
states,
iter_num=self.iter_num,
k=FLAGS.schedsamp_k,
use_state=FLAGS.use_state,
num_masks=FLAGS.num_masks,
cdna=FLAGS.model == 'CDNA',
dna=FLAGS.model == 'DNA',
stp=FLAGS.model == 'STP',
context_frames=FLAGS.context_frames)
# L2 loss, PSNR for eval.
loss, psnr_all = 0.0, 0.0
for i, x, gx in zip(
range(len(gen_images)), images[FLAGS.context_frames:],
gen_images[FLAGS.context_frames - 1:]):
recon_cost = mean_squared_error(x, gx)
psnr_i = peak_signal_to_noise_ratio(x, gx)
psnr_all += psnr_i
summaries.append(
tf.summary.scalar(prefix + '_recon_cost' + str(i), recon_cost))
summaries.append(tf.summary.scalar(prefix + '_psnr' + str(i), psnr_i))
loss += recon_cost
for i, state, gen_state in zip(
range(len(gen_states)), states[FLAGS.context_frames:],
gen_states[FLAGS.context_frames - 1:]):
state_cost = mean_squared_error(state, gen_state) * 1e-4
summaries.append(
tf.summary.scalar(prefix + '_state_cost' + str(i), state_cost))
loss += state_cost
summaries.append(tf.summary.scalar(prefix + '_psnr_all', psnr_all))
self.psnr_all = psnr_all
self.loss = loss = loss / np.float32(len(images) - FLAGS.context_frames)
summaries.append(tf.summary.scalar(prefix + '_loss', loss))
self.lr = tf.placeholder_with_default(FLAGS.learning_rate, ())
self.train_op = tf.train.AdamOptimizer(self.lr).minimize(loss)
self.summ_op = tf.summary.merge(summaries)
def __init__(self,
images=None,
actions=None,
states=None,
sequence_length=None,
reuse_scope=None,
prefix=None):
if sequence_length is None:
sequence_length = FLAGS.sequence_length
if prefix is None:
prefix = tf.placeholder(tf.string, [])
self.prefix = prefix
self.iter_num = tf.placeholder(tf.float32, [])
summaries = []
# Split into timesteps.
#分为多个时间步
actions = tf.split(axis=1,
num_or_size_splits=int(actions.get_shape()[1]),
value=actions)
actions = [tf.squeeze(act) for act in actions]
states = tf.split(axis=1,
num_or_size_splits=int(states.get_shape()[1]),
value=states)
states = [tf.squeeze(st) for st in states]
images = tf.split(axis=1,
num_or_size_splits=int(images.get_shape()[1]),
value=images)
images = [tf.squeeze(img) for img in images]
self.images = images
if reuse_scope is None:
gen_images, gen_states = construct_model(
images,
actions,
states,
iter_num=self.iter_num,
k=FLAGS.schedsamp_k,
use_state=FLAGS.use_state,
num_masks=FLAGS.num_masks,
cdna=FLAGS.model == 'CDNA',
dna=FLAGS.model == 'DNA',
stp=FLAGS.model == 'STP',
context_frames=FLAGS.context_frames)
else: # If it's a validation or test model. 如果是一个验证或者测试模型
with tf.variable_scope(reuse_scope, reuse=True):
gen_images, gen_states = construct_model(
images,
actions,
states,
iter_num=self.iter_num,
k=FLAGS.schedsamp_k,
use_state=FLAGS.use_state,
num_masks=FLAGS.num_masks,
cdna=FLAGS.model == 'CDNA',
dna=FLAGS.model == 'DNA',
stp=FLAGS.model == 'STP',
context_frames=FLAGS.context_frames)
self.gen_images = gen_images
# L2 loss, PSNR for eval. L2损失PSNR为评估值
loss, psnr_all = 0.0, 0.0
for i, x, gx in zip(range(len(gen_images)),
images[FLAGS.context_frames:],
gen_images[FLAGS.context_frames - 1:]):
recon_cost = mean_squared_error(x, gx)
psnr_i = peak_signal_to_noise_ratio(x, gx)
psnr_all += psnr_i
summaries.append(
tf.summary.scalar(prefix + '_recon_cost' + str(i), recon_cost))
summaries.append(
tf.summary.scalar(prefix + '_psnr' + str(i), psnr_i))
loss += recon_cost
for i, state, gen_state in zip(range(len(gen_states)),
states[FLAGS.context_frames:],
gen_states[FLAGS.context_frames - 1:]):
state_cost = mean_squared_error(state, gen_state) * 1e-4
summaries.append(
tf.summary.scalar(prefix + '_state_cost' + str(i), state_cost))
loss += state_cost
summaries.append(tf.summary.scalar(prefix + '_psnr_all', psnr_all))
self.psnr_all = psnr_all
self.loss = loss = loss / np.float32(
len(images) - FLAGS.context_frames)
summaries.append(tf.summary.scalar(prefix + '_loss', loss))
self.lr = tf.placeholder_with_default(FLAGS.learning_rate, ())
self.train_op = tf.train.AdamOptimizer(self.lr).minimize(loss)
self.summ_op = tf.summary.merge(summaries)
def __init__(self,
images=None,
actions=None,
states=None,
sequence_length=None,
reuse_scope=None,
prefix=None):
if sequence_length is None:
sequence_length = FLAGS.sequence_length
if prefix is None:
prefix = tf.placeholder(tf.string, [])
self.prefix = prefix
self.iter_num = tf.placeholder(tf.float32, [])
summaries = []
# Split into timesteps.
actions = tf.split(axis=1, num_or_size_splits=int(actions.get_shape()[1]), value=actions)
actions = [tf.squeeze(act) for act in actions]
states = tf.split(axis=1, num_or_size_splits=int(states.get_shape()[1]), value=states)
states = [tf.squeeze(st) for st in states]
images = tf.split(axis=1, num_or_size_splits=int(images.get_shape()[1]), value=images)
images = [tf.squeeze(img) for img in images]
if reuse_scope is None:
gen_images, gen_states, latent_loss = construct_model(
images,
actions,
states,
iter_num=self.iter_num,
k=FLAGS.schedsamp_k,
use_state=FLAGS.use_state,
num_masks=FLAGS.num_masks,
cdna=FLAGS.model == 'CDNA',
dna=FLAGS.model == 'DNA',
stp=FLAGS.model == 'STP',
context_frames=FLAGS.context_frames)
else: # If it's a validation or test model.
with tf.variable_scope(reuse_scope, reuse=True):
gen_images, gen_states, latent_loss = construct_model(
images,
actions,
states,
iter_num=self.iter_num,
k=FLAGS.schedsamp_k,
use_state=FLAGS.use_state,
num_masks=FLAGS.num_masks,
cdna=FLAGS.model == 'CDNA',
dna=FLAGS.model == 'DNA',
stp=FLAGS.model == 'STP',
context_frames=FLAGS.context_frames)
# L2 loss, PSNR for eval.
loss, psnr_all = 0.0, 0.0
for i, x, gx in zip(
range(len(gen_images)), images[FLAGS.context_frames:],
gen_images[FLAGS.context_frames - 1:]):
recon_cost = mean_squared_error(x, gx)
psnr_i = peak_signal_to_noise_ratio(x, gx)
psnr_all += psnr_i
summaries.append(
tf.summary.scalar(prefix + '_recon_cost' + str(i), recon_cost))
summaries.append(tf.summary.scalar(prefix + '_psnr' + str(i), psnr_i))
loss += recon_cost
for i, state, gen_state in zip(
range(len(gen_states)), states[FLAGS.context_frames:],
gen_states[FLAGS.context_frames - 1:]):
state_cost = mean_squared_error(state, gen_state) * 1e-4
summaries.append(
tf.summary.scalar(prefix + '_state_cost' + str(i), state_cost))
loss += state_cost
summaries.append(tf.summary.scalar(prefix + '_psnr_all', psnr_all))
self.psnr_all = psnr_all
loss_mse = loss / np.float32(len(images) - FLAGS.context_frames)
loss = loss_mse
if FLAGS.stochastic_model:
values = [0.0, FLAGS.latent_loss_multiplier]
# KL loss is zero till 3rd stage
boundaries = [FLAGS.num_iterations_1st_stage + FLAGS.num_iterations_2nd_stage]
latent_multiplier = tf.train.piecewise_constant(
tf.cast(self.iter_num, tf.int32),
boundaries, values)
loss += latent_multiplier * latent_loss
summaries.append(tf.summary.scalar(prefix + '_loss_latent', latent_loss))
self.loss = loss
self.images = images
self.gen_images = gen_images
summaries.append(tf.summary.scalar(prefix + '_loss', loss))
summaries.append(tf.summary.scalar(prefix + '_loss_mse', loss_mse))
summaries.append(tf.summary.image(prefix + '_real', images[-1]))
summaries.append(tf.summary.image(prefix + '_gen', gen_images[-1]))
self.lr = tf.placeholder_with_default(FLAGS.learning_rate, ())
self.train_op = tf.train.AdamOptimizer(self.lr).minimize(loss)
self.summ_op = tf.summary.merge(summaries)
