def class_score_mmm(name='class_score'):
# Create a custom scalar summary as a margin chart.
# Displays the mean value and the min and max values as the upper and lower limits.
tf_class_scores = tf.placeholder(tf.float32, shape=(None, ))
tf_mean_op = summary_lib.scalar(name, tf.math.reduce_mean(tf_class_scores))
tf_min_op = summary_lib.scalar(name + '/min',
tf.math.reduce_min(tf_class_scores))
tf_max_op = summary_lib.scalar(name + '/max',
tf.math.reduce_max(tf_class_scores))
tf_class_score_chart = layout_pb2.Chart(
title=name,
margin=layout_pb2.MarginChartContent(
series=[
layout_pb2.MarginChartContent.Series(
value=tf_mean_op.name.split(
':')[0], #name + '/scalar_summary',
lower=tf_min_op.name.split(':')
[0], #name + '/min/scalar_summary',
upper=tf_max_op.name.split(':')[
0] #name + '/max/scalar_summary'
),
], ))
return tf_class_scores, tf_class_score_chart
def write_loss(iterations, max_iterations, trainer, train_writer):
print("Iteration: %08d/%08d" % (iterations + 1, max_iterations))
members = [attr for attr in dir(trainer) \
if not callable(getattr(trainer, attr)) and not attr.startswith("__") and 'loss' in attr]
for m in members:
train_writer.add_summary(summary.scalar(m, getattr(trainer, m)), iterations + 1)
members = [attr for attr in dir(trainer) \
if not callable(getattr(trainer, attr)) and not attr.startswith("__") and 'acc' in attr]
for m in members:
train_writer.add_summary(summary.scalar(m, getattr(trainer, m)), iterations + 1)
def train_Gnet(self, count):
self.netG.zero_grad()
errG_total = 0
flag = count % 100
batch_size = self.real_imgs[0].size(0)
criterion, mu, logvar = self.criterion, self.mu, self.logvar
real_labels = self.real_labels[:batch_size]
for i in range(self.num_Ds):
outputs = self.netsD[i](self.fake_imgs[i], mu)
errG = criterion(outputs[0], real_labels)
map_loss = criterion(outputs[1], self.real_maps[i])
if len(outputs) > 1 and cfg.TRAIN.COEFF.UNCOND_LOSS > 0:
errG_patch = cfg.TRAIN.COEFF.UNCOND_LOSS *\
criterion(outputs[2], real_labels)
errG = errG + errG_patch
errG_total = errG_total + errG + map_loss
if flag == 0:
summary_D = summary.scalar('G_loss%d' % i, errG.data[0])
self.summary_writer.add_summary(summary_D, count)
# Compute color consistency losses
if cfg.TRAIN.COEFF.COLOR_LOSS > 0:
if self.num_Ds > 1:
mu1, covariance1 = compute_mean_covariance(self.fake_imgs[-1])
mu2, covariance2 = \
compute_mean_covariance(self.fake_imgs[-2].detach())
like_mu2 = cfg.TRAIN.COEFF.COLOR_LOSS * nn.MSELoss()(mu1, mu2)
like_cov2 = cfg.TRAIN.COEFF.COLOR_LOSS * 5 * \
nn.MSELoss()(covariance1, covariance2)
errG_total = errG_total + like_mu2 + like_cov2
if flag == 0:
sum_mu = summary.scalar('G_like_mu2', like_mu2.data[0])
self.summary_writer.add_summary(sum_mu, count)
sum_cov = summary.scalar('G_like_cov2', like_cov2.data[0])
self.summary_writer.add_summary(sum_cov, count)
if self.num_Ds > 2:
mu1, covariance1 = compute_mean_covariance(self.fake_imgs[-2])
mu2, covariance2 = \
compute_mean_covariance(self.fake_imgs[-3].detach())
like_mu1 = cfg.TRAIN.COEFF.COLOR_LOSS * nn.MSELoss()(mu1, mu2)
like_cov1 = cfg.TRAIN.COEFF.COLOR_LOSS * 5 * \
nn.MSELoss()(covariance1, covariance2)
errG_total = errG_total + like_mu1 + like_cov1
if flag == 0:
sum_mu = summary.scalar('G_like_mu1', like_mu1.data[0])
self.summary_writer.add_summary(sum_mu, count)
sum_cov = summary.scalar('G_like_cov1', like_cov1.data[0])
self.summary_writer.add_summary(sum_cov, count)
kl_loss = KL_loss(mu, logvar) * cfg.TRAIN.COEFF.KL
errG_total = errG_total + kl_loss
errG_total.backward()
self.optimizerG.step()
return kl_loss, errG_total
def update_custom_scalar_plots(self, new_vals, l2n_val, epoch):
sess = K.get_session()
# This construct lets us only define placeholders once
if l2n_val in self.placeholder_tensors:
# Grab the placeholder tensors used previously
placeholder_list = self.placeholder_tensors[l2n_val]
else:
# Create the tensor placeholders (for my custom scalar plots...)
placeholder_list = [
tf.placeholder(tf.int32, shape=[]),
tf.placeholder(tf.int32, shape=[]),
tf.placeholder(tf.float32, shape=[]),
tf.placeholder(tf.float32, shape=[]),
tf.placeholder(tf.float32, shape=[])
]
self.placeholder_tensors[l2n_val] = placeholder_list
# This construct lets us only define summaries once
if l2n_val in self.custom_scalar_summaries:
# Grab the placeholder tensors used previously
summaries_list = self.custom_scalar_summaries[l2n_val]
else:
# Create the plot summaries
summaries_list = [
summary_lib.scalar(
"validation_false_positives_maxf1_l2n_" + str(l2n_val),
placeholder_list[0]),
summary_lib.scalar(
"validation_false_negatives_maxf1_l2n_" + str(l2n_val),
placeholder_list[1]),
summary_lib.scalar(
"validation_precision_maxf1_l2n_" + str(l2n_val),
placeholder_list[2]),
summary_lib.scalar(
"validation_recall_maxf1_l2n_" + str(l2n_val),
placeholder_list[3]),
summary_lib.scalar(
"validation_maxf1_l2n_" + str(l2n_val),
placeholder_list[4])
]
self.custom_scalar_summaries[l2n_val] = summaries_list
for summary, placeholder, val in zip(summaries_list,
placeholder_list,
new_vals):
# Execute the summary with the current actual values
run_summary = sess.run(summary, feed_dict={
placeholder: val
})
# Push the new values to the events file
self.get_writer().add_summary(run_summary, epoch)
def write_loss(iterations, max_iterations, trainer, train_writer):
print("Iteration: %08d/%08d" % (iterations + 1, max_iterations))
members = [attr for attr in dir(trainer) \
if not callable(getattr(trainer, attr)) and not attr.startswith("__") and 'loss' in attr]
for m in members:
train_writer.add_summary(summary.scalar(m, getattr(trainer, m)),
iterations + 1)
members = [attr for attr in dir(trainer) \
if not callable(getattr(trainer, attr)) and not attr.startswith("__") and 'acc' in attr]
for m in members:
train_writer.add_summary(summary.scalar(m, getattr(trainer, m)),
iterations + 1)
def train_Gnet(self, count):
self.netG.zero_grad()
errG_total = 0
flag = count % 100
batch_size = self.real_imgs[0].size(0)
criterion, mu, logvar = self.criterion, self.mu, self.logvar
real_labels = self.real_labels[:batch_size]
for i in range(self.num_Ds):
outputs = self.netsD[i](self.fake_imgs[i], mu)
errG = criterion(outputs[0], real_labels)
if len(outputs) > 1 and cfg.TRAIN.COEFF.UNCOND_LOSS > 0:
errG_patch = cfg.TRAIN.COEFF.UNCOND_LOSS *\
criterion(outputs[1], real_labels)
errG = errG + errG_patch
errG_total = errG_total + errG
if flag == 0:
summary_D = summary.scalar('G_loss%d' % i, errG.data[0])
self.summary_writer.add_summary(summary_D, count)
# Compute color consistency losses
if cfg.TRAIN.COEFF.COLOR_LOSS > 0:
if self.num_Ds > 1:
mu1, covariance1 = compute_mean_covariance(self.fake_imgs[-1])
mu2, covariance2 = \
compute_mean_covariance(self.fake_imgs[-2].detach())
like_mu2 = cfg.TRAIN.COEFF.COLOR_LOSS * nn.MSELoss()(mu1, mu2)
like_cov2 = cfg.TRAIN.COEFF.COLOR_LOSS * 5 * \
nn.MSELoss()(covariance1, covariance2)
errG_total = errG_total + like_mu2 + like_cov2
if flag == 0:
sum_mu = summary.scalar('G_like_mu2', like_mu2.data[0])
self.summary_writer.add_summary(sum_mu, count)
sum_cov = summary.scalar('G_like_cov2', like_cov2.data[0])
self.summary_writer.add_summary(sum_cov, count)
if self.num_Ds > 2:
mu1, covariance1 = compute_mean_covariance(self.fake_imgs[-2])
mu2, covariance2 = \
compute_mean_covariance(self.fake_imgs[-3].detach())
like_mu1 = cfg.TRAIN.COEFF.COLOR_LOSS * nn.MSELoss()(mu1, mu2)
like_cov1 = cfg.TRAIN.COEFF.COLOR_LOSS * 5 * \
nn.MSELoss()(covariance1, covariance2)
errG_total = errG_total + like_mu1 + like_cov1
if flag == 0:
sum_mu = summary.scalar('G_like_mu1', like_mu1.data[0])
self.summary_writer.add_summary(sum_mu, count)
sum_cov = summary.scalar('G_like_cov1', like_cov1.data[0])
self.summary_writer.add_summary(sum_cov, count)
kl_loss = KL_loss(mu, logvar) * cfg.TRAIN.COEFF.KL
errG_total = errG_total + kl_loss
errG_total.backward()
self.optimizerG.step()
return kl_loss, errG_total
def train_Gnet(self, count):
self.netG.zero_grad()
errG_total = 0
flag = count % 100
batch_size = self.real_imgs[0].size(0)
criterion = self.criterion
real_labels = self.real_labels[:batch_size]
for i in range(self.num_Ds):
netD = self.netsD[i]
outputs = netD(self.fake_imgs[i])
errG = criterion(outputs[0], real_labels)
# errG = self.stage_coeff[i] * errG
errG_total = errG_total + errG
if flag == 0:
summary_G = summary.scalar('G_loss%d' % i, errG.data[0])
self.summary_writer.add_summary(summary_G, count)
# Compute color preserve losses
if cfg.TRAIN.COEFF.COLOR_LOSS > 0:
if self.num_Ds > 1:
mu1, covariance1 = compute_mean_covariance(self.fake_imgs[-1])
mu2, covariance2 = \
compute_mean_covariance(self.fake_imgs[-2].detach())
like_mu2 = cfg.TRAIN.COEFF.COLOR_LOSS * nn.MSELoss()(mu1, mu2)
like_cov2 = cfg.TRAIN.COEFF.COLOR_LOSS * 5 * \
nn.MSELoss()(covariance1, covariance2)
errG_total = errG_total + like_mu2 + like_cov2
if self.num_Ds > 2:
mu1, covariance1 = compute_mean_covariance(self.fake_imgs[-2])
mu2, covariance2 = \
compute_mean_covariance(self.fake_imgs[-3].detach())
like_mu1 = cfg.TRAIN.COEFF.COLOR_LOSS * nn.MSELoss()(mu1, mu2)
like_cov1 = cfg.TRAIN.COEFF.COLOR_LOSS * 5 * \
nn.MSELoss()(covariance1, covariance2)
errG_total = errG_total + like_mu1 + like_cov1
if flag == 0:
sum_mu = summary.scalar('G_like_mu2', like_mu2.data[0])
self.summary_writer.add_summary(sum_mu, count)
sum_cov = summary.scalar('G_like_cov2', like_cov2.data[0])
self.summary_writer.add_summary(sum_cov, count)
if self.num_Ds > 2:
sum_mu = summary.scalar('G_like_mu1', like_mu1.data[0])
self.summary_writer.add_summary(sum_mu, count)
sum_cov = summary.scalar('G_like_cov1', like_cov1.data[0])
self.summary_writer.add_summary(sum_cov, count)
errG_total.backward()
self.optimizerG.step()
return errG_total
def train_Gnet(self, count):
self.netG.zero_grad()
errG_total = 0
flag = count % 100
batch_size = self.real_imgs[0].size(0)
criterion = self.criterion
real_labels = self.real_labels[:batch_size]
for i in range(self.num_Ds):
netD = self.netsD[i]
outputs = netD(self.fake_imgs[i])
errG = criterion(outputs[0], real_labels)
# errG = self.stage_coeff[i] * errG
errG_total = errG_total + errG
if flag == 0:
summary_G = summary.scalar('G_loss%d' % i, errG.data[0])
self.summary_writer.add_summary(summary_G, count)
# Compute color preserve losses
if cfg.TRAIN.COEFF.COLOR_LOSS > 0:
if self.num_Ds > 1:
mu1, covariance1 = compute_mean_covariance(self.fake_imgs[-1])
mu2, covariance2 = \
compute_mean_covariance(self.fake_imgs[-2].detach())
like_mu2 = cfg.TRAIN.COEFF.COLOR_LOSS * nn.MSELoss()(mu1, mu2)
like_cov2 = cfg.TRAIN.COEFF.COLOR_LOSS * 5 * \
nn.MSELoss()(covariance1, covariance2)
errG_total = errG_total + like_mu2 + like_cov2
if self.num_Ds > 2:
mu1, covariance1 = compute_mean_covariance(self.fake_imgs[-2])
mu2, covariance2 = \
compute_mean_covariance(self.fake_imgs[-3].detach())
like_mu1 = cfg.TRAIN.COEFF.COLOR_LOSS * nn.MSELoss()(mu1, mu2)
like_cov1 = cfg.TRAIN.COEFF.COLOR_LOSS * 5 * \
nn.MSELoss()(covariance1, covariance2)
errG_total = errG_total + like_mu1 + like_cov1
if flag == 0:
sum_mu = summary.scalar('G_like_mu2', like_mu2.data[0])
self.summary_writer.add_summary(sum_mu, count)
sum_cov = summary.scalar('G_like_cov2', like_cov2.data[0])
self.summary_writer.add_summary(sum_cov, count)
if self.num_Ds > 2:
sum_mu = summary.scalar('G_like_mu1', like_mu1.data[0])
self.summary_writer.add_summary(sum_mu, count)
sum_cov = summary.scalar('G_like_cov1', like_cov1.data[0])
self.summary_writer.add_summary(sum_cov, count)
errG_total.backward()
self.optimizerG.step()
return errG_total
def train_Dnet(self, idx, count):
flag = count % 100
batch_size = self.real_imgs[0].size(0)
criterion = self.criterion
netD, optD = self.netsD[idx], self.optimizersD[idx]
real_imgs = self.real_imgs[idx]
fake_imgs = self.fake_imgs[idx]
real_labels = self.real_labels[:batch_size]
fake_labels = self.fake_labels[:batch_size]
#
netD.zero_grad()
#
real_logits = netD(real_imgs)
fake_logits = netD(fake_imgs.detach())
#
errD_real = criterion(real_logits[0], real_labels)
errD_fake = criterion(fake_logits[0], fake_labels)
#
errD = errD_real + errD_fake
errD.backward()
# update parameters
optD.step()
# log
if flag == 0:
summary_D = summary.scalar('D_loss%d' % idx, errD.data[0])
self.summary_writer.add_summary(summary_D, count)
return errD
def train_Dnet(self, idx, count):
flag = count % 100
batch_size = self.real_imgs[0].size(0)
criterion = self.criterion
netD, optD = self.netsD[idx], self.optimizersD[idx]
real_imgs = self.real_imgs[idx]
fake_imgs = self.fake_imgs[idx]
real_labels = self.real_labels[:batch_size]
fake_labels = self.fake_labels[:batch_size]
#
netD.zero_grad()
#
real_logits = netD(real_imgs)
fake_logits = netD(fake_imgs.detach())
#
errD_real = criterion(real_logits[0], real_labels)
errD_fake = criterion(fake_logits[0], fake_labels)
#
errD = errD_real + errD_fake
errD.backward()
# update parameters
optD.step()
# log
if flag == 0:
summary_D = summary.scalar('D_loss%d' % idx, errD.data[0])
self.summary_writer.add_summary(summary_D, count)
return errD
def loss_with_metrics(img_ab_out, img_ab_true, name=''):
# Loss is mean square error
cost = tf.reduce_mean(
tf.squared_difference(img_ab_out, img_ab_true), name="mse")
# Metrics for tensorboard
summary = summary_lib.scalar(name, cost)
return cost, summary
def test_log_scalar_summary():
logdir = './experiment/scalar'
writer = FileWriter(logdir)
for i in range(10):
s = scalar('scalar', i)
writer.add_summary(s, i + 1)
writer.flush()
writer.close()
def train_Dnet(self, idx, count):
flag = count % 100
batch_size = self.real_imgs[0].size(0)
criterion, mu = self.criterion, self.mu
netD, optD = self.netsD[idx], self.optimizersD[idx]
real_imgs = self.real_imgs[idx]
wrong_imgs = self.wrong_imgs[idx]
fake_imgs = self.fake_imgs[idx]
#
netD.zero_grad()
# Forward
real_labels = self.real_labels[:batch_size]
fake_labels = self.fake_labels[:batch_size]
real_map = self.real_maps[idx]
fake_map = self.fake_maps[idx]
# for real
real_logits = netD(real_imgs, mu.detach())
wrong_logits = netD(wrong_imgs, mu.detach())
fake_logits = netD(fake_imgs.detach(), mu.detach())
#
errD_real = criterion(real_logits[0], real_labels)
errD_wrong = criterion(wrong_logits[0], fake_labels)
errD_fake = criterion(fake_logits[0], fake_labels)
#error terms from probability maps
maploss_real = criterion(real_logits[1], real_map)
maploss_fake = criterion(wrong_logits[1], fake_map)
if len(real_logits) > 1 and cfg.TRAIN.COEFF.UNCOND_LOSS > 0:
errD_real_uncond = cfg.TRAIN.COEFF.UNCOND_LOSS * \
criterion(real_logits[2], real_labels)
errD_wrong_uncond = cfg.TRAIN.COEFF.UNCOND_LOSS * \
criterion(wrong_logits[2], real_labels)
errD_fake_uncond = cfg.TRAIN.COEFF.UNCOND_LOSS * \
criterion(fake_logits[2], fake_labels)
#
errD_real = errD_real + errD_real_uncond
errD_wrong = errD_wrong + errD_wrong_uncond
errD_fake = errD_fake + errD_fake_uncond
#
errD = errD_real + errD_wrong + errD_fake + maploss_real + maploss_fake
else:
#errD = errD_real + 0.5 * (errD_wrong + errD_fake)
errD = errD_real + errD_wrong + errD_fake + maploss_real + maploss_fake
# backward
errD.backward()
# update parameters
optD.step()
# log
if flag == 0:
summary_D = summary.scalar('D_loss%d' % idx, errD.data[0])
self.summary_writer.add_summary(summary_D, count)
return errD
def test_event_logging():
logdir = './experiment/'
summary_writer = FileWriter(logdir)
scalar_value = 1.0
s = scalar('test_scalar', scalar_value)
summary_writer.add_summary(s, global_step=1)
summary_writer.close()
assert os.path.isdir(logdir)
assert len(os.listdir(logdir)) == 1
summary_writer = FileWriter(logdir)
scalar_value = 1.0
s = scalar('test_scalar', scalar_value)
summary_writer.add_summary(s, global_step=1)
summary_writer.close()
assert os.path.isdir(logdir)
assert len(os.listdir(logdir)) == 2
# clean up.
shutil.rmtree(logdir)
def train_Dnet(self, count):
flag = count % 100
batch_size = self.real_imgs[0].size(0)
criterion = self.citerion
netD = self.netD
optD = self.optimizerD
real_imgs = self.real_imgs[0]
wrong_imgs = self.wrong_imgs[0]
fake_imgs = self.fake_imgs[-1] # Take only the last image
netD.zero_grad()
real_labels = self.real_labels[:batch_size]
fake_labels = self.fake_labels[:batch_size]
# Calculating the logits
mu = self.mus[-1]
real_logits = netD(real_imgs, mu.detach())
wrong_logits = netD(wrong_imgs, mu.detach())
fake_logits = netD(fake_imgs.detach(), mu.detach())
# Calculating the error
errD_real = criterion(real_logits[0], real_labels)
errD_wrong = criterion(wrong_logits[0], fake_labels)
errD_fake = criterion(fake_logits[0], fake_labels)
if len(real_logits) > 1 and cfg.TRAIN.COEFF.UNCOND_LOSS > 0:
errD_real_uncond = cfg.TRAIN.COEFF.UNCOND_LOSS * criterion(
real_logits[1], real_labels)
errD_wrong_uncond = cfg.TRAIN.COEFF.UNCOND_LOSS * criterion(
wrong_logits[1], fake_labels)
errD_fake_uncond = cfg.TRAIN.COEFF.UNCOND_LOSS * criterion(
fake_logits[1], fake_labels)
errD_real += errD_real_uncond
errD_wrong += errD_wrong_uncond
errD_fake += errD_fake_uncond
errD = errD_real + errD_wrong + errD_fake
else:
errD = errD_real + 0.5 * (errD_wrong + errD_fake)
# Calculating the gradients
errD.backward()
# Backproping
optD.step()
if flag == 0:
summary_D = summary.scalar('D_loss%d', errD.data[0])
self.summary_writer.add_summary(summary_D, count)
return errD
def write_tensorboard(summary_writer, metrics: Dict[str, float],
checkpoint: int):
"""
Writes a Tensorboard scalar event to the given SummaryWriter.
:param summary_writer: A Tensorboard SummaryWriter instance.
:param metrics: Mapping of metric names to their values.
:param checkpoint: Current checkpoint.
"""
from tensorboard.summary import scalar # pylint: disable=import-error
for name, value in metrics.items():
summary_writer.add_summary(scalar(name=name, scalar=value),
global_step=checkpoint)
def add_summary(self, labels):
summaries = []
is_pay = labels['is_pay']
pay_price = labels['pay_price']
p = tf.boolean_mask(pay_price, is_pay)
p_hat = tf.boolean_mask(self.output_dict['pred_prices'], is_pay)
all_rmsle = _rmsle(p, p_hat, name='rmsle')
with tf.name_scope('train' if self.mode ==
tf.estimator.ModeKeys.TRAIN else 'eval'):
summaries.append(summary_lib.scalar('loss', self.loss))
summaries.append(summary_lib.scalar('rmsle', all_rmsle))
# ranges = [(0, 20), (20, 50), (50, 100), (100, 200), (200, 500), (500, 1000), (1000, 2000), (2000, 5000),
# (5000, 999999)]
# for min_v, max_v in ranges:
# name = '{}_{}_rmsle'.format(min_v, max_v)
# tmp_rmsle = _rmsle(p, p_hat, min_v=min_v, max_v=max_v, name=name)
# summaries.append(summary_lib.scalar(name, tmp_rmsle))
self.print_dict['loss'] = self.loss
self.print_dict['rmsle'] = all_rmsle
self.summary_op = tf.summary.merge(summaries)
def test_scalar_summary():
scalar_value = 1.0
s = scalar('test_scalar', scalar_value)
values = s.value
assert len(values) == 1
assert values[0].tag == 'test_scalar'
assert values[0].simple_value == 1.0
byte_str = s.SerializeToString()
s_recovered = summary_pb2.Summary()
s_recovered.ParseFromString(byte_str)
assert values[0].tag == s_recovered.value[0].tag
assert values[0].simple_value == s_recovered.value[0].simple_value
def train(self):
start_idx = 0
net = self.net
criterion = nn.CrossEntropyLoss()
lr = cfg.TRAIN.LEARNING_RATE
optimizer = optim.Adam(net.parameters(), lr=lr, betas=(0, 0.99))
count = 0
start_idx = 0
iteration = len(self.seqs) // self.batch_size + 1
for epoch in range(self.epoch):
for i in tqdm(range(iteration)):
x, lengths = prepare_batch(self.seqs, start_idx,
self.batch_size)
if self.cuda:
x = Variable(torch.LongTensor(x)).cuda()
else:
x = Variable(torch.LongTensor(x))
logits = net(x)
loss = []
for i in range(logits.size(0)):
logit = logits[i][:lengths[i] - 1]
target = x[i][1:lengths[i]]
loss.append(criterion(logit, target))
loss = sum(loss) / len(loss)
net.zero_grad()
loss.backward()
optimizer.step()
end = time.time()
summary_net = summary.scalar("Loss", loss.data[0])
self.summary_writer.add_summary(summary_net, count)
count += 1
print("epoch %d done. train loss: %f" % (epoch + 1, loss.data[0]))
if count % cfg.TRAIN.LR_DECAY_INTERVAL == 0:
lr = lr * 0.95
optimizer = optim.Adam(net.parameters(),
lr=lr,
betas=(0.9, 0.999))
print("decayed learning rate")
save_model(net, self.model_dir)
f = open("output/Data/data.pkl", "wb")
pickle.dump(self.data, f)
f.close
def train(w, x, loops):
for i in range(loops):
dw, loss = grad_function(w, x)
# gradient descent
w -= 0.1 * dw
if i % 10 == 0:
print('Iter {}, training loss {}'.format(i, loss))
# summary1 = scalar('loss', loss)
# train_writer.add_summary(summary1, i)
# print(loss)
for ele in loss:
summary1 = scalar('loss', ele)
train_writer.add_summary(summary1, i)
train_writer.flush()
train_writer.close()
def train_Dnet(self, idx, count):
flag = count % 100
batch_size = self.real_imgs[0].size(0)
criterion, mu = self.criterion, self.mu
netD, optD = self.netsD[idx], self.optimizersD[idx]
real_imgs = self.real_imgs[idx]
wrong_imgs = self.wrong_imgs[idx]
fake_imgs = self.fake_imgs[idx]
#
netD.zero_grad()
# Forward
real_labels = self.real_labels[:batch_size]
fake_labels = self.fake_labels[:batch_size]
# for real
real_logits = netD(real_imgs, mu.detach())
wrong_logits = netD(wrong_imgs, mu.detach())
fake_logits = netD(fake_imgs.detach(), mu.detach())
#
errD_real = criterion(real_logits[0], real_labels)
errD_wrong = criterion(wrong_logits[0], fake_labels)
errD_fake = criterion(fake_logits[0], fake_labels)
if len(real_logits) > 1 and cfg.TRAIN.COEFF.UNCOND_LOSS > 0:
errD_real_uncond = cfg.TRAIN.COEFF.UNCOND_LOSS * \
criterion(real_logits[1], real_labels)
errD_wrong_uncond = cfg.TRAIN.COEFF.UNCOND_LOSS * \
criterion(wrong_logits[1], real_labels)
errD_fake_uncond = cfg.TRAIN.COEFF.UNCOND_LOSS * \
criterion(fake_logits[1], fake_labels)
#
errD_real = errD_real + errD_real_uncond
errD_wrong = errD_wrong + errD_wrong_uncond
errD_fake = errD_fake + errD_fake_uncond
#
errD = errD_real + errD_wrong + errD_fake
else:
errD = errD_real + 0.5 * (errD_wrong + errD_fake)
# backward
errD.backward()
# update parameters
optD.step()
# log
if flag == 0:
summary_D = summary.scalar('D_loss%d' % idx, errD.data[0])
self.summary_writer.add_summary(summary_D, count)
return errD
def write_tensorboard(summary_writer,
metrics: Dict[str, Union[float, int]],
checkpoint: int):
"""
Writes a Tensorboard scalar event to the given SummaryWriter.
:param summary_writer: A Tensorboard SummaryWriter instance.
:param metrics: Mapping of metric names to their values.
:param checkpoint: Current checkpoint.
"""
try:
from tensorboard.summary import scalar # pylint: disable=import-error
except ImportError:
raise RuntimeError("Please install tensorboard.")
for name, value in metrics.items():
summary_writer.add_summary(
scalar(
name=name, scalar=value), global_step=checkpoint)
def train_Gnet(self, count):
self.netG.zero_grad()
errG_total = 0
flag = count % 100
batch_size = self.real_imgs[0].size(0)
criterion = self.citerion
mus, logvars = self.mus, self.logvars
real_labels = self.real_labels[:batch_size]
# Looping through each time-step.
for i in range(len(self.fake_imgs)):
logits = self.netD(self.fake_imgs[i], mus[i])
errG = criterion(logits[0], real_labels)
if len(logits) > 1 and cfg.TRAIN.COEFF.UNCOND_LOSS > 0:
errG_uncond = cfg.TRAIN.COEFF.UNCOND_LOSS * criterion(
logits[1], real_labels)
errG += errG_uncond
errG_total += errG
if flag == 0:
summary_D = summary.scalar('G_loss%d' % i, errG.data[0])
self.summary_writer.add_summary(summary_D, count)
kl_loss = 0
for i in range(len(self.fake_imgs)):
kl_loss += KL_loss(mus[i], logvars[i]) * cfg.TRAIN.COEFF.KL
errG_total += kl_loss
# Compute the gradients
errG_total.backward()
# BPTT
self.optimizerG.step()
return kl_loss, errG_total
def add_scalar_summary(summary_writer, name, value, step):
value = unwrap_scalar_variable(value)
summ = summary.scalar(name=name, scalar=value)
summary_writer.add_summary(summary=summ, global_step=step)
def train(self):
self.netG, self.netsD, self.num_Ds,\
self.inception_model, start_count = load_network(self.gpus)
avg_param_G = copy_G_params(self.netG)
self.optimizerG, self.optimizersD = \
define_optimizers(self.netG, self.netsD)
self.criterion = nn.BCELoss()
self.real_labels = \
Variable(torch.FloatTensor(self.batch_size).fill_(1))
self.fake_labels = \
Variable(torch.FloatTensor(self.batch_size).fill_(0))
self.gradient_one = torch.FloatTensor([1.0])
self.gradient_half = torch.FloatTensor([0.5])
nz = cfg.GAN.Z_DIM
noise = Variable(torch.FloatTensor(self.batch_size, nz))
fixed_noise = \
Variable(torch.FloatTensor(self.batch_size, nz).normal_(0, 1))
if cfg.CUDA:
self.criterion.cuda()
self.real_labels = self.real_labels.cuda()
self.fake_labels = self.fake_labels.cuda()
self.gradient_one = self.gradient_one.cuda()
self.gradient_half = self.gradient_half.cuda()
noise, fixed_noise = noise.cuda(), fixed_noise.cuda()
predictions = []
count = start_count
start_epoch = start_count // (self.num_batches)
for epoch in range(start_epoch, self.max_epoch):
start_t = time.time()
for step, data in enumerate(self.data_loader, 0):
#######################################################
# (0) Prepare training data
######################################################
self.imgs_tcpu, self.real_imgs, self.wrong_imgs, \
self.txt_embedding = self.prepare_data(data)
#######################################################
# (1) Generate fake images
######################################################
noise.data.normal_(0, 1)
self.fake_imgs, self.mu, self.logvar = \
self.netG(noise, self.txt_embedding)
#######################################################
# (2) Update D network
######################################################
errD_total = 0
for i in range(self.num_Ds):
errD = self.train_Dnet(i, count)
errD_total += errD
#######################################################
# (3) Update G network: maximize log(D(G(z)))
######################################################
kl_loss, errG_total = self.train_Gnet(count)
for p, avg_p in zip(self.netG.parameters(), avg_param_G):
avg_p.mul_(0.999).add_(0.001, p.data)
# for inception score
pred = self.inception_model(self.fake_imgs[-1].detach())
predictions.append(pred.data.cpu().numpy())
if count % 100 == 0:
summary_D = summary.scalar('D_loss', errD_total.data[0])
summary_G = summary.scalar('G_loss', errG_total.data[0])
summary_KL = summary.scalar('KL_loss', kl_loss.data[0])
self.summary_writer.add_summary(summary_D, count)
self.summary_writer.add_summary(summary_G, count)
self.summary_writer.add_summary(summary_KL, count)
count = count + 1
if count % cfg.TRAIN.SNAPSHOT_INTERVAL == 0:
save_model(self.netG, avg_param_G, self.netsD, count,
self.model_dir)
# Save images
backup_para = copy_G_params(self.netG)
load_params(self.netG, avg_param_G)
#
self.fake_imgs, _, _ = \
self.netG(fixed_noise, self.txt_embedding)
save_img_results(self.imgs_tcpu, self.fake_imgs,
self.num_Ds, count, self.image_dir,
self.summary_writer)
#
load_params(self.netG, backup_para)
# Compute inception score
if len(predictions) > 500:
predictions = np.concatenate(predictions, 0)
mean, std = compute_inception_score(predictions, 10)
# print('mean:', mean, 'std', std)
m_incep = summary.scalar('Inception_mean', mean)
self.summary_writer.add_summary(m_incep, count)
#
mean_nlpp, std_nlpp = \
negative_log_posterior_probability(predictions, 10)
m_nlpp = summary.scalar('NLPP_mean', mean_nlpp)
self.summary_writer.add_summary(m_nlpp, count)
#
predictions = []
end_t = time.time()
print('''[%d/%d][%d]
Loss_D: %.2f Loss_G: %.2f Loss_KL: %.2f Time: %.2fs
'''
# D(real): %.4f D(wrong):%.4f D(fake) %.4f
%
(epoch, self.max_epoch, self.num_batches, errD_total.data[0],
errG_total.data[0], kl_loss.data[0], end_t - start_t))
save_model(self.netG, avg_param_G, self.netsD, count, self.model_dir)
self.summary_writer.close()
def train(self, data_loader, stage=1):
if stage == 1:
netG, netD = self.load_network_stageI()
else:
netG, netD = self.load_network_stageII()
nz = cfg.Z_DIM
batch_size = self.batch_size
noise = Variable(torch.FloatTensor(batch_size, nz))
fixed_noise = \
Variable(torch.FloatTensor(batch_size, nz).normal_(0, 1),
volatile=True)
real_labels = Variable(torch.FloatTensor(batch_size).fill_(1))
fake_labels = Variable(torch.FloatTensor(batch_size).fill_(0))
if cfg.CUDA:
noise, fixed_noise = noise.cuda(), fixed_noise.cuda()
real_labels, fake_labels = real_labels.cuda(), fake_labels.cuda()
generator_lr = cfg.TRAIN.GENERATOR_LR
discriminator_lr = cfg.TRAIN.DISCRIMINATOR_LR
lr_decay_step = cfg.TRAIN.LR_DECAY_EPOCH
optimizerD = \
optim.Adam(netD.parameters(),
lr=cfg.TRAIN.DISCRIMINATOR_LR, betas=(0.5, 0.999))
netG_para = []
for p in netG.parameters():
if p.requires_grad:
netG_para.append(p)
optimizerG = optim.Adam(netG_para,
lr=cfg.TRAIN.GENERATOR_LR,
betas=(0.5, 0.999))
count = 0
for epoch in range(self.max_epoch):
start_t = time.time()
if epoch % lr_decay_step == 0 and epoch > 0:
generator_lr *= 0.5
for param_group in optimizerG.param_groups:
param_group['lr'] = generator_lr
discriminator_lr *= 0.5
for param_group in optimizerD.param_groups:
param_group['lr'] = discriminator_lr
for i, data in enumerate(data_loader, 0):
######################################################
# (1) Prepare training data
######################################################
real_img_cpu, txt_embedding = data
real_imgs = Variable(real_img_cpu)
txt_embedding = Variable(txt_embedding)
if cfg.CUDA:
real_imgs = real_imgs.cuda()
txt_embedding = txt_embedding.cuda()
#######################################################
# (2) Generate fake images
######################################################
noise.data.normal_(0, 1)
inputs = (txt_embedding, noise)
_, fake_imgs, mu, logvar = \
nn.parallel.data_parallel(netG, inputs, self.gpus)
############################
# (3) Update D network
###########################
netD.zero_grad()
errD, errD_real, errD_wrong, errD_fake = \
compute_discriminator_loss(netD, real_imgs, fake_imgs,
real_labels, fake_labels,
mu, self.gpus)
errD.backward()
optimizerD.step()
############################
# (2) Update G network
###########################
netG.zero_grad()
errG = compute_generator_loss(netD, fake_imgs,
real_labels, mu, self.gpus)
kl_loss = KL_loss(mu, logvar)
errG_total = errG + kl_loss * cfg.TRAIN.COEFF.KL
errG_total.backward()
optimizerG.step()
count = count + 1
if i % 100 == 0:
summary_D = summary.scalar('D_loss', errD.data[0])
summary_D_r = summary.scalar('D_loss_real', errD_real)
summary_D_w = summary.scalar('D_loss_wrong', errD_wrong)
summary_D_f = summary.scalar('D_loss_fake', errD_fake)
summary_G = summary.scalar('G_loss', errG.data[0])
summary_KL = summary.scalar('KL_loss', kl_loss.data[0])
self.summary_writer.add_summary(summary_D, count)
self.summary_writer.add_summary(summary_D_r, count)
self.summary_writer.add_summary(summary_D_w, count)
self.summary_writer.add_summary(summary_D_f, count)
self.summary_writer.add_summary(summary_G, count)
self.summary_writer.add_summary(summary_KL, count)
# save the image result for each epoch
inputs = (txt_embedding, fixed_noise)
lr_fake, fake, _, _ = \
nn.parallel.data_parallel(netG, inputs, self.gpus)
save_img_results(real_img_cpu, fake, epoch, self.image_dir)
if lr_fake is not None:
save_img_results(None, lr_fake, epoch, self.image_dir)
end_t = time.time()
print('''[%d/%d][%d/%d] Loss_D: %.4f Loss_G: %.4f Loss_KL: %.4f
Loss_real: %.4f Loss_wrong:%.4f Loss_fake %.4f
Total Time: %.2fsec
'''
% (epoch, self.max_epoch, i, len(data_loader),
errD.data[0], errG.data[0], kl_loss.data[0],
errD_real, errD_wrong, errD_fake, (end_t - start_t)))
if epoch % self.snapshot_interval == 0:
save_model(netG, netD, epoch, self.model_dir)
#
save_model(netG, netD, self.max_epoch, self.model_dir)
#
self.summary_writer.close()
def train(iter_cnt, model, corpus, args, optimizer):
train_writer = FileWriter(args.run_path + "/train", flush_secs=5)
pos_file_path = "{}.pos.txt".format(args.train)
neg_file_path = "{}.neg.txt".format(args.train)
pos_batch_loader = FileLoader(
[tuple([pos_file_path, os.path.dirname(args.train)])], args.batch_size)
neg_batch_loader = FileLoader(
[tuple([neg_file_path, os.path.dirname(args.train)])], args.batch_size)
#neg_batch_loader = RandomLoader(
# corpus = corpus,
# exclusive_set = zip(pos_batch_loader.data_left, pos_batch_loader.data_right),
# batch_size = args.batch_size
#)
#neg_batch_loader = CombinedLoader(
# neg_batch_loader_1,
# neg_batch_loader_2,
# args.batch_size
#)
use_content = False
if args.use_content:
use_content = True
embedding_layer = model.embedding_layer
criterion = model.compute_loss
start = time.time()
tot_loss = 0.0
tot_cnt = 0
for batch, labels in tqdm(
pad_iter(corpus,
embedding_layer,
pos_batch_loader,
neg_batch_loader,
use_content,
pad_left=False)):
iter_cnt += 1
model.zero_grad()
labels = labels.type(torch.LongTensor)
new_batch = []
if args.use_content:
for x in batch:
for y in x:
new_batch.append(y)
batch = new_batch
if args.cuda:
batch = [x.cuda() for x in batch]
labels = labels.cuda()
batch = map(Variable, batch)
labels = Variable(labels)
repr_left = None
repr_right = None
if not use_content:
repr_left = model(batch[0])
repr_right = model(batch[1])
else:
repr_left = model(batch[0]) + model(batch[1])
repr_right = model(batch[2]) + model(batch[3])
output = model.compute_similarity(repr_left, repr_right)
loss = criterion(output, labels)
loss.backward()
prev_emb = embedding_layer.embedding.weight.cpu().data.numpy()
optimizer.step()
current_emb = embedding_layer.embedding.weight.cpu().data.numpy()
diff = np.sum(np.absolute(current_emb - prev_emb))
tot_loss += loss.data[0] * output.size(0)
tot_cnt += output.size(0)
if iter_cnt % 100 == 0:
outputManager.say("\r" + " " * 50)
outputManager.say("\r{} loss: {:.4f} eps: {:.0f} ".format(
iter_cnt, tot_loss / tot_cnt, tot_cnt / (time.time() - start)))
s = summary.scalar('loss', tot_loss / tot_cnt)
train_writer.add_summary(s, iter_cnt)
outputManager.say("\n")
train_writer.close()
#if model.criterion.startswith('classification'):
# print model.output_op.weight.min().data[0], model.output_op.weight.max().data[0]
return iter_cnt
def run():
"""Run custom scalar demo and generate event files."""
step = tf.placeholder(tf.float32, shape=[])
with tf.name_scope('loss'):
# Specify 2 different loss values, each tagged differently.
summary_lib.scalar('foo', tf.pow(0.9, step))
summary_lib.scalar('bar', tf.pow(0.85, step + 2))
# Log metric baz as well as upper and lower bounds for a margin chart.
middle_baz_value = step + 4 * tf.random_uniform([]) - 2
summary_lib.scalar('baz', middle_baz_value)
summary_lib.scalar('baz_lower',
middle_baz_value - 6.42 - tf.random_uniform([]))
summary_lib.scalar('baz_upper',
middle_baz_value + 6.42 + tf.random_uniform([]))
with tf.name_scope('trigFunctions'):
summary_lib.scalar('cosine', tf.cos(step))
summary_lib.scalar('sine', tf.sin(step))
summary_lib.scalar('tangent', tf.tan(step))
merged_summary = tf.summary.merge_all()
with tf.Session() as sess, tf.summary.FileWriter(LOGDIR) as writer:
# We only need to specify the layout once (instead of per step).
layout_summary = summary_lib.custom_scalar_pb(
layout_pb2.Layout(category=[
layout_pb2.Category(
title='losses',
chart=[
layout_pb2.Chart(
title='losses',
multiline=layout_pb2.MultilineChartContent(
tag=[r'loss(?!.*margin.*)'], )),
layout_pb2.Chart(
title='baz',
margin=layout_pb2.MarginChartContent(series=[
layout_pb2.MarginChartContent.Series(
value='loss/baz/scalar_summary',
lower='loss/baz_lower/scalar_summary',
upper='loss/baz_upper/scalar_summary'),
], )),
]),
layout_pb2.Category(
title='trig functions',
chart=[
layout_pb2.Chart(
title='wave trig functions',
multiline=layout_pb2.MultilineChartContent(tag=[
r'trigFunctions/cosine', r'trigFunctions/sine'
], )),
# The range of tangent is different. Give it its own chart.
layout_pb2.Chart(
title='tan',
multiline=layout_pb2.MultilineChartContent(
tag=[r'trigFunctions/tangent'], )),
],
# This category we care less about. Make it initially closed.
closed=True),
]))
writer.add_summary(layout_summary)
for i in xrange(42):
summary = sess.run(merged_summary, feed_dict={step: i})
writer.add_summary(summary, global_step=i)
def main(argv):
(opts, args) = parser.parse_args(argv)
# Load experiment setting
assert isinstance(opts, object)
config = NetConfig(opts.config)
batch_size = config.hyperparameters['batch_size']
max_iterations = config.hyperparameters['max_iterations']
trainer = []
exec("trainer=%s(config.hyperparameters)" %
config.hyperparameters['trainer'])
trainer.cuda(opts.gpu)
iterations = 0
train_writer = tensorboard.FileWriter(
"%s/%s" %
(opts.log, os.path.splitext(os.path.basename(opts.config))[0]))
snapshot_directory = prepare_snapshot_folder(config.snapshot_prefix)
image_directory, snapshot_directory = prepare_snapshot_and_image_folder(
config.snapshot_prefix, iterations, config.image_save_iterations)
# Load datasets
train_loader_a = get_data_loader(config.datasets['train_a'], batch_size)
train_loader_b = get_data_loader(config.datasets['train_b'], batch_size)
test_loader_b = get_data_loader(
config.datasets['test_b'],
batch_size=config.hyperparameters['test_batch_size'])
best_score = 0
for ep in range(0, MAX_EPOCHS):
for it, ((images_a, labels_a), (images_b, labels_b)) in enumerate(
itertools.izip(train_loader_a, train_loader_b)):
if images_a.size(0) != batch_size or images_b.size(
0) != batch_size:
continue
trainer.dis.train()
images_a = Variable(images_a.cuda(opts.gpu))
labels_a = Variable(labels_a.cuda(opts.gpu)).view(images_a.size(0))
images_b = Variable(images_b.cuda(opts.gpu))
# Main training code
trainer.dis_update(images_a, labels_a, images_b,
config.hyperparameters)
x_aa, x_ba, x_ab, x_bb = trainer.gen_update(
images_a, images_b, config.hyperparameters)
# Dump training stats in log file
if (iterations + 1) % config.display == 0:
write_loss(iterations, max_iterations, trainer, train_writer)
# # Save network weights
if (iterations + 1) % config.snapshot_save_iterations == 0:
trainer.dis.eval()
score = 0
num_samples = 0
for tit, (test_images_b,
test_labels_b) in enumerate(test_loader_b):
test_images_b = Variable(test_images_b.cuda(opts.gpu))
test_labels_b = Variable(test_labels_b.cuda(
opts.gpu)).view(test_images_b.size(0))
cls_outputs = trainer.dis.classify_b(test_images_b)
_, cls_predicts = torch.max(cls_outputs.data, 1)
cls_acc = (cls_predicts == test_labels_b.data).sum()
score += cls_acc
num_samples += test_images_b.size(0)
score /= 1.0 * num_samples
print('Classification accuracy for Test_B dataset: %4.4f' %
score)
if score > best_score:
best_score = score
trainer.save(config.snapshot_prefix, iterations=-1)
train_writer.add_summary(summary.scalar('test_b_acc', score),
iterations + 1)
img_name = image_directory + "/images_a.jpg"
torchvision.utils.save_image(images_a.data / 2 + 0.5, img_name)
img_name = image_directory + "/images_b.jpg"
torchvision.utils.save_image(images_b.data / 2 + 0.5, img_name)
img_name = image_directory + "/x_aa.jpg"
torchvision.utils.save_image(x_aa.data / 2 + 0.5, img_name)
img_name = image_directory + "/x_ab.jpg"
torchvision.utils.save_image(x_ab.data / 2 + 0.5, img_name)
img_name = image_directory + "/x_bb.jpg"
torchvision.utils.save_image(x_bb.data / 2 + 0.5, img_name)
img_name = image_directory + "/x_ba.jpg"
torchvision.utils.save_image(x_ba.data / 2 + 0.5, img_name)
iterations += 1
if iterations == max_iterations:
return
def train(self, data_loader, stage, text_dim, gf_dim, condition_dim, z_dim, df_dim, res_num):
if stage == 1:
netG, netD = self.load_network_stageI(text_dim, gf_dim, condition_dim, z_dim, df_dim)
else:
netG, netD = self.load_network_stageII(text_dim, gf_dim, condition_dim, z_dim, df_dim, res_num)
batch_size = self.batch_size
noise = torch.FloatTensor(batch_size, self.nz)
fixed_noise = torch.FloatTensor(batch_size, self.nz).normal_(0, 1)
real_labels = torch.ones(batch_size)
fake_labels = torch.zeros(batch_size)
if self.cuda:
noise, fixed_noise = noise.cuda(), fixed_noise.cuda()
real_labels, fake_labels = real_labels.cuda(), fake_labels.cuda()
optimizerD = optim.Adam(netD.parameters(),
lr=self.discriminator_lr, betas=(0.5, 0.999))
netG_para = []
for p in netG.parameters():
if p.requires_grad:
netG_para.append(p)
optimizerG = optim.Adam(netG_para,
self.generator_lr,
betas=(0.5, 0.999))
count = 0
for epoch in range(self.max_epoch):
start_t = time.time()
if epoch % self.lr_decay_step == 0 and epoch > 0:
self.generator_lr *= 0.5
for param_group in optimizerG.param_groups:
param_group['lr'] = self.generator_lr
self.discriminator_lr *= 0.5
for param_group in optimizerD.param_groups:
param_group['lr'] = self.discriminator_lr
for i, data in enumerate(data_loader, 0):
######################################################
# (1) Prepare training data
######################################################
real_imgs, txt_embedding = data
if self.cuda:
real_imgs = real_imgs.cuda()
txt_embedding = txt_embedding.cuda()
#######################################################
# (2) Generate fake images
######################################################
noise.data.normal_(0, 1)
inputs = (txt_embedding, noise)
_, fake_imgs, mu, logvar = \
nn.parallel.data_parallel(netG, inputs, self.gpus)
############################
# (3) Update D network
###########################
netD.zero_grad()
errD, errD_real, errD_wrong, errD_fake = \
compute_discriminator_loss(netD, real_imgs, fake_imgs,
real_labels, fake_labels,
mu, self.gpus)
errD.backward()
optimizerD.step()
############################
# (2) Update G network
###########################
netG.zero_grad()
errG = compute_generator_loss(netD, fake_imgs,
real_labels, mu, self.gpus)
if self.regularizer == 'KL':
regularizer_loss = KL_loss(mu, logvar)
else:
regularizer_loss = JSD_loss(mu, logvar)
errG_total = errG + regularizer_loss * self.coef_kl
errG_total.backward()
optimizerG.step()
count = count + 1
if i % 100 == 0:
summary_D = summary.scalar('D_loss', errD.item())
summary_D_r = summary.scalar('D_loss_real', errD_real)
summary_D_w = summary.scalar('D_loss_wrong', errD_wrong)
summary_D_f = summary.scalar('D_loss_fake', errD_fake)
summary_G = summary.scalar('G_loss', errG.item())
summary_KL = summary.scalar('Regularizer_loss', regularizer_loss.item())
self.summary_writer.add_summary(summary_D, count)
self.summary_writer.add_summary(summary_D_r, count)
self.summary_writer.add_summary(summary_D_w, count)
self.summary_writer.add_summary(summary_D_f, count)
self.summary_writer.add_summary(summary_G, count)
self.summary_writer.add_summary(summary_KL, count)
# save the image result for each epoch
inputs = (txt_embedding, fixed_noise)
lr_fake, fake, _, _ = \
nn.parallel.data_parallel(netG, inputs, self.gpus)
save_img_results(real_imgs.cpu(), fake, epoch, self.image_dir)
if lr_fake is not None:
save_img_results(None, lr_fake, epoch, self.image_dir)
end_t = time.time()
print('''[%d/%d][%d/%d] Loss_D: %.4f Loss_G: %.4f Loss_KL: %.4f
Loss_real: %.4f Loss_wrong:%.4f Loss_fake %.4f
Total Time: %.2fsec
'''
% (epoch, self.max_epoch, i, len(data_loader),
errD.item(), errG.item(), regularizer_loss.item(),
errD_real, errD_wrong, errD_fake, (end_t - start_t)))
if epoch % self.snapshot_interval == 0:
save_model(netG, netD, epoch, self.model_dir)
#
save_model(netG, netD, self.max_epoch, self.model_dir)
#
self.summary_writer.close()
def main(argv):
(opts, args) = parser.parse_args(argv)
# Load experiment setting
assert isinstance(opts, object)
config = NetConfig(opts.config)
train_writer = tensorboard.FileWriter("%s/%s" % (opts.log,os.path.splitext(os.path.basename(opts.config))[0]))
max_iterations = config.hyperparameters['max_iterations']
batch_size = config.hyperparameters['batch_size']
vae_enc_w = config.hyperparameters['vae_enc_w']
vae_ll_w = config.hyperparameters['vae_ll_w']
gan_w = config.hyperparameters['gan_w']
ch = config.hyperparameters['ch']
gen_net = config.hyperparameters['gen']
dis_net = config.hyperparameters['dis']
image_size = config.datasets['a']['image_size']
input_dims = list()
input_dims.append(config.datasets['a']['channels'])
input_dims.append(config.datasets['b']['channels'])
# Load datasets
train_loader_a = get_data_loader(config.datasets['a'], batch_size)
train_loader_b = get_data_loader(config.datasets['b'], batch_size)
train_loader_a2 = get_data_loader(config.datasets['a'], batch_size)
train_loader_b2 = get_data_loader(config.datasets['b'], batch_size)
trainer = UNITTrainer(gen_net, dis_net, batch_size, ch, input_dims, image_size, opts.lr)
iterations = 0
if opts.resume == 1:
iterations = resume(trainer, config.snapshot_prefix)
trainer.cuda(opts.gpu)
directory = os.path.dirname(config.snapshot_prefix)
image_directory = directory + "/images"
if not os.path.exists(directory):
os.makedirs(directory)
if not os.path.exists(image_directory):
os.makedirs(image_directory)
write_html(directory + "/index.html", iterations + 1, config.image_save_iterations, image_directory, image_size)
for ep in range(0, MAX_EPOCHS):
for it, (images_a, images_b, images_a2, images_b2) in enumerate(itertools.izip(train_loader_a, train_loader_b, train_loader_a2, train_loader_b2)):
if images_a.size(0) != batch_size or images_b.size(0) != batch_size:
continue
images_a = Variable(images_a.cuda(opts.gpu))
images_b = Variable(images_b.cuda(opts.gpu))
images_a2 = Variable(images_a2.cuda(opts.gpu))
images_b2 = Variable(images_b2.cuda(opts.gpu))
# Main training code
trainer.dis_update(images_a, images_b, images_a2, images_b2)
x_aa, x_ba, x_ab, x_bb = trainer.gen_update(images_a, images_b, gan_w, vae_ll_w, vae_enc_w)
# Dump training stats in log file
if (iterations+1) % config.display == 0:
print("Iteration: %08d/%08d" %(iterations+1,max_iterations))
members = [attr for attr in dir(trainer) \
if not callable(getattr(trainer, attr)) and not attr.startswith("__") and 'loss' in attr]
for m in members:
train_writer.add_summary(summary.scalar(m, getattr(trainer, m)), iterations + 1)
members = [attr for attr in dir(trainer) \
if not callable(getattr(trainer, attr)) and not attr.startswith("__") and 'acc' in attr]
for m in members:
train_writer.add_summary(summary.scalar(m, getattr(trainer, m)), iterations + 1)
# Save intermediate visualization results
if (iterations+1) % config.image_save_iterations == 0:
assembled_images = make_save_image(images_a[0:1,::], x_aa[0:1,::], x_ab[0:1,::], images_b[0:1,::], x_ba[0:1,::], x_bb[0:1,::])
img_filename = '%s/gen_%08d.jpg' % (image_directory, iterations + 1)
torchvision.utils.save_image(assembled_images.data / 2 + 0.5, img_filename, nrow=1)
write_html(directory + "/index.html", iterations + 1, config.image_save_iterations, image_directory, image_size)
else:
assembled_images = make_save_image(images_a[0:1,::], x_aa[0:1,::], x_ab[0:1,::], images_b[0:1,::],x_ba[0:1,::], x_bb[0:1,::])
img_filename = '%s/gen.jpg' % (image_directory)
torchvision.utils.save_image(assembled_images.data / 2 + 0.5, img_filename, nrow=1)
# Save network weights
if (iterations+1) % config.snapshot_save_iterations == 0:
gen_filename = '%s_gen_%08d.pkl' % (config.snapshot_prefix, iterations + 1)
dis_filename = '%s_dis_%08d.pkl' % (config.snapshot_prefix, iterations + 1)
torch.save(trainer.gen.state_dict(), gen_filename)
torch.save(trainer.dis.state_dict(), dis_filename)
iterations += 1
if iterations == max_iterations:
return
def train(self, data_loader, stage=1, max_objects=3):
if stage == 1:
netG, netD = self.load_network_stageI()
else:
netG, netD = self.load_network_stageII()
nz = cfg.Z_DIM
batch_size = self.batch_size
noise = Variable(torch.FloatTensor(batch_size, nz))
# with torch.no_grad():
fixed_noise = Variable(torch.FloatTensor(batch_size, nz).normal_(0, 1),
requires_grad=False)
real_labels = Variable(torch.FloatTensor(batch_size).fill_(1))
fake_labels = Variable(torch.FloatTensor(batch_size).fill_(0))
if cfg.CUDA:
noise, fixed_noise = noise.cuda(), fixed_noise.cuda()
real_labels, fake_labels = real_labels.cuda(), fake_labels.cuda()
generator_lr = cfg.TRAIN.GENERATOR_LR
discriminator_lr = cfg.TRAIN.DISCRIMINATOR_LR
lr_decay_step = cfg.TRAIN.LR_DECAY_EPOCH
netG_para = []
for p in netG.parameters():
if p.requires_grad:
netG_para.append(p)
optimizerD = optim.Adam(netD.parameters(),
lr=cfg.TRAIN.DISCRIMINATOR_LR,
betas=(0.5, 0.999))
optimizerG = optim.Adam(netG_para,
lr=cfg.TRAIN.GENERATOR_LR,
betas=(0.5, 0.999))
count = 0
for epoch in range(self.max_epoch):
start_t = time.time()
if epoch % lr_decay_step == 0 and epoch > 0:
generator_lr *= 0.5
for param_group in optimizerG.param_groups:
param_group['lr'] = generator_lr
discriminator_lr *= 0.5
for param_group in optimizerD.param_groups:
param_group['lr'] = discriminator_lr
for i, data in enumerate(data_loader, 0):
######################################################
# (1) Prepare training data
######################################################
real_img_cpu, bbox, label, txt_embedding = data
real_imgs = Variable(real_img_cpu)
txt_embedding = Variable(txt_embedding)
if cfg.CUDA:
real_imgs = real_imgs.cuda()
if cfg.STAGE == 1:
bbox = bbox.cuda()
elif cfg.STAGE == 2:
bbox = [bbox[0].cuda(), bbox[1].cuda()]
label = label.cuda()
txt_embedding = txt_embedding.cuda()
if cfg.STAGE == 1:
bbox = bbox.view(-1, 4)
transf_matrices_inv = compute_transformation_matrix_inverse(
bbox)
transf_matrices_inv = transf_matrices_inv.view(
real_imgs.shape[0], max_objects, 2, 3)
transf_matrices = compute_transformation_matrix(bbox)
transf_matrices = transf_matrices.view(
real_imgs.shape[0], max_objects, 2, 3)
elif cfg.STAGE == 2:
_bbox = bbox[0].view(-1, 4)
transf_matrices_inv = compute_transformation_matrix_inverse(
_bbox)
transf_matrices_inv = transf_matrices_inv.view(
real_imgs.shape[0], max_objects, 2, 3)
_bbox = bbox[1].view(-1, 4)
transf_matrices_inv_s2 = compute_transformation_matrix_inverse(
_bbox)
transf_matrices_inv_s2 = transf_matrices_inv_s2.view(
real_imgs.shape[0], max_objects, 2, 3)
transf_matrices_s2 = compute_transformation_matrix(_bbox)
transf_matrices_s2 = transf_matrices_s2.view(
real_imgs.shape[0], max_objects, 2, 3)
# produce one-hot encodings of the labels
_labels = label.long()
# remove -1 to enable one-hot converting
_labels[_labels < 0] = 80
# label_one_hot = torch.cuda.FloatTensor(noise.shape[0], max_objects, 81).fill_(0)
label_one_hot = torch.FloatTensor(noise.shape[0], max_objects,
81).fill_(0)
label_one_hot = label_one_hot.scatter_(2, _labels, 1).float()
#######################################################
# (2) Generate fake images
######################################################
noise.data.normal_(0, 1)
if cfg.STAGE == 1:
inputs = (txt_embedding, noise, transf_matrices_inv,
label_one_hot)
elif cfg.STAGE == 2:
inputs = (txt_embedding, noise, transf_matrices_inv,
transf_matrices_s2, transf_matrices_inv_s2,
label_one_hot)
_, fake_imgs, mu, logvar, _ = nn.parallel.data_parallel(
netG, inputs, self.gpus)
# _, fake_imgs, mu, logvar, _ = netG(txt_embedding, noise, transf_matrices_inv, label_one_hot)
############################
# (3) Update D network
###########################
netD.zero_grad()
if cfg.STAGE == 1:
errD, errD_real, errD_wrong, errD_fake = \
compute_discriminator_loss(netD, real_imgs, fake_imgs,
real_labels, fake_labels,
label_one_hot, transf_matrices, transf_matrices_inv,
mu, self.gpus)
elif cfg.STAGE == 2:
errD, errD_real, errD_wrong, errD_fake = \
compute_discriminator_loss(netD, real_imgs, fake_imgs,
real_labels, fake_labels,
label_one_hot, transf_matrices_s2, transf_matrices_inv_s2,
mu, self.gpus)
errD.backward(retain_graph=True)
optimizerD.step()
############################
# (2) Update G network
###########################
netG.zero_grad()
if cfg.STAGE == 1:
errG = compute_generator_loss(netD, fake_imgs, real_labels,
label_one_hot,
transf_matrices,
transf_matrices_inv, mu,
self.gpus)
elif cfg.STAGE == 2:
errG = compute_generator_loss(netD, fake_imgs, real_labels,
label_one_hot,
transf_matrices_s2,
transf_matrices_inv_s2, mu,
self.gpus)
kl_loss = KL_loss(mu, logvar)
errG_total = errG + kl_loss * cfg.TRAIN.COEFF.KL
errG_total.backward()
optimizerG.step()
count += 1
if i % 500 == 0:
summary_D = summary.scalar('D_loss', errD.item())
summary_D_r = summary.scalar('D_loss_real', errD_real)
summary_D_w = summary.scalar('D_loss_wrong', errD_wrong)
summary_D_f = summary.scalar('D_loss_fake', errD_fake)
summary_G = summary.scalar('G_loss', errG.item())
summary_KL = summary.scalar('KL_loss', kl_loss.item())
self.summary_writer.add_summary(summary_D, count)
self.summary_writer.add_summary(summary_D_r, count)
self.summary_writer.add_summary(summary_D_w, count)
self.summary_writer.add_summary(summary_D_f, count)
self.summary_writer.add_summary(summary_G, count)
self.summary_writer.add_summary(summary_KL, count)
# save the image result for each epoch
with torch.no_grad():
if cfg.STAGE == 1:
inputs = (txt_embedding, noise,
transf_matrices_inv, label_one_hot)
elif cfg.STAGE == 2:
inputs = (txt_embedding, noise,
transf_matrices_inv, transf_matrices_s2,
transf_matrices_inv_s2, label_one_hot)
lr_fake, fake, _, _, _ = nn.parallel.data_parallel(
netG, inputs, self.gpus)
save_img_results(real_img_cpu, fake, epoch,
self.image_dir)
if lr_fake is not None:
save_img_results(None, lr_fake, epoch,
self.image_dir)
with torch.no_grad():
if cfg.STAGE == 1:
inputs = (txt_embedding, noise, transf_matrices_inv,
label_one_hot)
elif cfg.STAGE == 2:
inputs = (txt_embedding, noise, transf_matrices_inv,
transf_matrices_s2, transf_matrices_inv_s2,
label_one_hot)
lr_fake, fake, _, _, _ = nn.parallel.data_parallel(
netG, inputs, self.gpus)
save_img_results(real_img_cpu, fake, epoch, self.image_dir)
if lr_fake is not None:
save_img_results(None, lr_fake, epoch, self.image_dir)
end_t = time.time()
print(
'''[%d/%d][%d/%d] Loss_D: %.4f Loss_G: %.4f Loss_KL: %.4f
Loss_real: %.4f Loss_wrong:%.4f Loss_fake %.4f
Total Time: %.2fsec
''' %
(epoch, self.max_epoch, i, len(data_loader), errD.item(),
errG.item(), kl_loss.item(), errD_real, errD_wrong, errD_fake,
(end_t - start_t)))
if epoch % self.snapshot_interval == 0:
save_model(netG, netD, optimizerG, optimizerD, epoch,
self.model_dir)
#
save_model(netG, netD, optimizerG, optimizerD, epoch, self.model_dir)
#
self.summary_writer.close()
def main(argv):
(opts, args) = parser.parse_args(argv)
# Load experiment setting
assert isinstance(opts, object)
config = NetConfig(opts.config)
batch_size = config.hyperparameters['batch_size']
max_iterations = config.hyperparameters['max_iterations']
cmd = "trainer=%s(config.hyperparameters)" % config.hyperparameters['trainer']
local_dict = locals()
exec(cmd,globals(),local_dict)
trainer = local_dict['trainer']
trainer.cuda(opts.gpu)
iterations = 0
train_writer = tensorboard.FileWriter("%s/%s" % (opts.log,os.path.splitext(os.path.basename(opts.config))[0]))
snapshot_directory = prepare_snapshot_folder(config.snapshot_prefix)
image_directory, snapshot_directory = prepare_snapshot_and_image_folder(config.snapshot_prefix, iterations, config.image_save_iterations)
# Load datasets
train_loader_a = get_data_loader(config.datasets['train_a'], batch_size)
train_loader_b = get_data_loader(config.datasets['train_b'], batch_size)
test_loader_b = get_data_loader(config.datasets['test_b'], batch_size = config.hyperparameters['test_batch_size'])
best_score = 0
for ep in range(0, MAX_EPOCHS):
for it, ((images_a, labels_a), (images_b,labels_b)) in enumerate(itertools.izip(train_loader_a, train_loader_b)):
if images_a.size(0) != batch_size or images_b.size(0) != batch_size:
continue
trainer.dis.train()
images_a = Variable(images_a.cuda(opts.gpu))
labels_a = Variable(labels_a.cuda(opts.gpu)).view(images_a.size(0))
images_b = Variable(images_b.cuda(opts.gpu))
# Main training code
trainer.dis_update(images_a, labels_a, images_b, config.hyperparameters)
x_aa, x_ba, x_ab, x_bb = trainer.gen_update(images_a, images_b, config.hyperparameters)
# Dump training stats in log file
if (iterations+1) % config.display == 0:
write_loss(iterations, max_iterations, trainer, train_writer)
# # Save network weights
if (iterations+1) % config.snapshot_save_iterations == 0:
trainer.dis.eval()
score = 0
num_samples = 0
for tit, (test_images_b, test_labels_b) in enumerate(test_loader_b):
test_images_b = Variable(test_images_b.cuda(opts.gpu))
test_labels_b = Variable(test_labels_b.cuda(opts.gpu)).view(test_images_b.size(0))
cls_outputs = trainer.dis.classify_b(test_images_b)
_, cls_predicts = torch.max(cls_outputs.data, 1)
cls_acc = (cls_predicts == test_labels_b.data).sum()
score += cls_acc
num_samples += test_images_b.size(0)
score /= 1.0 * num_samples
print('Classification accuracy for Test_B dataset: %4.4f' % score)
if score > best_score:
best_score = score
trainer.save(config.snapshot_prefix, iterations=-1)
train_writer.add_summary(summary.scalar('test_b_acc', score), iterations + 1)
img_name = image_directory + "/images_a.jpg"
torchvision.utils.save_image(images_a.data / 2 + 0.5, img_name)
img_name = image_directory + "/images_b.jpg"
torchvision.utils.save_image(images_b.data / 2 + 0.5, img_name)
img_name = image_directory + "/x_aa.jpg"
torchvision.utils.save_image(x_aa.data / 2 + 0.5, img_name)
img_name = image_directory + "/x_ab.jpg"
torchvision.utils.save_image(x_ab.data / 2 + 0.5, img_name)
img_name = image_directory + "/x_bb.jpg"
torchvision.utils.save_image(x_bb.data / 2 + 0.5, img_name)
img_name = image_directory + "/x_ba.jpg"
torchvision.utils.save_image(x_ba.data / 2 + 0.5, img_name)
iterations += 1
if iterations == max_iterations:
return
def train(self):
self.netG, self.netsD, self.num_Ds,\
self.inception_model, start_count = load_network(self.gpus)
avg_param_G = copy_G_params(self.netG)
self.optimizerG, self.optimizersD = \
define_optimizers(self.netG, self.netsD)
self.criterion = nn.BCELoss()
self.SATcriterion = nn.CrossEntropyLoss()
self.real_labels = Variable(
torch.FloatTensor(self.batch_size).fill_(1))
self.fake_labels = Variable(
torch.FloatTensor(self.batch_size).fill_(0))
self.gradient_one = torch.FloatTensor([1.0])
self.gradient_half = torch.FloatTensor([0.5])
nz = cfg.GAN.Z_DIM
noise = Variable(torch.FloatTensor(self.batch_size, nz))
fixed_noise = Variable(
torch.FloatTensor(self.batch_size, nz).normal_(0, 1))
# Data parameters
data_folder = 'birds_output' # folder with data files saved by create_input_files.py
data_name = 'CUB_5_cap_per_img_5_min_word_freq' # base name shared by data files
normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
# Show, Attend, and Tell Dataloader
train_loader = torch.utils.data.DataLoader(
CaptionDataset(data_folder,
data_name,
'TRAIN',
transform=transforms.Compose([normalize])),
batch_size=self.batch_size,
shuffle=True,
num_workers=int(cfg.WORKERS),
pin_memory=True)
if cfg.CUDA:
self.criterion.cuda()
self.SATcriterion.cuda() # Compute SATloss
self.real_labels = self.real_labels.cuda()
self.fake_labels = self.fake_labels.cuda()
self.gradient_one = self.gradient_one.cuda()
self.gradient_half = self.gradient_half.cuda()
noise, fixed_noise = noise.cuda(), fixed_noise.cuda()
predictions = []
count = start_count
start_epoch = start_count // (self.num_batches)
for epoch in range(start_epoch, self.max_epoch):
start_t = time.time()
# for step, data in enumerate(self.data_loader, 0):
for step, data in enumerate(zip(self.data_loader, train_loader),
0):
data_1 = data[0]
_, caps, caplens = data[1]
data = data_1
#######################################################
# (0) Prepare training data
######################################################
self.imgs_tcpu, self.real_imgs, self.wrong_imgs, \
self.txt_embedding = self.prepare_data(data)
# Testing line for real samples
if epoch == start_epoch and step == 0:
print('Checking real samples at first...')
save_real(self.imgs_tcpu, self.image_dir)
#######################################################
# (1) Generate fake images
######################################################
noise.data.normal_(0, 1)
self.fake_imgs, self.mu, self.logvar = \
self.netG(noise, self.txt_embedding)
# len(self.fake_imgs) = NUM_BRANCHES
# self.fake_imgs[0].shape = [batch_size, 3, 64, 64]
# self.fake_imgs[1].shape = [batch_size, 3, 128, 128]
# self.fake_imgs[2].shape = [batch_size, 3, 256, 256]
#######################################################
# (*) Forward fake images to SAT
######################################################
from SATmodels import Encoder, DecoderWithAttention
from torch.nn.utils.rnn import pack_padded_sequence
fine_tune_encoder = False
# Read word map
word_map_file = os.path.join(data_folder,
'WORDMAP_' + data_name + '.json')
with open(word_map_file, 'r') as j:
word_map = json.load(j)
# Define the encoder/decoder structure for SAT model
decoder = DecoderWithAttention(attention_dim=512,
embed_dim=512,
decoder_dim=512,
vocab_size=len(word_map),
dropout=0.5).cuda()
decoder_optimizer = torch.optim.Adam(params=filter(
lambda p: p.requires_grad, decoder.parameters()),
lr=4e-4)
encoder = Encoder().cuda()
encoder.fine_tune(fine_tune_encoder)
encoder_optimizer = torch.optim.Adam(
params=filter(lambda p: p.requires_grad,
encoder.parameters()),
lr=1e-4) if fine_tune_encoder else None
SATloss = 0
# Compute the SAT loss after forwarding the SAT model
for idx in range(len(self.fake_imgs)):
img = encoder(self.fake_imgs[idx])
scores, caps_sorted, decode_lengths, alphas, sort_ind = decoder(
img, caps, caplens)
targets = caps_sorted[:, 1:]
scores, _ = pack_padded_sequence(scores,
decode_lengths,
batch_first=True).cuda()
targets, _ = pack_padded_sequence(targets,
decode_lengths,
batch_first=True).cuda()
SATloss += self.SATcriterion(scores, targets) + 1 * (
(1. - alphas.sum(dim=1))**2).mean()
# Set zero_grad for encoder/decoder
decoder_optimizer.zero_grad()
if encoder_optimizer is not None:
encoder_optimizer.zero_grad()
#######################################################
# (2) Update D network
######################################################
errD_total = 0
for i in range(self.num_Ds):
errD = self.train_Dnet(i, count)
errD_total += errD
#######################################################
# (3) Update G network: maximize log(D(G(z)))
######################################################
kl_loss, errG_total = self.train_Gnet(count)
for p, avg_p in zip(self.netG.parameters(), avg_param_G):
avg_p.mul_(0.999).add_(0.001, p.data)
# Combine with G and SAT first, then back propagation
errG_total += SATloss
errG_total.backward()
self.optimizerG.step()
#######################################################
# (*) Update SAT network:
######################################################
# Update weights
decoder_optimizer.step()
if encoder_optimizer is not None:
encoder_optimizer.step()
#######################################################
# (*) Prediction and Inception score:
######################################################
pred = self.inception_model(self.fake_imgs[-1].detach())
predictions.append(pred.data.cpu().numpy())
if count % 100 == 0:
summary_D = summary.scalar('D_loss', errD_total.item())
summary_G = summary.scalar('G_loss', errG_total.item())
summary_KL = summary.scalar('KL_loss', kl_loss.item())
self.summary_writer.add_summary(summary_D, count)
self.summary_writer.add_summary(summary_G, count)
self.summary_writer.add_summary(summary_KL, count)
count += 1
#######################################################
# (*) Save Images/Log/Model per SNAPSHOT_INTERVAL:
######################################################
if count % cfg.TRAIN.SNAPSHOT_INTERVAL == 0:
save_model(self.netG, avg_param_G, self.netsD, count,
self.model_dir)
# Save images
backup_para = copy_G_params(self.netG)
load_params(self.netG, avg_param_G)
#
self.fake_imgs, _, _ = self.netG(fixed_noise,
self.txt_embedding)
save_img_results(self.imgs_tcpu, self.fake_imgs,
self.num_Ds, count, self.image_dir,
self.summary_writer)
#
load_params(self.netG, backup_para)
# Compute inception score
if len(predictions) > 500:
predictions = np.concatenate(predictions, 0)
mean, std = compute_inception_score(predictions, 10)
# print('mean:', mean, 'std', std)
m_incep = summary.scalar('Inception_mean', mean)
self.summary_writer.add_summary(m_incep, count)
#
mean_nlpp, std_nlpp = negative_log_posterior_probability(
predictions, 10)
m_nlpp = summary.scalar('NLPP_mean', mean_nlpp)
self.summary_writer.add_summary(m_nlpp, count)
#
predictions = []
end_t = time.time()
print('''[%d/%d][%d]
Loss_D: %.2f Loss_G: %.2f Loss_KL: %.2f Time: %.2fs
'''
# D(real): %.4f D(wrong):%.4f D(fake) %.4f
%
(epoch, self.max_epoch, self.num_batches, errD_total.item(),
errG_total.item(), kl_loss.item(), end_t - start_t))
save_model(self.netG, avg_param_G, self.netsD, count, self.model_dir)
self.summary_writer.close()
def train(self):
self.netG, self.netsD, self.num_Ds,\
self.inception_model, start_count = load_network(self.gpus)
avg_param_G = copy_G_params(self.netG)
self.optimizerG, self.optimizersD = \
define_optimizers(self.netG, self.netsD)
self.criterion = nn.BCELoss()
self.real_labels = \
Variable(torch.FloatTensor(self.batch_size).fill_(1))
self.fake_labels = \
Variable(torch.FloatTensor(self.batch_size).fill_(0))
self.gradient_one = torch.FloatTensor([1.0])
self.gradient_half = torch.FloatTensor([0.5])
nz = cfg.GAN.Z_DIM
noise = Variable(torch.FloatTensor(self.batch_size, nz))
fixed_noise = \
Variable(torch.FloatTensor(self.batch_size, nz).normal_(0, 1))
if cfg.CUDA:
self.criterion.cuda()
self.real_labels = self.real_labels.cuda()
self.fake_labels = self.fake_labels.cuda()
self.gradient_one = self.gradient_one.cuda()
self.gradient_half = self.gradient_half.cuda()
noise, fixed_noise = noise.cuda(), fixed_noise.cuda()
predictions = []
count = start_count
start_epoch = start_count // (self.num_batches)
for epoch in range(start_epoch, self.max_epoch):
start_t = time.time()
for step, data in enumerate(self.data_loader, 0):
#######################################################
# (0) Prepare training data
######################################################
self.imgs_tcpu, self.real_imgs, self.wrong_imgs, \
self.txt_embedding = self.prepare_data(data)
#######################################################
# (1) Generate fake images
######################################################
noise.data.normal_(0, 1)
self.fake_imgs, self.mu, self.logvar = \
self.netG(noise, self.txt_embedding)
#######################################################
# (2) Update D network
######################################################
errD_total = 0
for i in range(self.num_Ds):
errD = self.train_Dnet(i, count)
errD_total += errD
#######################################################
# (3) Update G network: maximize log(D(G(z)))
######################################################
kl_loss, errG_total = self.train_Gnet(count)
for p, avg_p in zip(self.netG.parameters(), avg_param_G):
avg_p.mul_(0.999).add_(0.001, p.data)
# for inception score
pred = self.inception_model(self.fake_imgs[-1].detach())
predictions.append(pred.data.cpu().numpy())
if count % 100 == 0:
summary_D = summary.scalar('D_loss', errD_total.data[0])
summary_G = summary.scalar('G_loss', errG_total.data[0])
summary_KL = summary.scalar('KL_loss', kl_loss.data[0])
self.summary_writer.add_summary(summary_D, count)
self.summary_writer.add_summary(summary_G, count)
self.summary_writer.add_summary(summary_KL, count)
count = count + 1
if count % cfg.TRAIN.SNAPSHOT_INTERVAL == 0:
save_model(self.netG, avg_param_G, self.netsD, count, self.model_dir)
# Save images
backup_para = copy_G_params(self.netG)
load_params(self.netG, avg_param_G)
#
self.fake_imgs, _, _ = \
self.netG(fixed_noise, self.txt_embedding)
save_img_results(self.imgs_tcpu, self.fake_imgs, self.num_Ds,
count, self.image_dir, self.summary_writer)
#
load_params(self.netG, backup_para)
# Compute inception score
if len(predictions) > 500:
predictions = np.concatenate(predictions, 0)
mean, std = compute_inception_score(predictions, 10)
# print('mean:', mean, 'std', std)
m_incep = summary.scalar('Inception_mean', mean)
self.summary_writer.add_summary(m_incep, count)
#
mean_nlpp, std_nlpp = \
negative_log_posterior_probability(predictions, 10)
m_nlpp = summary.scalar('NLPP_mean', mean_nlpp)
self.summary_writer.add_summary(m_nlpp, count)
#
predictions = []
end_t = time.time()
print('''[%d/%d][%d]
Loss_D: %.2f Loss_G: %.2f Loss_KL: %.2f Time: %.2fs
''' # D(real): %.4f D(wrong):%.4f D(fake) %.4f
% (epoch, self.max_epoch, self.num_batches,
errD_total.data[0], errG_total.data[0],
kl_loss.data[0], end_t - start_t))
save_model(self.netG, avg_param_G, self.netsD, count, self.model_dir)
self.summary_writer.close()
def run():
"""Run custom scalar demo and generate event files."""
step = tf.placeholder(tf.float32, shape=[])
with tf.name_scope('loss'):
# Specify 2 different loss values, each tagged differently.
summary_lib.scalar('foo', tf.pow(0.9, step))
summary_lib.scalar('bar', tf.pow(0.85, step + 2))
# Log metric baz as well as upper and lower bounds for a margin chart.
middle_baz_value = step + 4 * tf.random_uniform([]) - 2
summary_lib.scalar('baz', middle_baz_value)
summary_lib.scalar('baz_lower',
middle_baz_value - 6.42 - tf.random_uniform([]))
summary_lib.scalar('baz_upper',
middle_baz_value + 6.42 + tf.random_uniform([]))
with tf.name_scope('trigFunctions'):
summary_lib.scalar('cosine', tf.cos(step))
summary_lib.scalar('sine', tf.sin(step))
summary_lib.scalar('tangent', tf.tan(step))
merged_summary = tf.summary.merge_all()
with tf.Session() as sess, tf.summary.FileWriter(LOGDIR) as writer:
# We only need to specify the layout once (instead of per step).
layout_summary = summary_lib.custom_scalar_pb(
layout_pb2.Layout(category=[
layout_pb2.Category(
title='losses',
chart=[
layout_pb2.Chart(
title='losses',
multiline=layout_pb2.MultilineChartContent(
tag=[r'loss(?!.*margin.*)'],)),
layout_pb2.Chart(
title='baz',
margin=layout_pb2.MarginChartContent(
series=[
layout_pb2.MarginChartContent.Series(
value='loss/baz/scalar_summary',
lower='loss/baz_lower/scalar_summary',
upper='loss/baz_upper/scalar_summary'
),
],)),
]),
layout_pb2.Category(
title='trig functions',
chart=[
layout_pb2.Chart(
title='wave trig functions',
multiline=layout_pb2.MultilineChartContent(
tag=[
r'trigFunctions/cosine', r'trigFunctions/sine'
],)),
# The range of tangent is different. Give it its own chart.
layout_pb2.Chart(
title='tan',
multiline=layout_pb2.MultilineChartContent(
tag=[r'trigFunctions/tangent'],)),
],
# This category we care less about. Make it initially closed.
closed=True),
]))
writer.add_summary(layout_summary)
for i in xrange(42):
summary = sess.run(merged_summary, feed_dict={step: i})
writer.add_summary(summary, global_step=i)
def evaluate(iter_cnt, filepath, model, corpus, args, logging=True):
if logging:
valid_writer = FileWriter(args.run_path + "/valid", flush_secs=5)
pos_file_path = "{}.pos.txt".format(filepath)
neg_file_path = "{}.neg.txt".format(filepath)
pos_batch_loader = FileLoader(
[tuple([pos_file_path, os.path.dirname(args.eval)])], args.batch_size)
neg_batch_loader = FileLoader(
[tuple([neg_file_path, os.path.dirname(args.eval)])], args.batch_size)
batchify = lambda bch: make_batch(model.embedding_layer, bch)
model.eval()
criterion = model.compute_loss
auc_meter = AUCMeter()
scores = [np.asarray([], dtype='float32') for i in range(2)]
for loader_id, loader in tqdm(
enumerate((neg_batch_loader, pos_batch_loader))):
for data in tqdm(loader):
data = map(corpus.get, data)
batch = None
if not args.eval_use_content:
batch = (batchify(data[0][0]), batchify(data[1][0]))
else:
batch = (map(batchify, data[0]), map(batchify, data[1]))
new_batch = []
for x in batch:
for y in x:
new_batch.append(y)
batch = new_batch
labels = torch.ones(batch[0].size(1)).type(
torch.LongTensor) * loader_id
if args.cuda:
batch = [x.cuda() for x in batch]
labels = labels.cuda()
if not args.eval_use_content:
batch = (Variable(batch[0], volatile=True),
Variable(batch[1], volatile=True))
else:
batch = (Variable(batch[0], volatile=True),
Variable(batch[1], volatile=True),
Variable(batch[2], volatile=True),
Variable(batch[3], volatile=True))
labels = Variable(labels)
if not args.eval_use_content:
repr_left = model(batch[0])
repr_right = model(batch[1])
else:
repr_left = model(batch[0]) + model(batch[1])
repr_right = model(batch[2]) + model(batch[3])
output = model.compute_similarity(repr_left, repr_right)
if model.criterion.startswith('classification'):
assert output.size(1) == 2
output = nn.functional.log_softmax(output)
current_scores = -output[:, loader_id].data.cpu().squeeze(
).numpy()
output = output[:, 1]
else:
assert output.size(1) == 1
current_scores = output.data.cpu().squeeze().numpy()
auc_meter.add(output.data, labels.data)
scores[loader_id] = np.append(scores[loader_id], current_scores)
auc_score = auc_meter.value()
auc10_score = auc_meter.value(0.1)
auc05_score = auc_meter.value(0.05)
auc02_score = auc_meter.value(0.02)
auc01_score = auc_meter.value(0.01)
if model.criterion.startswith('classification'):
avg_score = (scores[1].mean() + scores[0].mean()) * 0.5
else:
avg_score = scores[1].mean() - scores[0].mean()
outputManager.say(
"\r[{}] auc(.01): {:.3f} auc(.02): {:.3f} auc(.05): {:.3f}"
" auc(.1): {:.3f} auc: {:.3f}"
" scores: {:.2f} ({:.2f} {:.2f})\n".format(
os.path.basename(filepath).split('.')[0], auc01_score, auc02_score,
auc05_score, auc10_score, auc_score, avg_score, scores[1].mean(),
scores[0].mean()))
if logging:
s = summary.scalar('auc', auc_score)
valid_writer.add_summary(s, iter_cnt)
s = summary.scalar('auc (fpr<0.1)', auc10_score)
valid_writer.add_summary(s, iter_cnt)
s = summary.scalar('auc (fpr<0.05)', auc05_score)
valid_writer.add_summary(s, iter_cnt)
s = summary.scalar('auc (fpr<0.02)', auc02_score)
valid_writer.add_summary(s, iter_cnt)
s = summary.scalar('auc (fpr<0.01)', auc01_score)
valid_writer.add_summary(s, iter_cnt)
valid_writer.close()
return auc05_score
def train(self, data_loader, stage=1):
if stage == 1:
netG, netD, start_epoch = self.load_network_stageI()
else:
netG, netD, start_epoch = self.load_network_stageII()
nz = cfg.Z_DIM
batch_size = self.batch_size
noise = Variable(torch.FloatTensor(batch_size, nz))
fixed_noise = \
Variable(torch.FloatTensor(batch_size, nz).normal_(0, 1),
requires_grad=False)
real_labels = Variable(torch.FloatTensor(batch_size).fill_(1))
fake_labels = Variable(torch.FloatTensor(batch_size).fill_(0))
if cfg.CUDA:
noise, fixed_noise = noise.cuda(), fixed_noise.cuda()
real_labels, fake_labels = real_labels.cuda(), fake_labels.cuda()
generator_lr = cfg.TRAIN.GENERATOR_LR
discriminator_lr = cfg.TRAIN.DISCRIMINATOR_LR
lr_decay_step = cfg.TRAIN.LR_DECAY_EPOCH
optimizerG, optimizerD = self.load_optimizers(netG, netD)
count = 0
for epoch in range(start_epoch, self.max_epoch):
start_t = time.time()
if epoch % lr_decay_step == 0 and epoch > 0:
generator_lr *= 0.5
for param_group in optimizerG.param_groups:
param_group['lr'] = generator_lr
discriminator_lr *= 0.5
for param_group in optimizerD.param_groups:
param_group['lr'] = discriminator_lr
for i, data in enumerate(data_loader, 0):
######################################################
# (1) Prepare training data
######################################################
real_img_cpu, txt_embedding = data
real_imgs = Variable(real_img_cpu)
txt_embedding = Variable(txt_embedding)
if cfg.CUDA:
real_imgs = real_imgs.cuda()
txt_embedding = txt_embedding.float().cuda()
######################################################
# (2) Generate fake images
######################################################
noise.data.normal_(0, 1)
inputs = (txt_embedding, noise)
_, fake_imgs, mu, logvar = \
nn.parallel.data_parallel(netG, inputs, self.gpus)
######################################################
# (3) Update D network
######################################################
netD.zero_grad()
errD, errD_real, errD_wrong, errD_fake = \
compute_discriminator_loss(netD, real_imgs, fake_imgs,
real_labels, fake_labels,
mu, self.gpus)
errD.backward()
optimizerD.step()
######################################################
# (2) Update G network
######################################################
netG.zero_grad()
errG = compute_generator_loss(netD, fake_imgs, real_labels, mu,
self.gpus)
kl_loss = KL_loss(mu, logvar)
errG_total = errG + kl_loss * cfg.TRAIN.COEFF.KL
errG_total.backward()
optimizerG.step()
count = count + 1
if i % 100 == 0:
summary_D = summary.scalar('D_loss', errD.item())
summary_D_r = summary.scalar('D_loss_real', errD_real)
summary_D_w = summary.scalar('D_loss_wrong', errD_wrong)
summary_D_f = summary.scalar('D_loss_fake', errD_fake)
summary_G = summary.scalar('G_loss', errG.item())
summary_KL = summary.scalar('KL_loss', kl_loss.item())
self.summary_writer.add_summary(summary_D, count)
self.summary_writer.add_summary(summary_D_r, count)
self.summary_writer.add_summary(summary_D_w, count)
self.summary_writer.add_summary(summary_D_f, count)
self.summary_writer.add_summary(summary_G, count)
self.summary_writer.add_summary(summary_KL, count)
# save the image result for each epoch
inputs = (txt_embedding, fixed_noise)
lr_fake, fake, _, _ = \
nn.parallel.data_parallel(netG, inputs, self.gpus)
save_img_results(real_img_cpu, fake, epoch, self.image_dir)
if lr_fake is not None:
save_img_results(None, lr_fake, epoch, self.image_dir)
end_t = time.time()
print(
'''[%d/%d][%d/%d] Loss_D: %.4f Loss_G: %.4f Loss_KL: %.4f
Loss_real: %.4f Loss_wrong:%.4f Loss_fake %.4f
Total Time: %.2fsec
''' %
(epoch, self.max_epoch, i, len(data_loader), errD.item(),
errG.item(), kl_loss.item(), errD_real, errD_wrong, errD_fake,
(end_t - start_t)))
if epoch % self.snapshot_interval == 0:
save_model(netG, netD, epoch, self.model_dir)
save_optimizer(optimizerG, optimizerD, self.model_dir)
save_model(netG, netD, self.max_epoch, self.model_dir)
self.summary_writer.close()
def actor_learner_thread(thread_id):
global TMAX, T, lock, epoch
#kv = mx.kvstore.create(args.kv_store)
devs = mx.gpu(1)
#devs = mx.cpu()
dataiter = getGame()
act_dim = dataiter.act_dim
module = getNet(act_dim)
forward_net = sym.get_symbol_forward_bn(act_dim)
forward_module = A3CModule(forward_net,
data_names=('data', ),
label_names=None,
context=devs)
forward_module.bind(data_shapes=[
('data', (1, args.agent_history_length, args.resized_width,
args.resized_height)),
],
label_shapes=None,
for_training=False,
grad_req='null')
act_dim = dataiter.act_dim
# Set up per-episode counters
ep_reward = 0
ep_t = 0
s_t = dataiter.get_initial_state()
terminal = False
# anneal e-greedy probability
final_epsilon = sample_final_epsilon()
initial_epsilon = 0.5
epsilon = 0.5
t = 0
replayMemory = []
episode_max_p = 0
while T < TMAX:
tic = time.time()
with lock:
module.copy_from_module(Module)
forward_module.copy_from_module(module)
t_start = t
epoch += 1
s_batch = []
s1_batch = []
r_batch = []
a_batch = []
R_batch = []
td_batch = []
V_batch = []
terminal_batch = []
while not (terminal or ((t - t_start) == args.t_max)):
batch = mx.io.DataBatch(data=[mx.nd.array([s_t])], label=None)
forward_module.forward(batch, is_train=False)
policy_out, value_out = forward_module.get_outputs()
probs = policy_out.asnumpy()[0]
v_t = value_out.asnumpy()
episode_max_p = max(episode_max_p, max(probs))
#print 'prob', probs, 'pi', policy_out.asnumpy(), 'value', value_out.asnumpy()
#print mx.nd.SoftmaxActivation(policy_out2).asnumpy()
# total_loss.asnumpy(), 'loss_out', loss_out.asnumpy()
action_index = action_select(act_dim, probs, epsilon)
a_t = np.zeros([act_dim])
a_t[action_index] = 1
#print a_t
# scale down eplision
if epsilon > final_epsilon:
epsilon -= (initial_epsilon - final_epsilon) / \
args.anneal_epsilon_timesteps
if T > args.expore_steps:
epsilon = 0
s_t1, r_t, terminal, info = dataiter.act(action_index)
r_t = np.clip(r_t, -1, 1)
t += 1
T += 1
ep_reward += r_t
s_batch.append(s_t)
s1_batch.append(s_t1)
a_batch.append(a_t)
r_batch.append(r_t)
V_batch.append(v_t)
R_batch.append(0)
td_batch.append(0)
terminal_batch.append(terminal)
s_t = s_t1
if terminal:
R_t = np.zeros((1, 1))
else:
batch = mx.io.DataBatch(data=[mx.nd.array([s_t1])], label=None)
forward_module.forward(batch, is_train=False)
#R_t = np.clip(module.get_outputs()[1].asnumpy(), - 2, 2)
R_t = forward_module.get_outputs()[1].asnumpy()
#print R_t
for i in reversed(range(0, t - t_start)):
R_t = r_batch[i] + args.gamma * R_t
R_batch[i] = R_t
#print 'R_t!', R_t
# print mx.nd.array([s_batch[i]]), mx.nd.array(R_t),
# mx.nd.array([a_batch[i]])
td_batch[i] = R_t - V_batch[i]
if len(replayMemory) + len(s_batch) > args.replay_memory_length:
replayMemory[0:(len(s_batch) + len(replayMemory)) -
args.replay_memory_length] = []
for i in range(0, t - t_start):
replayMemory.append(
(s_batch[i], a_batch[i], r_batch[i], s1_batch[i], R_batch[i],
td_batch[i], terminal_batch[i]))
if len(replayMemory) < args.batch_size:
continue
minibatch = random.sample(replayMemory, args.batch_size)
s_minibatch = ([data[0] for data in minibatch])
a_minibatch = ([data[1] for data in minibatch])
R_minibatch = ([data[4] for data in minibatch])
td_minibatch = ([data[5] for data in minibatch])
batch = mx.io.DataBatch(data=[
mx.nd.array(s_minibatch),
mx.nd.array(np.array(R_minibatch).reshape(-1, 1)),
mx.nd.array(a_minibatch),
mx.nd.array(np.array(td_minibatch).reshape(-1, 1))
],
label=None)
#print t, t_start, len(s_batch), len(R_batch), len(a_batch)
# print mx.nd.array(s_batch), mx.nd.array(np.reshape(R_batch,(-1, 1))),
# mx.nd.array(a_batch)
module.clear_gradients()
module.forward(batch, is_train=True)
module.backward()
with lock:
Module.add_gradients_from_module(module)
#Module.clip_gradients(10)
#Module.update()
#Module.clear_gradients()
if terminal:
print "THREAD:", thread_id, "/ TIME", T, "/ TIMESTEP", t, "/ EPSILON", epsilon, "/ REWARD", ep_reward, "/ P_MAX %.4f" % episode_max_p, "/ EPSILON PROGRESS", t / float(
args.anneal_epsilon_timesteps)
elapsed_time = time.time() - start_time
steps_per_sec = T / elapsed_time
print(
"### Performance : {} STEPS in {:.0f} sec. {:.0f} STEPS/sec. {:.2f}M STEPS/hour"
.format(T, elapsed_time, steps_per_sec,
steps_per_sec * 3600 / 1000000.))
s = summary.scalar('score', ep_reward)
summary_writer.add_summary(s, T)
summary_writer.flush()
ep_reward = 0
episode_max_p = 0
terminal = False
s_t = dataiter.get_initial_state()
if args.save_every != 0 and epoch % args.save_every == 0:
save_params(args.save_model_prefix, Module, epoch)
