def main():
#DataGenerator
imsize = cfg.TREE.BASE_SIZE * (2**(cfg.TREE.BRANCH_NUM - 1)) #64, 3
image_transform = transforms.Compose([
transforms.Resize(int(imsize * 76 / 64)),
transforms.RandomCrop(imsize),
transforms.RandomHorizontalFlip()
])
#cfg.DATA_DIR = "data/birds"
dataset = TextDataset(cfg.DATA_DIR,
"train",
base_size=cfg.TREE.BASE_SIZE,
transform=image_transform)
assert dataset
traingenerator = DataGenerator(dataset, batchsize=cfg.TRAIN.BATCH_SIZE)
##Create model
G_model, D_model, GRD_model, CR_model, RNN_model = model_create(dataset)
print("loadmodel_completed")
#Preparation for learning
total_epoch = cfg.TRAIN.MAX_EPOCH
batch_size = traingenerator.batchsize
step_epoch = int(len(dataset) / batch_size)
wrong_step = 3
wrong_step_epoch = int(step_epoch / wrong_step)
image_list, captions_ar, captions_ar_prezeropad, \
z_code, eps_code, mask, keys_list, captions_label, \
real_label, fake_label = next(traingenerator)
traingenerator.count = 0
#for image plot
test_noise = deepcopy(z_code[:20])
test_eps = deepcopy(eps_code[:20])
test_cap_pd = deepcopy(captions_ar_prezeropad[:20])
test_cap = deepcopy(captions_ar[:20])
test_mask = deepcopy(mask[:20])
test_mask = np.where(test_mask == 1, -float("inf"), 0)
#Start learning
print("batch_size: {} step_epoch : {} srong_step_epoch {}".format(
batch_size, step_epoch, wrong_step_epoch))
for epoch in range(total_epoch):
total_D_loss = 0
total_D_acc = 0
total_D_wrong_loss = 0
total_D_wrong_acc = 0
total_G_loss = 0
total_G_des_loss = 0
total_G_enc_loss = 0
print("----------------EPOCH: {} START----------------".format(epoch))
for batch in tqdm(range(step_epoch)):
image_list, captions_ar, captions_ar_prezeropad, \
z_code, eps_code, mask, keys_list, captions_label, \
real_label, fake_label = next(traingenerator)
mask = np.where(mask == 1, -float("inf"), 0)
if cfg.TREE.BRANCH_NUM == 1:
real_image = image_list[0]
if cfg.TREE.BRANCH_NUM == 2:
real_image = image_list[1]
if cfg.TREE.BRANCH_NUM == 3:
real_image = image_list[2]
#D learning
if cfg.TREE.BRANCH_NUM == 1:
fake_image = G_model.predict(
[captions_ar_prezeropad, eps_code, z_code])
else: # 2 or 3
fake_image = G_model.predict(
[captions_ar_prezeropad, eps_code, z_code, mask])
if batch % 1 == 0:
histDr = D_model.train_on_batch(
[real_image, captions_ar_prezeropad],
[real_label, real_label],
)
total_D_loss += histDr[0]
total_D_acc += (histDr[3] + histDr[4]) / 2
histDf = D_model.train_on_batch(
[fake_image, captions_ar_prezeropad],
[fake_label, fake_label],
)
total_D_loss += histDf[0]
total_D_acc += (histDf[3] + histDf[4]) / 2
if batch % wrong_step == 0:
histDw = D_model.train_on_batch(
[real_image[:-1], captions_ar_prezeropad[1:]],
[fake_label[:-1], fake_label[:-1]],
)
total_D_wrong_loss += histDw[0]
total_D_wrong_acc += (histDw[3] + histDw[4]) / 2
#G learning
if cfg.TREE.BRANCH_NUM == 1:
histGRD = GRD_model.train_on_batch(
[captions_ar_prezeropad, eps_code, z_code, captions_ar],
[real_label, real_label, captions_label],
)
else: # 2 or 3
histGRD = GRD_model.train_on_batch(
[
captions_ar_prezeropad, eps_code, z_code, mask,
captions_ar
],
[real_label, real_label, captions_label],
)
total_G_loss += histGRD[0]
total_G_des_loss += (histGRD[1] + histGRD[2]) / 2
total_G_enc_loss += histGRD[3]
#Calculation of loss
D_loss = total_D_loss / step_epoch / 2
D_acc = total_D_acc / step_epoch / 2
D_wrong_loss = total_D_wrong_loss / wrong_step_epoch
D_wrong_acc = total_D_wrong_acc / wrong_step_epoch
G_loss = total_G_loss / step_epoch
G_des_loss = total_G_des_loss / step_epoch
G_enc_loss = total_G_enc_loss / step_epoch
print(
"D_loss: {:.5f} D_wrong_loss: {:.5f} D_acc: {:.5f} D_wrong_acc: {:.5f}"
.format(D_loss, D_wrong_loss, D_acc, D_wrong_acc))
print(
"G_loss: {:.5f} G_discriminator_loss: {:.5f} G_encoder_loss: {:.5f}"
.format(G_loss, G_des_loss, G_enc_loss))
if epoch % 4 == 0:
G_save_path = "model/G_epoch{}.h5".format(epoch)
G_model.save_weights(G_save_path)
D_save_path = "model/D_epoch{}.h5".format(epoch)
D_model.save_weights(D_save_path)
#Save image
if epoch % 1 == 0:
sample_images(epoch, test_noise, test_eps, test_cap_pd, test_mask,
G_model)
def main():
#DataGenerator
trainable = cfg.Train
imsize = cfg.TREE.BASE_SIZE * (2**(cfg.TREE.BRANCH_NUM - 1)) #64, 3
image_transform = transforms.Compose([
transforms.Resize(int(imsize * 76 / 64)),
transforms.RandomCrop(imsize),
transforms.RandomHorizontalFlip()
])
cfg.DATA_DIR = "data/pascal"
dataset_train = TextDataset_Pascal(
cfg.DATA_DIR,
"train",
base_size=cfg.TREE.BASE_SIZE,
transform=image_transform)
assert dataset_train
dataset_test = TextDataset_Pascal(
cfg.DATA_DIR,
"test",
base_size=cfg.TREE.BASE_SIZE,
transform=image_transform)
assert dataset_test
traingenerator = DataGenerator(dataset_train, batchsize=cfg.TRAIN.BATCH_SIZE)
##Create model
G_model, D_model, GRD_model, CR_model, RNN_model, SIM_model = model_create_new(dataset_train)
print("loadmodel_completed")
if trainable:
# Record Tensorboard Log
sess = tf.Session()
logdir = "tensorboard/" + datetime.datetime.now().strftime("%Y%m%d-%H%M%S") + "/"
writer = tf.summary.FileWriter(logdir, sess.graph)
D_LOSS = tf.placeholder(tf.float32, [])
D_LOSS_wrong = tf.placeholder(tf.float32, [])
D_ACC_realism = tf.placeholder(tf.float32, [])
D_ACC_consistency = tf.placeholder(tf.float32, [])
D_ACC_wong_consistency = tf.placeholder(tf.float32, [])
D_ACC_wong_realism = tf.placeholder(tf.float32, [])
G_LOSS = tf.placeholder(tf.float32, [])
G_LOSS_enc = tf.placeholder(tf.float32, [])
tf.summary.scalar("D_LOSS", D_LOSS)
tf.summary.scalar("D_LOSS_wrong", D_LOSS_wrong)
tf.summary.scalar("D_ACC_realism", D_ACC_realism)
tf.summary.scalar("D_ACC_consitency", D_ACC_consistency)
tf.summary.scalar("D_ACC_wong_consistency", D_ACC_wong_consistency)
tf.summary.scalar("D_ACC_wong_realism", D_ACC_wong_realism)
tf.summary.scalar("G_LOSS", G_LOSS)
tf.summary.scalar("G_LOSS_enc", G_LOSS_enc)
merged = tf.summary.merge_all()
# Preparation for learning
total_epoch = cfg.TRAIN.MAX_EPOCH
batch_size = traingenerator.batchsize
step_epoch = int(len(dataset_train) / batch_size)
wrong_step = 1
wrong_step_epoch = int(step_epoch / wrong_step)
eval_step = 20
image_list, captions_ar, captions_ar_prezeropad, \
z_code, eps_code, mask, keys_list, captions_label, \
real_label, fake_label = next(traingenerator)
traingenerator.count = 0
# for image plot
test_noise = deepcopy(z_code[:20])
test_eps = deepcopy(eps_code[:20])
test_cap_pd = deepcopy(captions_ar_prezeropad[:20])
test_cap = deepcopy(captions_ar[:20])
test_mask = deepcopy(mask[:20])
test_mask = np.where(test_mask == 1, -float("inf"), 0)
test_real_image = deepcopy(image_list[0][:20])
# Start learning
print("batch_size: {} step_epoch : {} wrong_step_epoch {}".format(
batch_size, step_epoch, wrong_step_epoch))
for epoch in range(total_epoch):
total_D_loss = 0
total_D_acc = 0
total_D_acc_realism = 0
total_D_acc_consitency = 0
total_D_wrong_loss = 0
total_D_wrong_acc = 0
total_D_wrong_acc_realism = 0
total_D_wrong_acc_consitency = 0
total_G_loss = 0
total_G_des_loss = 0
total_G_enc_loss = 0
print("----------------EPOCH: {} START----------------".format(epoch))
for batch in tqdm(range(step_epoch)):
image_list, captions_ar, captions_ar_prezeropad, \
z_code, eps_code, mask, keys_list, captions_label, \
real_label, fake_label = next(traingenerator)
mask = np.where(mask == 1, -float("inf"), 0)
if cfg.TREE.BRANCH_NUM == 1:
real_image = image_list[0]
if cfg.TREE.BRANCH_NUM == 2:
real_image = image_list[1]
if cfg.TREE.BRANCH_NUM == 3:
real_image = image_list[2]
num_image = len(real_image)
#D learning
if cfg.TREE.BRANCH_NUM == 1:
fake_image = G_model.predict(
[captions_ar_prezeropad, eps_code, z_code])
else: # 2 or 3
fake_image = G_model.predict(
[captions_ar_prezeropad, eps_code, z_code, mask])
if batch % 1 == 0:
histDr = D_model.train_on_batch(
[real_image, captions_ar_prezeropad],
[real_label, real_label],
)
# histDr = D_model.test_on_batch(
# [real_image, captions_ar_prezeropad],
# [real_label, real_label],
# )
total_D_loss += histDr[0]
total_D_acc_realism += histDr[3]
total_D_acc_consitency += histDr[4]
total_D_acc += (histDr[3] + histDr[4]) / 2
histDf = D_model.train_on_batch(
[fake_image, captions_ar_prezeropad],
[fake_label, fake_label],
)
# histDf = D_model.test_on_batch(
# [fake_image, captions_ar_prezeropad],
# [fake_label, fake_label],
# )
total_D_loss += histDf[0]
total_D_acc += (histDf[3] + histDf[4]) / 2
if batch % wrong_step == 0:
histDw = D_model.train_on_batch(
[real_image[:-1], captions_ar_prezeropad[1:]],
[real_label[:-1], fake_label[:-1]],
)
temp2 = D_model.predict([real_image[:-1], captions_ar_prezeropad[1:]])
# histDw = D_model.test_on_batch(
# [real_image[:-1], captions_ar_prezeropad[1:]],
# [real_label[:-1], fake_label[:-1]],
# )
total_D_wrong_loss += histDw[0]
total_D_wrong_acc_realism += histDw[3]
total_D_wrong_acc_consitency += (histDw[4])
#G learning
if cfg.TREE.BRANCH_NUM == 1:
histGRD = GRD_model.train_on_batch(
[captions_ar_prezeropad, eps_code, z_code, captions_ar],
[real_label, real_label, captions_label],
)
# histGRD = GRD_model.test_on_batch(
# [captions_ar_prezeropad, eps_code, z_code, captions_ar],
# [real_label, real_label, captions_label],
# )
else: # 2 or 3
histGRD = GRD_model.train_on_batch(
[captions_ar_prezeropad, eps_code, z_code, mask, captions_ar],
[real_label, real_label, captions_label],g))
)
# histGRD = GRD_model.test_on_batch(
# [captions_ar_prezeropad, eps_code, z_code, mask, captions_ar],
# [real_label, real_label, captions_label],
# )
total_G_loss += histGRD[0]
total_G_des_loss += (histGRD[1] + histGRD[2]) / 2
total_G_enc_loss += histGRD[3]
#Calculation of loss
D_loss = total_D_loss / step_epoch / 2
D_acc = total_D_acc / step_epoch / 2
D_wrong_loss = total_D_wrong_loss / wrong_step_epoch
D_wrong_acc = total_D_wrong_acc / wrong_step_epoch
G_loss = total_G_loss / step_epoch/ (cfg.TRAIN.RNN_DEC_LOSS_W + 1)
G_des_loss = total_G_des_loss / step_epoch
G_enc_loss = total_G_enc_loss / step_epoch / cfg.TRAIN.RNN_DEC_LOSS_W
D_wrong_acc_realism = total_D_wrong_acc_realism / step_epoch
D_wrong_acc_consistency = total_D_wrong_acc_consitency / wrong_step_epoch
D_acc_realism = total_D_acc_realism / step_epoch
D_acc_consistency = total_D_acc_consitency / step_epoch
if epoch % 1 == 0:
summary = sess.run(merged, feed_dict={D_LOSS: D_loss,
D_LOSS_wrong: D_wrong_loss,
D_ACC_realism: D_acc_realism,
D_ACC_consistency: D_acc_consistency,
D_ACC_wong_consistency: D_wrong_acc_realism,
D_ACC_wong_realism: D_wrong_acc_consistency,
G_LOSS: G_loss,
G_LOSS_enc: G_enc_loss
})
writer.add_summary(summary, epoch)
print(
"\nD_loss: {:.5f} D_wrong_loss: {:.5f} D_wrong_acc_realism: {:.5f} D_wrong_acc_consistency: {:.5f} D_acc_realism: {:.5f} D_acc_consistency: {:.5f} "
.format(D_loss, D_wrong_loss, D_wrong_acc_realism, D_wrong_acc_consistency, D_acc_realism, D_acc_consistency))
print(
"G_loss: {:.5f} G_discriminator_loss: {:.5f} G_encoder_loss: {:.5f}"
.format(G_loss, G_des_loss, G_enc_loss))
