gen_optimizer.add_hook(chainer.optimizer.GradientClipping(gen_grad_clip)) dis_optimizer = optimizers.Adam(alpha=1e-4) dis_optimizer.setup(discriminator) dis_optimizer.add_hook(NamedWeightDecay(dis_l2_reg_lambda, '/out/')) if not args.gen: print('Start pre-training generator...') for epoch in range(gen_pretrain_epoch): pre_train_loss = [] for _ in range(gen_data_loader.num_batch): batch = gen_data_loader.next_batch() g_loss = generator.pretrain_step(batch) # gen_optimizer.zero_grads() gen_optimizer.reallocate_cleared_grads() g_loss.backward() gen_optimizer.update() pre_train_loss.append(g_loss.data) # print('pre-train epoch ', epoch, 'train_loss ', np.mean(pre_train_loss)) generate_samples_neg(generator, gen_batch_size, 1000, eval_file) likelihood_data_loader.create_batches(eval_file) test_loss = target_loss(sess, target_lstm, likelihood_data_loader) print('pre-train epoch: {} test loss: {} lr: {}'.format(epoch, test_loss, gen_optimizer.lr)) test_count += 1 summary = sess.run(target_loss_summary, feed_dict={loss_: test_loss}) summary_writer.add_summary(summary, test_count)
pre_train_loss = [] perm = np.random.permutation(train_num) for i in range(0, train_num, batch_size): batch = train_comment_data[perm[i:i+batch_size]] tag_batch = train_tag_data[perm[i:i + batch_size]] if args.ae_pretrain: g_loss = generator.pretrain_step_autoencoder(batch) enc_optimizer.zero_grads() gen_optimizer.zero_grads() g_loss.backward() enc_optimizer.update() gen_optimizer.update() pre_train_loss.append(float(g_loss.data)) else: g_loss = generator.pretrain_step(batch, tag_batch) gen_optimizer.zero_grads() g_loss.backward() gen_optimizer.update() pre_train_loss.append(float(g_loss.data)) # progress report gen_train_count += 1 progress_report(gen_train_count, start, batch_size) # test test_loss = [] perm = np.random.permutation(test_num) for i in range(0, test_num, batch_size): batch = test_comment_data[perm[i:i + batch_size]]