def initialize_networks(args, device):
# network
En_A = networks.encoder(in_nc=args.in_ngc,
nf=args.ngf,
img_size=args.img_size).to(device)
En_B = networks.encoder(in_nc=args.in_ngc,
nf=args.ngf,
img_size=args.img_size).to(device)
De_A = networks.decoder(out_nc=args.out_ngc, nf=args.ngf).to(device)
De_B = networks.decoder(out_nc=args.out_ngc, nf=args.ngf).to(device)
Disc_A = networks.discriminator(in_nc=args.in_ndc,
out_nc=args.out_ndc,
nf=args.ndf,
img_size=args.img_size).to(device)
Disc_B = networks.discriminator(in_nc=args.in_ndc,
out_nc=args.out_ndc,
nf=args.ndf,
img_size=args.img_size).to(device)
print('---------- Networks initialized -------------')
utils.print_network(En_A)
utils.print_network(En_B)
utils.print_network(De_A)
utils.print_network(De_B)
utils.print_network(Disc_A)
utils.print_network(Disc_B)
print('-----------------------------------------------')
all_networks = [En_A, En_B, De_A, De_B, Disc_A, Disc_B]
return all_networks
def test_discriminator_invalid_input(self):
wrong_dim_input = tf.zeros([5, 32, 32])
with self.assertRaisesRegexp(ValueError, 'Shape must be rank 4'):
networks.discriminator(wrong_dim_input)
not_fully_defined = tf.placeholder(tf.float32, [3, None, 32, 3])
with self.assertRaisesRegexp(ValueError, 'Shape .* is not fully defined'):
networks.compression_model(not_fully_defined)
def get_GAN(shape,discriminator,generator,optimizer):
discriminator.trainable = False
inp = Input(shape=shape)
x = generator(inp)
out = discriminator(x)
GAN = Model(inputs=inp,outputs=[x,out])
GAN.compile(loss=[vgg_loss,'binary_crossentropy'],loss_weights=[1,1e-3],optimizer=optimizer)
return GAN
def get_discr_pred(self, inputs, scope, reuse=False):
"""Build and get Dsicriminator preds.
Based on ICML paper
"""
with tf.variable_scope('discriminator/' + scope, reuse=reuse):
logits, probs = discriminator(self.opts, inputs)
clipped = tf.clip_by_value(probs, 1e-6, 1 - 1e-6)
return logits, clipped
def _discriminator_fn(x):
"""Builds discriminator network."""
return networks.discriminator(x,
progress,
_num_filters_fn,
resolution_schedule,
num_blocks=num_blocks,
kernel_size=kernel_size)
def train():
real_img = tf.placeholder(tf.float32, [None, H, W, 3])
label = tf.placeholder(tf.int32, [None])
z = tf.placeholder(tf.float32, [None, 100])
one_hot_label = tf.one_hot(label, NUMS_CLASS)
labeled_z = tf.concat([z, one_hot_label], axis=1)
G = generator("generator")
D = discriminator("discriminator")
fake_img = G(labeled_z)
class_fake_logits, adv_fake_logits = D(fake_img, NUMS_CLASS)
class_real_logits, adv_real_logits = D(real_img, NUMS_CLASS)
loss_d_real = -tf.reduce_mean(tf.log(adv_real_logits + EPSILON))
loss_d_fake = -tf.reduce_mean(tf.log(1 - adv_fake_logits + EPSILON))
loss_cls_real = -tf.reduce_mean(
tf.log(
tf.reduce_sum(class_real_logits * one_hot_label, axis=1) +
EPSILON))
loss_cls_fake = -tf.reduce_mean(
tf.log(
tf.reduce_sum(class_fake_logits * one_hot_label, axis=1) +
EPSILON))
D_loss = loss_d_real + loss_d_fake + loss_cls_real
G_loss = -tf.reduce_mean(tf.log(adv_fake_logits + EPSILON)) + loss_cls_fake
D_opt = tf.train.AdamOptimizer(2e-4,
beta1=0.5).minimize(D_loss,
var_list=D.var_list())
G_opt = tf.train.AdamOptimizer(2e-4,
beta1=0.5).minimize(G_loss,
var_list=G.var_list())
sess = tf.Session()
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver()
data, labels = read_face_data("./dataset/face_woman_man.mat")
for i in range(50000):
s = time.time()
for j in range(1):
BATCH, LABELS, Z = get_batch_face(data, labels, BATCHSIZE)
BATCH = BATCH / 127.5 - 1.0
sess.run(D_opt, feed_dict={real_img: BATCH, label: LABELS, z: Z})
sess.run(G_opt, feed_dict={real_img: BATCH, label: LABELS, z: Z})
e = time.time()
if i % 100 == 0:
[D_LOSS, G_LOSS, FAKE_IMG] = sess.run([D_loss, G_loss, fake_img],
feed_dict={
real_img: BATCH,
label: LABELS,
z: Z
})
Image.fromarray(np.uint8((FAKE_IMG[0, :, :, :] + 1) *
127.5)).save("./results/" + str(i) + "_" +
str(int(LABELS[0])) + ".jpg")
print("Iteration: %d, D_loss: %f, G_loss: %f, update_time: %f" %
(i, D_LOSS, G_LOSS, e - s))
if i % 500 == 0:
saver.save(sess, "./save_para/model.ckpt")
pass
def gradient_penalty(real_img, fake_image, scope='discriminator'):
difference = real_img-fake_image
alpha = tf.random_uniform(shape=[1,1,1,1], minval=0., maxval=1.)
interpolates = real_img+alpha*difference
gradients = tf.gradients(discriminator(interpolates, reuse=True, scope=scope), [interpolates])[0]
slopes = tf.sqrt(tf.reduce_sum(tf.square(gradients),reduction_indices=[1,2,3]))
return tf.reduce_mean((slopes-1.)**2)
def test_discriminator(self):
batch_size = 5
image = tf.random_uniform([batch_size, 32, 32, 3], -1, 1)
dis_output = networks.discriminator(image, None)
with self.test_session(use_gpu=True) as sess:
sess.run(tf.global_variables_initializer())
dis_output_np = dis_output.eval()
self.assertAllEqual([batch_size, 1], dis_output_np.shape)
def _discriminator_fn(x):
"""Builds discriminator network."""
return networks.discriminator(
x,
progress,
_num_filters_fn,
resolution_schedule,
num_blocks=num_blocks,
kernel_size=kernel_size)
def test_discriminator_graph(self):
# Check graph construction for a number of image size/depths and batch
# sizes.
for batch_size, patch_size in zip([3, 6], [70, 128]):
tf.reset_default_graph()
img = tf.ones([batch_size, patch_size, patch_size, 3])
disc_output = networks.discriminator(img)
self.assertEqual(2, disc_output.shape.ndims)
self.assertEqual(batch_size, disc_output.shape.as_list()[0])
def test_discriminator_graph(self):
# Check graph construction for a number of image size/depths and batch
# sizes.
for batch_size, patch_size in zip([3, 6], [70, 128]):
tf.reset_default_graph()
img = tf.ones([batch_size, patch_size, patch_size, 3])
disc_output = networks.discriminator(img)
self.assertEqual(2, disc_output.shape.ndims)
self.assertEqual(batch_size, disc_output.shape[0])
def train(epochs=1,batch_size=128):
if(not os.path.isdir('/content/output/')):
os.mkdir('/content/output/')
downscale_factor = 4
batch_count = int(x_train_hr.shape[0] / batch_size)
shape = (img_shape[0]//downscale_factor, img_shape[1]//downscale_factor, img_shape[2])
gen = generator(shape).generator_model()
disc = discriminator(img_shape).discriminator_model()
adam = Adam(lr=1E-4, beta_1=0.9, beta_2=0.999, epsilon=1e-08)
gen.compile(loss=vgg_loss, optimizer=adam)
disc.compile(loss="binary_crossentropy", optimizer=adam)
shape = (img_shape[0]//downscale_factor, img_shape[1]//downscale_factor, 3)
gan = get_GAN(shape, disc, gen, adam)
for e in range(1, epochs+1):
print ('-'*15, 'Epoch %d' % e, '-'*15)
for _ in range(batch_count):
rand_nums = np.random.randint(0, x_train_hr.shape[0], size=batch_size)
image_batch_hr = x_train_hr[rand_nums]
image_batch_lr = x_train_lr[rand_nums]
generated_images_sr = gen.predict(image_batch_lr)
real_data_Y = np.ones(batch_size) - np.random.random_sample(batch_size)*0.2
fake_data_Y = np.random.random_sample(batch_size)*0.2
disc.trainable = True
d_loss_real = disc.train_on_batch(image_batch_hr, real_data_Y)
d_loss_fake = disc.train_on_batch(generated_images_sr, fake_data_Y)
#d_loss = 0.5 * np.add(d_loss_fake, d_loss_real)
rand_nums = np.random.randint(0, x_train_hr.shape[0], size=batch_size)
image_batch_hr = x_train_hr[rand_nums]
image_batch_lr = x_train_lr[rand_nums]
gan_Y = np.ones(batch_size) - np.random.random_sample(batch_size)*0.2
disc.trainable = False
loss_gan = gan.train_on_batch(image_batch_lr, [image_batch_hr,gan_Y])
print("Loss HR , Loss LR, Loss GAN")
print(d_loss_real, d_loss_fake, loss_gan)
if e == 1 or e % 5 == 0:
plot_generated_images(e, gen)
if e % 50 == 0:
generator.save('./output/gen_model%d.h5' % e)
discriminator.save('./output/dis_model%d.h5' % e)
gan.save('./output/gan_model%d.h5' % e)
train(20000,4)
def train(dataset, gpu_id):
params = param.getGeneralParams()
gpu = '/gpu:' + str(gpu_id)
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
config.allow_soft_placement = True
set_session(tf.Session(config=config))
with tf.device(gpu):
vgg_model = myVGG.vgg_norm()
networks.make_trainable(vgg_model, False)
response_weights = sio.loadmat('mean_response.mat')
fgbg = networks.network_fgbg(params, vgg_model, response_weights)
#fgbg.load_weights('../results/networks/fgbg_vgg/140000.h5')
disc = networks.discriminator(params)
gan = networks.gan(fgbg, disc, params, vgg_model, response_weights,
0.01, 1e-4)
gan.load_weights('../results/networks/fgbg_gan/2000.h5')
outputs = [fgbg.outputs[0]]
#outputs.append(fgbg.get_layer('mask_src').output)
#outputs.append(fgbg.get_layer('fg_stack').output)
#outputs.append(fgbg.get_layer('bg_src').output)
#outputs.append(fgbg.get_layer('bg_tgt').output)
#outputs.append(fgbg.get_layer('fg_tgt').output)
outputs.append(fgbg.get_layer('fg_mask_tgt').output)
model = Model(fgbg.inputs, outputs)
test = datareader.makeActionExampleList('test_vids.txt', 1)
feed = datageneration.warpExampleGenerator(test,
params,
do_augment=False,
return_pose_vectors=True)
n_frames = len(test)
true_action = np.zeros((256, 256, 3, n_frames))
pred_action = np.zeros((256, 256, 3, n_frames))
mask = np.zeros((256, 256, 1, n_frames))
for i in xrange(n_frames):
print i
X, Y = next(feed)
pred = model.predict(X[:-2])
true_action[:, :, :, i] = convert(np.reshape(Y, (256, 256, 3)))
pred_action[:, :, :, i] = convert(np.reshape(pred[0], (256, 256, 3)))
mask[:, :, :, i] = pred[1]
sio.savemat('results/action/1_gan.mat', {
'true': true_action,
'pred': pred_action,
'mask': mask
})
def test_discriminator_grad_norm_progress(self):
stable_stage_num_images = 2
transition_stage_num_images = 3
current_image_id_ph = tf.placeholder(tf.int32, [])
progress = networks.compute_progress(current_image_id_ph,
stable_stage_num_images,
transition_stage_num_images,
num_blocks=3)
x = tf.random_normal([2, 16, 16, 3])
logits, _ = networks.discriminator(
x, progress, _num_filters_stub,
networks.ResolutionSchedule(start_resolutions=(4, 4),
scale_base=2,
num_resolutions=3))
fake_loss = tf.reduce_sum(tf.square(logits))
grad_norms = [
_get_grad_norm(
fake_loss,
tf.trainable_variables('.*/progressive_gan_block_1/.*')),
_get_grad_norm(
fake_loss,
tf.trainable_variables('.*/progressive_gan_block_2/.*')),
_get_grad_norm(
fake_loss,
tf.trainable_variables('.*/progressive_gan_block_3/.*'))
]
grad_norms_output = None
with self.test_session(use_gpu=True) as sess:
sess.run(tf.global_variables_initializer())
grad_norms_output = np.array([
sess.run(grad_norms, feed_dict={current_image_id_ph: i})
for i in range(15) # total num of images
])
# The gradient of block_1 is always on.
self.assertEqual(
np.argmax(grad_norms_output[:, 0] > 0), 0,
'gradient norms {} for block 1 is not always on'.format(
grad_norms_output[:, 0]))
# The gradient of block_2 is on after 1 stable stage.
self.assertEqual(
np.argmax(grad_norms_output[:, 1] > 0), 3,
'gradient norms {} for block 2 is not on at step 3'.format(
grad_norms_output[:, 1]))
# The gradient of block_3 is on after 2 stable stage + 1 transition stage.
self.assertEqual(
np.argmax(grad_norms_output[:, 2] > 0), 8,
'gradient norms {} for block 3 is not on at step 8'.format(
grad_norms_output[:, 2]))
def get_discr_pred(self, inputs, scope, reuse=False):
"""Build and get Discriminator preds.
Based on ICML paper
"""
with tf.variable_scope('discriminator/' + scope, reuse=reuse):
logits, probs = discriminator(self.opts, inputs)
"""
if scope == 'TC':
logits, probs = discriminator(self.opts, inputs)
elif scope == 'dimwise':
logits, probs = dimwise_discriminator(self.opts, inputs)
else:
raise Exception('Unknown {} scope for Discriminator'.format(scope))
"""
clipped = tf.clip_by_value(probs, 1e-6, 1 - 1e-6)
return logits, clipped
def discriminator_fn_specgram(images, **kwargs):
"""Builds discriminator network."""
shape = images.shape
normalizer = data_normalizer.registry[kwargs['data_normalizer']](kwargs)
images = normalizer.normalize_op(images)
images.set_shape(shape)
logits, end_points = networks.discriminator(
images,
kwargs['progress'],
lambda block_id: _num_filters_fn(block_id, **kwargs),
kwargs['resolution_schedule'],
num_blocks=kwargs['num_blocks'],
kernel_size=kwargs['kernel_size'],
simple_arch=kwargs['simple_arch'])
with tf.variable_scope('discriminator_cond'):
x = tf.layers.flatten(end_points['last_conv'])
end_points['classification_logits'] = layers.custom_dense(
x=x, units=kwargs['num_tokens'], scope='classification_logits')
return logits, end_points
def __init__(self, options):
super(ArtGAN, self).__init__()
# build model
self.encoder = encoder(options)
self.decoder = decoder(options)
self.discriminator = discriminator(options)
self.discriminator_weight = {
"pred_1": 1.,
"pred_2": 1.,
"pred_4": 1.,
"pred_6": 1.,
"pred_7": 1.
}
self.loss = nn.BCEWithLogitsLoss(reduction='mean')
self.mse = nn.MSELoss(reduction='mean')
self.abs = nn.L1Loss(reduction='mean')
# Setup the optimizers
dis_params = list(self.discriminator.parameters())
gen_params = list(self.encoder.parameters()) + list(
self.decoder.parameters())
self.dis_opt = torch.optim.Adam(
[p for p in dis_params if p.requires_grad],
lr=options.lr,
betas=(0.5, 0.999),
weight_decay=0.0001,
amsgrad=True)
self.gen_opt = torch.optim.Adam(
[p for p in gen_params if p.requires_grad],
lr=options.lr,
betas=(0.5, 0.999),
weight_decay=0.0001,
amsgrad=True)
self.dis_scheduler = get_scheduler(self.dis_opt, options)
self.gen_scheduler = get_scheduler(self.gen_opt, options)
# Network weight initialization
self.apply(weights_init(options.init))
self.discriminator.apply(weights_init('gaussian'))
self.gener_loss = torch.tensor(0.)
self.discr_loss = torch.tensor(0.)
def __init__(self, opts):
super(DivCo_DCGAN, self).__init__()
# parameters
lr = 0.0002
self.nz = opts.nz
self.opt = opts
self.class_num = opts.class_num
self.G = networks.generator(opts)
self.D = networks.discriminator(opts)
self.gen_opt = torch.optim.Adam(self.G.parameters(),
lr=lr,
betas=(0.5, 0.999),
weight_decay=0.0001)
self.dis_opt = torch.optim.Adam(self.D.parameters(),
lr=lr,
betas=(0.5, 0.999),
weight_decay=0.0001)
self.BCE_loss = torch.nn.BCELoss()
self.cross_entropy_loss = torch.nn.CrossEntropyLoss()
def __init__(self):
# Paper: Context Encoders: Feature Learning by Inpainting
self.inputs = tf.placeholder(tf.float32, [None, IMG_H, IMG_W, IMG_C])
self.patch = tf.placeholder(tf.float32, [None, MASK_H, MASK_W, IMG_C])
self.train_phase = tf.placeholder(tf.bool)
G = generator("generator")
D = discriminator("discriminator")
self.patch_fake = G(self.inputs, self.train_phase)
self.fake_logits = D(self.patch_fake, self.train_phase)
self.real_logits = D(self.patch, self.train_phase)
self.D_loss = -tf.reduce_mean(
tf.log(self.real_logits + EPSILON) +
tf.log(1 - self.fake_logits + EPSILON))
self.G_loss = -tf.reduce_mean(
tf.log(self.fake_logits + EPSILON)) + 100 * tf.reduce_mean(
tf.reduce_sum(tf.square(self.patch - self.patch_fake),
[1, 2, 3]))
self.D_Opt = tf.train.AdamOptimizer(2e-4).minimize(
self.D_loss, var_list=D.get_var())
self.G_Opt = tf.train.AdamOptimizer(2e-4).minimize(
self.G_loss, var_list=G.get_var())
self.sess = tf.Session()
self.sess.run(tf.global_variables_initializer())
def test_discriminator_invalid_input(self):
with self.assertRaisesRegexp(ValueError, 'Shape must be rank 4'):
networks.discriminator(tf.zeros([28, 28, 3]))
def test_discriminator_run(self):
img_batch = tf.zeros([3, 70, 70, 3])
disc_output = networks.discriminator(img_batch)
with self.test_session() as sess:
sess.run(tf.global_variables_initializer())
sess.run(disc_output)
# network
G = networks.generator(args.in_ngc, args.out_ngc, args.ngf, args.nb)
if args.latest_generator_model != '':
if torch.cuda.is_available():
G.load_state_dict(torch.load(args.latest_generator_model))
else:
# cpu mode
G.load_state_dict(
torch.load(args.latest_generator_model,
map_location=lambda storage, loc: storage))
G.to(device)
G.train()
D = []
for i in range(len(targets_dir)):
tmpD = networks.discriminator(args.in_ndc, args.out_ndc, args.ndf)
if args.latest_discriminator_model != '':
if torch.cuda.is_available():
tmpD.load_state_dict(
torch.load(targets_dir[i] + args.latest_discriminator_model))
else:
tmpD.load_state_dict(
torch.load(targets_dir[i] + args.latest_discriminator_model,
map_location=lambda storage, loc: storage))
tmpD.to(device)
tmpD.train()
D.append(tmpD)
VGG = networks.VGG19(init_weights=args.vgg_model, feature_mode=True)
VGG.to(device)
VGG.eval()
def main():
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
if torch.backends.cudnn.enabled:
torch.backends.cudnn.benchmark = True
prepare_result()
make_edge_promoting_img()
# data_loader
src_transform = transforms.Compose([
transforms.Resize((args.input_size, args.input_size)),
transforms.ToTensor(),
transforms.Normalize(mean=(0.5, 0.5, 0.5), std=(0.5, 0.5, 0.5))
])
tgt_transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize(mean=(0.5, 0.5, 0.5), std=(0.5, 0.5, 0.5))
])
train_loader_src = utils.data_load(os.path.join('data', args.src_data),
'train',
src_transform,
args.batch_size,
shuffle=True,
drop_last=True)
train_loader_tgt = utils.data_load(os.path.join('data', args.tgt_data),
'pair',
tgt_transform,
args.batch_size,
shuffle=True,
drop_last=True)
test_loader_src = utils.data_load(os.path.join('data', args.src_data),
'test',
src_transform,
1,
shuffle=True,
drop_last=True)
# network
G = networks.generator(args.in_ngc, args.out_ngc, args.ngf, args.nb)
if args.latest_generator_model != '':
if torch.cuda.is_available():
G.load_state_dict(torch.load(args.latest_generator_model))
else:
# cpu mode
G.load_state_dict(
torch.load(args.latest_generator_model,
map_location=lambda storage, loc: storage))
D = networks.discriminator(args.in_ndc, args.out_ndc, args.ndf)
if args.latest_discriminator_model != '':
if torch.cuda.is_available():
D.load_state_dict(torch.load(args.latest_discriminator_model))
else:
D.load_state_dict(
torch.load(args.latest_discriminator_model,
map_location=lambda storage, loc: storage))
VGG = networks.VGG19(init_weights=args.vgg_model, feature_mode=True)
G.to(device)
D.to(device)
VGG.to(device)
G.train()
D.train()
VGG.eval()
print('---------- Networks initialized -------------')
utils.print_network(G)
utils.print_network(D)
utils.print_network(VGG)
print('-----------------------------------------------')
# loss
BCE_loss = nn.BCELoss().to(device)
L1_loss = nn.L1Loss().to(device)
# Adam optimizer
G_optimizer = optim.Adam(G.parameters(),
lr=args.lrG,
betas=(args.beta1, args.beta2))
D_optimizer = optim.Adam(D.parameters(),
lr=args.lrD,
betas=(args.beta1, args.beta2))
G_scheduler = optim.lr_scheduler.MultiStepLR(
optimizer=G_optimizer,
milestones=[args.train_epoch // 2, args.train_epoch // 4 * 3],
gamma=0.1)
D_scheduler = optim.lr_scheduler.MultiStepLR(
optimizer=D_optimizer,
milestones=[args.train_epoch // 2, args.train_epoch // 4 * 3],
gamma=0.1)
pre_train_hist = {}
pre_train_hist['Recon_loss'] = []
pre_train_hist['per_epoch_time'] = []
pre_train_hist['total_time'] = []
""" Pre-train reconstruction """
if args.latest_generator_model == '':
print('Pre-training start!')
start_time = time.time()
for epoch in range(args.pre_train_epoch):
epoch_start_time = time.time()
Recon_losses = []
for x, _ in train_loader_src:
x = x.to(device)
# train generator G
G_optimizer.zero_grad()
x_feature = VGG((x + 1) / 2)
G_ = G(x)
G_feature = VGG((G_ + 1) / 2)
Recon_loss = 10 * L1_loss(G_feature, x_feature.detach())
Recon_losses.append(Recon_loss.item())
pre_train_hist['Recon_loss'].append(Recon_loss.item())
Recon_loss.backward()
G_optimizer.step()
per_epoch_time = time.time() - epoch_start_time
pre_train_hist['per_epoch_time'].append(per_epoch_time)
print('[%d/%d] - time: %.2f, Recon loss: %.3f' %
((epoch + 1), args.pre_train_epoch, per_epoch_time,
torch.mean(torch.FloatTensor(Recon_losses))))
total_time = time.time() - start_time
pre_train_hist['total_time'].append(total_time)
with open(os.path.join(args.name + '_results', 'pre_train_hist.pkl'),
'wb') as f:
pickle.dump(pre_train_hist, f)
with torch.no_grad():
G.eval()
for n, (x, _) in enumerate(train_loader_src):
x = x.to(device)
G_recon = G(x)
result = torch.cat((x[0], G_recon[0]), 2)
path = os.path.join(
args.name + '_results', 'Reconstruction',
args.name + '_train_recon_' + str(n + 1) + '.png')
plt.imsave(path,
(result.cpu().numpy().transpose(1, 2, 0) + 1) / 2)
if n == 4:
break
for n, (x, _) in enumerate(test_loader_src):
x = x.to(device)
G_recon = G(x)
result = torch.cat((x[0], G_recon[0]), 2)
path = os.path.join(
args.name + '_results', 'Reconstruction',
args.name + '_test_recon_' + str(n + 1) + '.png')
plt.imsave(path,
(result.cpu().numpy().transpose(1, 2, 0) + 1) / 2)
if n == 4:
break
else:
print('Load the latest generator model, no need to pre-train')
train_hist = {}
train_hist['Disc_loss'] = []
train_hist['Gen_loss'] = []
train_hist['Con_loss'] = []
train_hist['per_epoch_time'] = []
train_hist['total_time'] = []
print('training start!')
start_time = time.time()
real = torch.ones(args.batch_size, 1, args.input_size // 4,
args.input_size // 4).to(device)
fake = torch.zeros(args.batch_size, 1, args.input_size // 4,
args.input_size // 4).to(device)
for epoch in range(args.train_epoch):
epoch_start_time = time.time()
G.train()
Disc_losses = []
Gen_losses = []
Con_losses = []
for (x, _), (y, _) in zip(train_loader_src, train_loader_tgt):
e = y[:, :, :, args.input_size:]
y = y[:, :, :, :args.input_size]
x, y, e = x.to(device), y.to(device), e.to(device)
# train D
D_optimizer.zero_grad()
D_real = D(y)
D_real_loss = BCE_loss(D_real, real)
G_ = G(x)
D_fake = D(G_)
D_fake_loss = BCE_loss(D_fake, fake)
D_edge = D(e)
D_edge_loss = BCE_loss(D_edge, fake)
Disc_loss = D_real_loss + D_fake_loss + D_edge_loss
Disc_losses.append(Disc_loss.item())
train_hist['Disc_loss'].append(Disc_loss.item())
Disc_loss.backward()
D_optimizer.step()
# train G
G_optimizer.zero_grad()
G_ = G(x)
D_fake = D(G_)
D_fake_loss = BCE_loss(D_fake, real)
x_feature = VGG((x + 1) / 2)
G_feature = VGG((G_ + 1) / 2)
Con_loss = args.con_lambda * L1_loss(G_feature, x_feature.detach())
Gen_loss = D_fake_loss + Con_loss
Gen_losses.append(D_fake_loss.item())
train_hist['Gen_loss'].append(D_fake_loss.item())
Con_losses.append(Con_loss.item())
train_hist['Con_loss'].append(Con_loss.item())
Gen_loss.backward()
G_optimizer.step()
G_scheduler.step()
D_scheduler.step()
per_epoch_time = time.time() - epoch_start_time
train_hist['per_epoch_time'].append(per_epoch_time)
print(
'[%d/%d] - time: %.2f, Disc loss: %.3f, Gen loss: %.3f, Con loss: %.3f'
% ((epoch + 1), args.train_epoch, per_epoch_time,
torch.mean(torch.FloatTensor(Disc_losses)),
torch.mean(torch.FloatTensor(Gen_losses)),
torch.mean(torch.FloatTensor(Con_losses))))
if epoch % 2 == 1 or epoch == args.train_epoch - 1:
with torch.no_grad():
G.eval()
for n, (x, _) in enumerate(train_loader_src):
x = x.to(device)
G_recon = G(x)
result = torch.cat((x[0], G_recon[0]), 2)
path = os.path.join(
args.name + '_results', 'Transfer',
str(epoch + 1) + '_epoch_' + args.name + '_train_' +
str(n + 1) + '.png')
plt.imsave(path,
(result.cpu().numpy().transpose(1, 2, 0) + 1) /
2)
if n == 4:
break
for n, (x, _) in enumerate(test_loader_src):
x = x.to(device)
G_recon = G(x)
result = torch.cat((x[0], G_recon[0]), 2)
path = os.path.join(
args.name + '_results', 'Transfer',
str(epoch + 1) + '_epoch_' + args.name + '_test_' +
str(n + 1) + '.png')
plt.imsave(path,
(result.cpu().numpy().transpose(1, 2, 0) + 1) /
2)
if n == 4:
break
torch.save(
G.state_dict(),
os.path.join(args.name + '_results',
'generator_latest.pkl'))
torch.save(
D.state_dict(),
os.path.join(args.name + '_results',
'discriminator_latest.pkl'))
total_time = time.time() - start_time
train_hist['total_time'].append(total_time)
print("Avg one epoch time: %.2f, total %d epochs time: %.2f" %
(torch.mean(torch.FloatTensor(
train_hist['per_epoch_time'])), args.train_epoch, total_time))
print("Training finish!... save training results")
torch.save(G.state_dict(),
os.path.join(args.name + '_results', 'generator_param.pkl'))
torch.save(D.state_dict(),
os.path.join(args.name + '_results', 'discriminator_param.pkl'))
with open(os.path.join(args.name + '_results', 'train_hist.pkl'),
'wb') as f:
pickle.dump(train_hist, f)
def test_discriminator_run(self):
img_batch = tf.zeros([3, 70, 70, 3])
disc_output = networks.discriminator(img_batch)
with self.test_session() as sess:
sess.run(tf.global_variables_initializer())
sess.run(disc_output)
def train(model_name, gpu_id):
params = param.get_general_params()
network_dir = params['model_save_dir'] + '/' + model_name
if not os.path.isdir(network_dir):
os.mkdir(network_dir)
train_feed = data_generation.create_feed(params, params['data_dir'], 'train')
os.environ["CUDA_VISIBLE_DEVICES"] = str(gpu_id)
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
config.allow_soft_placement = True
gan_lr = 1e-4
disc_lr = 1e-4
disc_loss = 0.1
generator = networks.network_posewarp(params)
generator.load_weights('../models/vgg_100000.h5')
discriminator = networks.discriminator(params)
discriminator.compile(loss='binary_crossentropy', optimizer=Adam(lr=disc_lr))
vgg_model = truncated_vgg.vgg_norm()
networks.make_trainable(vgg_model, False)
response_weights = sio.loadmat('../data/vgg_activation_distribution_train.mat')
gan = networks.gan(generator, discriminator, params)
gan.compile(optimizer=Adam(lr=gan_lr),
loss=[networks.vgg_loss(vgg_model, response_weights, 12), 'binary_crossentropy'],
loss_weights=[1.0, disc_loss])
n_iters = 10000
batch_size = params['batch_size']
for step in range(n_iters):
x, y = next(train_feed)
gen = generator.predict(x)
# Train discriminator
x_tgt_img_disc = np.concatenate((y, gen))
x_src_pose_disc = np.concatenate((x[1], x[1]))
x_tgt_pose_disc = np.concatenate((x[2], x[2]))
L = np.zeros([2 * batch_size])
L[0:batch_size] = 1
inputs = [x_tgt_img_disc, x_src_pose_disc, x_tgt_pose_disc]
d_loss = discriminator.train_on_batch(inputs, L)
# Train the discriminator a couple of iterations before starting the gan
if step < 5:
util.printProgress(step, 0, [0, d_loss])
step += 1
continue
# TRAIN GAN
L = np.ones([batch_size])
x, y = next(train_feed)
g_loss = gan.train_on_batch(x, [y, L])
util.printProgress(step, 0, [g_loss[1], d_loss])
if step % params['model_save_interval'] == 0 and step > 0:
gan.save(network_dir + '/' + str(step) + '.h5')
def train(model_name, gpu_id):
params = param.getGeneralParams()
gpu = '/gpu:' + str(gpu_id)
network_dir = params['project_dir'] + '/results/networks/' + model_name
if not os.path.isdir(network_dir):
os.mkdir(network_dir)
train_feed = datageneration.createFeed(params, "train_vids.txt", 50000)
test_feed = datageneration.createFeed(params, "test_vids.txt", 5000)
batch_size = params['batch_size']
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
config.allow_soft_placement = True
set_session(tf.Session(config=config))
gan_lr = 5e-5
disc_lr = 5e-5
disc_loss = 0.1
vgg_model_num = 184000
with tf.device(gpu):
vgg_model = myVGG.vgg_norm()
networks.make_trainable(vgg_model, False)
response_weights = sio.loadmat('mean_response.mat')
generator = networks.network_fgbg(params, vgg_model, response_weights)
generator.load_weights('../results/networks/fgbg_vgg_new/' +
str(vgg_model_num) + '.h5')
discriminator = networks.discriminator(params)
discriminator.compile(loss=networks.wass, optimizer=RMSprop(disc_lr))
gan = networks.gan(generator, discriminator, params, vgg_model,
response_weights, disc_loss, gan_lr)
for step in xrange(vgg_model_num + 1, vgg_model_num + 5001):
for j in xrange(2):
for l in discriminator.layers:
weights = l.get_weights()
weights = [np.clip(w, -0.01, 0.01) for w in weights]
l.set_weights(weights)
X, Y = next(train_feed)
with tf.device(gpu):
gen = generator.predict(X)
#Train discriminator
networks.make_trainable(discriminator, True)
X_tgt_img_disc = np.concatenate((Y, gen))
X_src_pose_disc = np.concatenate((X[1], X[1]))
X_tgt_pose_disc = np.concatenate((X[2], X[2]))
L = np.ones(2 * batch_size)
L[0:batch_size] = -1
inputs = [X_tgt_img_disc, X_src_pose_disc, X_tgt_pose_disc]
d_loss = discriminator.train_on_batch(inputs, L)
networks.make_trainable(discriminator, False)
#TRAIN GAN
L = -1 * np.ones(batch_size)
X, Y = next(train_feed)
g_loss = gan.train_on_batch(X, [Y, L])
util.printProgress(step, 0, [g_loss[1], d_loss])
if (step % params['model_save_interval'] == 0):
gan.save(network_dir + '/' + str(step) + '.h5')
'''
##########################
##########################
# get hand written data #
print("generating stylization set.")
style_base = "stylization_set/another/"
style_set = get_style_set(style_base)
##########################
print("initializing model.")
NUM_EPOCH = 200 ################
learning_rate = 0.0003
s_enc = styleEncoder().cuda()
dec = Decoder().cuda()
dis = discriminator().cuda()
enc = fontEncoder().cuda()
cla = classifier().cuda()
s_MSE = nn.MSELoss().cuda()
adv_loss = nn.BCELoss().cuda()
cla_loss = nn.CrossEntropyLoss().cuda()
gen_loss = nn.L1Loss().cuda()
D_optimizer = torch.optim.Adam(dis.parameters(), lr=learning_rate)
G_optimizer = torch.optim.Adam(list(s_enc.parameters()) +
list(dec.parameters()) +
list(enc.parameters()) + list(cla.parameters()),
lr=learning_rate)
for epoch in range(NUM_EPOCH):
def train(model_name, gpu_id):
params = param.getGeneralParams()
gpu = '/gpu:' + str(gpu_id)
network_dir = params['project_dir'] + '/results/networks/' + model_name
if not os.path.isdir(network_dir):
os.mkdir(network_dir)
train_feed = datageneration.createFeed(params, "train_vids.txt")
test_feed = datageneration.createFeed(params, "test_vids.txt")
batch_size = params['batch_size']
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
config.allow_soft_placement = True
with tf.Session(config=config) as sess:
sess.run(tf.global_variables_initializer())
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord)
gan_lr = 1e-4
disc_lr = 1e-4
disc_loss = 0.1
with tf.device(gpu):
vgg_model = myVGG.vgg_norm()
networks.make_trainable(vgg_model, False)
response_weights = sio.loadmat('mean_response.mat')
#generator = networks.network_pix2pix(params,vgg_model,response_weights)
generator = networks.network_fgbg(params)
generator.load_weights('../results/networks/fgbg_vgg/100000.h5')
discriminator = networks.discriminator(params)
discriminator.compile(loss='binary_crossentropy',
optimizer=Adam(lr=disc_lr))
gan = networks.gan(generator, discriminator, params, vgg_model,
response_weights, disc_loss, gan_lr)
gan.compile(optimizer=Adam(lr=gan_lr),
loss=[
networks.vggLoss(vgg_model, response_weights),
'binary_crossentropy'
],
loss_weights=[1.0, disc_loss])
for step in xrange(10001):
X, Y = next(train_feed)
with tf.device(gpu):
gen = generator.predict(X) #[0:3])
#Train discriminator
X_tgt_img_disc = np.concatenate((Y, gen))
X_src_pose_disc = np.concatenate((X[1], X[1]))
X_tgt_pose_disc = np.concatenate((X[2], X[2]))
L = np.zeros([2 * batch_size])
L[0:batch_size] = 1
inputs = [X_tgt_img_disc, X_src_pose_disc, X_tgt_pose_disc]
d_loss = discriminator.train_on_batch(inputs, L)
#Train the discriminator a couple of iterations before starting the gan
if (step < 5):
util.printProgress(step, 0, [0, d_loss])
step += 1
continue
#TRAIN GAN
L = np.ones([batch_size])
X, Y = next(train_feed)
g_loss = gan.train_on_batch(X, [Y, L])
util.printProgress(step, 0, [g_loss[1], d_loss])
'''
#Test
if(step % params['test_interval'] == 0):
n_batches = 8
test_loss = np.zeros(2)
for j in xrange(n_batches):
X,Y = next(warp_test_feed)
#test_loss += np.array(generator.test_on_batch(X_warp,Y_warp))
L = np.zeros([batch_size,2])
L[:,1] = 1 #Fake images
test_loss_j = gan_warp.test_on_batch(X_warp, [Y_warp,L])
test_loss += np.array(test_loss_j[1:3])
test_loss /= (n_batches)
util.printProgress(step,1,test_loss)
'''
if (step % params['model_save_interval'] == 0 and step > 0):
gan.save(network_dir + '/' + str(step) + '.h5')
train_loader_B = utils.data_load(os.path.join('data', args.dataset), 'trainB', transform, args.batch_size, shuffle=True, drop_last=True)
test_loader_A = utils.data_load(os.path.join('data', args.dataset), 'testA', transform, 1, shuffle=True, drop_last=True)
test_loader_B = utils.data_load(os.path.join('data', args.dataset), 'testB', transform, 1, shuffle=True, drop_last=True)
print('------------ Datasets -------------')
print('TrainA:', len(train_loader_A))
print('TrainB:', len(train_loader_B))
print('TestA:', len(test_loader_A))
print('TestB:', len(test_loader_B))
print('-------------- End ----------------')
# network
En_A = networks.encoder(in_nc=args.in_ngc, nf=args.ngf, img_size=args.img_size).to(device)
En_B = networks.encoder(in_nc=args.in_ngc, nf=args.ngf, img_size=args.img_size).to(device)
De_A = networks.decoder(out_nc=args.out_ngc, nf=args.ngf).to(device)
De_B = networks.decoder(out_nc=args.out_ngc, nf=args.ngf).to(device)
Disc_A = networks.discriminator(in_nc=args.in_ndc, out_nc=args.out_ndc, nf=args.ndf, img_size=args.img_size).to(device)
Disc_B = networks.discriminator(in_nc=args.in_ndc, out_nc=args.out_ndc, nf=args.ndf, img_size=args.img_size).to(device)
En_A.train()
En_B.train()
De_A.train()
De_B.train()
Disc_A.train()
Disc_B.train()
print('---------- Networks initialized -------------')
utils.print_network(En_A)
utils.print_network(En_B)
utils.print_network(De_A)
utils.print_network(De_B)
utils.print_network(Disc_A)
utils.print_network(Disc_B)
print('-----------------------------------------------')
def train(dataset, gpu_id):
params = param.getGeneralParams()
gpu = '/gpu:' + str(gpu_id)
lift_params = param.getDatasetParams('weightlifting')
golf_params = param.getDatasetParams('golfswinghd')
workout_params = param.getDatasetParams('workout')
tennis_params = param.getDatasetParams('tennis')
aux_params = param.getDatasetParams('test-aux')
_, lift_test = datareader.makeWarpExampleList(lift_params, 0, 2000, 2, 1)
_, golf_test = datareader.makeWarpExampleList(golf_params, 0, 5000, 2, 2)
_, workout_test = datareader.makeWarpExampleList(workout_params, 0, 2000,
2, 3)
_, tennis_test = datareader.makeWarpExampleList(tennis_params, 0, 2000, 2,
4)
_, aux_test = datareader.makeWarpExampleList(aux_params, 0, 2000, 2, 5)
test = lift_test + golf_test + workout_test + tennis_test + aux_test
feed = datageneration.warpExampleGenerator(test,
params,
do_augment=False,
draw_skeleton=False,
skel_color=(0, 0, 255),
return_pose_vectors=True)
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
config.allow_soft_placement = True
with tf.Session(config=config) as sess:
sess.run(tf.global_variables_initializer())
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord)
with tf.device(gpu):
vgg_model = myVGG.vgg_norm()
networks.make_trainable(vgg_model, False)
response_weights = sio.loadmat('mean_response.mat')
gen = networks.network_fgbg(params,
vgg_model,
response_weights,
True,
loss='vgg')
disc = networks.discriminator(params)
gan = networks.gan(gen, disc, params, vgg_model, response_weights,
0.01, 1e-4)
gan.load_weights('../results/networks/gan/10000.h5')
np.random.seed(17)
n_batches = 25
for j in xrange(n_batches):
print j
X, Y = next(feed)
loss = gen.evaluate(X[0:-2], Y)
pred = gen.predict(X[0:-2])
sio.savemat(
'results/outputs/' + str(j) + '.mat', {
'X': X[0],
'Y': Y,
'pred': pred,
'loss': loss,
'src_pose': X[-2],
'tgt_pose': X[-1]
})
map_location=lambda storage, loc: storage,
strict=False))
if args.G_pre_trained_weight != '':
print("loaded G_e weight!")
if torch.cuda.is_available():
G_e.load_state_dict(torch.load(args.G_pre_trained_weight,
strict=False))
else:
# cpu mode
G_e.load_state_dict(
torch.load(args.G_pre_trained_weight,
map_location=lambda storage, loc: storage,
strict=False))
d = networks.TransformerDecoder()
D = networks.discriminator(args.in_ndc, args.out_ndc, args.ndf)
d1 = networks.TransformerDecoder()
D1 = networks.discriminator(args.in_ndc, args.out_ndc, args.ndf)
d2 = networks.TransformerDecoder()
D2 = networks.discriminator(args.in_ndc, args.out_ndc, args.ndf)
d3 = networks.TransformerDecoder()
D3 = networks.discriminator(args.in_ndc, args.out_ndc, args.ndf)
if args.latest_discriminator_model != '':
if torch.cuda.is_available():
d.load_state_dict(torch.load(args.latest_generator_model,
strict=False))
D.load_state_dict(torch.load(args.latest_discriminator_model))
def main(num_epochs=500, configs=configs):
# https://gist.github.com/f0k/738fa2eedd9666b78404ed1751336f56
# Prepare Theano variables for inputs. We don't need targets as we'll set them manually
C_in = T.tensor4('inputs')
G_in = T.matrix('random')
# Load the data
(X_train, y_train, X_test, y_test, X_val, y_val) = load_mnist()
# Classifier
C_network = discriminator(C_in, configs=configs)
C_out = lasagne.layers.get_output(C_network)
C_params = lasagne.layers.get_all_params(C_network, trainable=True)
# Define the synthesiser function (that tries to create 'real' images)
G_network = synthesiser(G_in, configs=configs)
G_params = lasagne.layers.get_all_params(G_network, trainable=True)
real_out = lasagne.layers.get_output(C_network)
# fake_out, second arg in get_output is optional inputs to pass through to C_network
fake_out = lasagne.layers.get_output(
C_network, lasagne.layers.get_output(G_network, deterministic=True))
# Define the objective, updates, and training functions
# Cost = Fakes are class=1, so for generator target is for all to be identified as real (0)
# eps = 1e-10
# alfa = 1-1e-5
alfa = 1
eps = 0
G_obj = lasagne.objectives.binary_crossentropy((fake_out + eps) * alfa,
1).mean()
# Cost = Discriminator needs real = 0, and identify fakes as 1
C_obj = lasagne.objectives.binary_crossentropy((real_out+eps)*alfa, 1).mean()+\
lasagne.objectives.binary_crossentropy((fake_out+eps)*alfa, 0).mean()
train_fn = {}
C_updates = lasagne.updates.adam(C_obj,
C_params,
learning_rate=2e-4,
beta1=0.5)
G_updates = lasagne.updates.adam(G_obj,
G_params,
learning_rate=2e-4,
beta1=0.5)
train_fn['discriminator'] = theano.function([C_in, G_in],
C_obj,
updates=C_updates,
name='C_training')
train_fn['generator'] = theano.function([G_in],
G_obj,
updates=G_updates,
name='G_training')
# Create the theano functions
classify = theano.function([C_in], C_out)
generate = theano.function([G_in],
lasagne.layers.get_output(G_network,
deterministic=True))
# The test prediction is running the discriminator deterministically. All the validation set are
# real, so the cost is when identifiying as fake (1)
test_prediction = lasagne.layers.get_output(C_network, deterministic=True)
test_loss = lasagne.objectives.binary_crossentropy(test_prediction, 1)
test_loss = test_loss.mean()
test_acc = T.mean(test_prediction > 0.5, dtype=theano.config.floatX)
# Loss is from the perspective of the discriminator, so the target is for all values to be labelled true (0)
test_generator = lasagne.layers.get_output(C_network,
lasagne.layers.get_output(
G_network,
deterministic=True),
deterministic=True)
test_loss_gen = lasagne.objectives.binary_crossentropy(test_generator,
0).mean()
test_acc_gen = T.mean(test_generator < 0.5, dtype=theano.config.floatX)
# Compile the training and validation functions
val_fn = theano.function([C_in], [test_loss, test_acc])
val_gen_fn = theano.function([G_in], [test_loss_gen, test_acc_gen])
# pdb.set_trace()
# Run
lossplots = run(X_train,
y_train,
X_test,
y_test,
X_val,
y_val,
num_epochs,
train_fn,
val_fn,
val_gen_fn,
G_params,
generate,
configs=configs)
networks = {}
networks['generator'] = G_network
networks['discriminator'] = C_network
return generate, networks, lossplots
transforms.Normalize(mean=(0.5, 0.5, 0.5), std=(0.5, 0.5, 0.5))
])
train_loader_src = utils.data_load(os.path.join('data', args.src_data), 'train', src_transform, args.batch_size, shuffle=True, drop_last=True)
train_loader_tgt = utils.data_load(os.path.join('data', args.tgt_data), 'pair', tgt_transform, args.batch_size, shuffle=True, drop_last=True)
test_loader_src = utils.data_load(os.path.join('data', args.src_data), 'test', src_transform, 1, shuffle=True, drop_last=True)
# network
G = networks.generator(args.in_ngc, args.out_ngc, args.ngf, args.nb)
if args.latest_generator_model != '':
if torch.cuda.is_available():
G.load_state_dict(torch.load(args.latest_generator_model))
else:
# cpu mode
G.load_state_dict(torch.load(args.latest_generator_model, map_location=lambda storage, loc: storage))
D = networks.discriminator(args.in_ndc, args.out_ndc, args.ndf)
if args.latest_discriminator_model != '':
if torch.cuda.is_available():
D.load_state_dict(torch.load(args.latest_discriminator_model))
else:
D.load_state_dict(torch.load(args.latest_discriminator_model, map_location=lambda storage, loc: storage))
VGG = networks.VGG19(init_weights=args.vgg_model, feature_mode=True)
G.to(device)
D.to(device)
VGG.to(device)
G.train()
D.train()
VGG.eval()
print('---------- Networks initialized -------------')
utils.print_network(G)
utils.print_network(D)
writer4.start()
writer5.start()
writer6.start()
#writer7.start()
#writer8.start()
#
gen = generator(target_size[0],
target_size[1],
1024,
noise_dim,
n_labels,
target_size[2],
tanh=True)
disc = discriminator(target_size[0],
target_size[1],
512,
n_labels,
target_size[2],
wgan=True)
opt = Adam(0.0002, 0.5)
disc.compile(loss=['binary_crossentropy', null_loss()],
optimizer=opt,
metrics=['accuracy'])
frozen_disc = Model(inputs=disc.inputs, outputs=disc.outputs)
frozen_disc.trainable = False
adv = Model(inputs=gen.input, outputs=frozen_disc(gen.output))
adv.compile(loss=['binary_crossentropy', null_loss()],
optimizer=opt,
metrics=['accuracy'])
for repeat in range(8):
plt.ion()
plt.figure(figsize=(10, 10))
def test_discriminator_invalid_input(self):
with self.assertRaisesRegexp(ValueError, 'Shape must be rank 4'):
networks.discriminator(tf.zeros([28, 28, 3]))