def sample_gaussian(shape, truncate_std=None, gpu=None):
"""
sample a gaussian(noise), capable to use truncated normal dist.
"""
y = torch.randn(shape)
y = y.cuda(gpu) if gpu is not None else y
if truncate_std is not None:
truncated_normal(y, mean=0, std=1, truncate_std=truncate_std)
return y
def __init__(self):
super(Net, self).__init__()
self.w1 = Parameter(truncated_normal(1, 40, std=1e-2), requires_grad=True)
self.w2 = Parameter(truncated_normal(40, 40, std=1e-2), requires_grad=True)
self.w3 = Parameter(truncated_normal(40, 1, std=1e-2), requires_grad=True)
self.b1 = Parameter(torch.zeros(40), requires_grad=True)
self.b2 = Parameter(torch.zeros(40), requires_grad=True)
self.b3 = Parameter(torch.zeros(1), requires_grad=True)
self.params = OrderedDict([('w1', self.w1), ('w2', self.w2), ('w3', self.w3),
('b1', self.b1), ('b2', self.b2), ('b3', self.b3)])
def find_distributions(self, station, clss, kind):
if self.source(station, clss, kind) == 'NoArrivals':
return lambda: float('Inf')
if self.source(station, clss, kind)[0] == 'Uniform':
return lambda: uniform(
self.source(station, clss, kind)[1],
self.source(station, clss, kind)[2])
if self.source(station, clss, kind)[0] == 'Deterministic':
return lambda: self.source(station, clss, kind)[1]
if self.source(station, clss, kind)[0] == 'Exponential':
return lambda: expovariate(self.source(station, clss, kind)[1])
if self.source(station, clss, kind)[0] == 'Normal':
return lambda: truncated_normal(
self.source(station, clss, kind)[1],
self.source(station, clss, kind)[2])
if self.source(station, clss, kind)[0] == 'Custom':
return lambda: random_choice(
array=self.source(station, clss, kind)[1],
probs=self.source(station, clss, kind)[2])
if self.source(station, clss, kind)[0] == 'UserDefined':
return lambda: self.check_userdef_dist(
self.source(station, clss, kind)[1])
if self.source(station, clss, kind)[0] == 'TimeDependent':
return lambda t: self.check_timedependent_dist(
self.source(station, clss, kind)[1], kind, t)
if self.source(station, clss, kind)[0] == 'Empirical':
if isinstance(self.source(station, clss, kind)[1], str):
return lambda: random_choice(
self.import_empirical(self.source(station, clss, kind)[1]))
return lambda: random_choice(self.source(station, clss, kind)[1])
def test_inception(self):
test_num = 50000
assert test_num % self.batch_size == 0, "test_num mod batch_size != 0"
imgs = np.zeros((test_num, 3, 128, 128))
with torch.no_grad():
for i in tqdm(range(0, test_num, self.batch_size)):
# randomly sample from 1000 classes # [0. 398)
label = torch.LongTensor(self.batch_size).random_(0, 398).to(
self.device)
noise = torch.FloatTensor(utils.truncated_normal(self.batch_size*self.z_dim))\
.view(self.batch_size, self.z_dim).to(self.device)
imgs[i:i + self.batch_size] = self.G(
noise, label).cpu().numpy() # [NCHW]
IS_mean, IS_std = get_inception_score(imgs)
print('IS_mean: {:.2f}, IS_std: {:.2f}'.format(IS_mean, IS_std))
def test_inter_fid(self):
test_num = 50000
assert test_num % self.batch_size == 0, "test_num mod batch_size != 0"
config = tf.ConfigProto()
config.gpu_options.per_process_gpu_memory_fraction = 0.5
self.build_fid_graph()
sess = tf.Session(config=config)
sess.run(tf.global_variables_initializer())
sess.graph.finalize()
# load dumped mean & cov from inception_activations
real_mean = np.load('{}/stat_real_ani.npz'.format(
self.real_fid_stat_dir))['mean']
real_cov = np.load('{}/stat_real_ani.npz'.format(
self.real_fid_stat_dir))['cov']
fake_act = np.zeros((test_num, 2048))
with torch.no_grad():
for i in tqdm(range(0, test_num, self.batch_size)):
# randomly sample from animals classes
label = torch.LongTensor(self.batch_size).random_(0, 398).to(
self.device)
noise = torch.FloatTensor(utils.truncated_normal(self.batch_size*self.z_dim)) \
.view(self.batch_size, self.z_dim).to(self.device)
x_sample = self.G(noise, label)
if i == 0:
sample_path = os.path.join(self.result_dir,
'test-{}.png'.format(i))
save_image(utils.denorm(x_sample[:64]),
sample_path,
nrow=8)
x_sample = np.transpose(x_sample.cpu().numpy(),
(0, 2, 3, 1)) # [NCHW] -> [NHWC]
fake_act[i:i + self.batch_size] = sess.run(
self.activations, {self.images_holder: x_sample})
fake_mean, fake_cov = np.mean(fake_act,
axis=0), np.cov(fake_act,
rowvar=False)
fid = numpy_fid(fake_mean, fake_cov, real_mean, real_cov)
print('[TinyGAN] Inter-class FID: {:.3f}'.format(fid))
sess.close()
def convolution(self,
input,
nOut,
kH,
kW,
strides,
padding='VALID',
boolDecay=True,
non_linearity=tf.nn.relu):
self.layer_key += 1
with tf.name_scope('Convolution_{0}'.format(self.layer_key)) as scope:
nIn = input.get_shape().dims[-1].value
stddev = math.sqrt(2.0 / (kH * kW * nIn))
filterInitializer = tf.constant_initializer(
truncated_normal(shape=(kH, kW, nIn, nOut),
stddev=stddev,
mean=0.))
kernel = tf.get_variable(initializer=filterInitializer,
shape=(kH, kW, nIn, nOut),
trainable=True,
name='filter_{0}'.format(self.layer_key))
conv = tf.nn.conv2d(input=input,
filter=kernel,
strides=strides,
padding=padding)
output_shape = [dim.value for dim in conv.get_shape()]
output_shape[0] = -1
biases = tf.get_variable(
initializer=tf.constant_initializer(0.01 + np.zeros(nOut)),
shape=nOut,
trainable=True,
name='biases_{0}'.format(self.layer_key))
conv_biases = conv + biases
if non_linearity:
nonlin = non_linearity(conv_biases)
else:
nonlin = conv_biases
if boolDecay:
self.variables += [kernel]
self.variables += [biases]
print(nonlin)
return nonlin
def save_images(generator_model, sample_images_path, submission_dir, z_dim, num_of_classes=120, num_images=10000):
sample_images_dir = Path(sample_images_path)
sample_images_dir.mkdir(exist_ok=True)
im_batch_size = 50
device = torch.device('cuda')
for i_batch in range(0, num_images, im_batch_size):
z = utils.truncated_normal((im_batch_size, z_dim), threshold=1)
gen_z = torch.from_numpy(z).float().to(device)
dog_labels = torch.squeeze(torch.randint(0, num_of_classes, (im_batch_size,), device=device))
gen_images = generator_model(gen_z, dog_labels)
images = gen_images.to('cpu').clone().detach()
images = images.numpy().transpose(0, 2, 3, 1)
for i_image in range(gen_images.size(0)):
save_image(utils.denorm(gen_images[i_image, :, :, :]), sample_images_dir / f'image_{i_batch + i_image:05d}.png')
submission_dir = Path(submission_dir)
submission_dir.mkdir(exist_ok=True)
shutil.make_archive(os.path.join(submission_dir, 'images'), 'zip', sample_images_path)
def test(self):
nrow = 8
n_samples = nrow * nrow
with torch.no_grad():
for i, (x_real, noise, label) in enumerate(tqdm(self.test_loader)):
if i == 10: break
x_real = x_real.to(self.device)
noise = noise.to(self.device)
label = label.to(self.device)
''' test flop '''
if i == 0:
from thop import profile
flops, params = profile(self.G,
inputs=(noise[0].unsqueeze(0),
label[0].unsqueeze(0)))
print(
'======================================================================='
)
print('FLOPS: {:.2f} B, Params.: {:.1f} M'.format(
flops / 10**9, params / 10**6))
print(
'======================================================================='
)
# recon
x_fake = self.G(noise, label)
comparison = torch.cat([x_real[:nrow], x_fake[:nrow].float()])
sample_path = os.path.join(self.result_dir,
'{}-rec.png'.format(i + 1))
save_image(utils.denorm(comparison.cpu()), sample_path)
# sample
noise2 = torch.FloatTensor(utils.truncated_normal(n_samples*self.z_dim)) \
.view(n_samples, self.z_dim).to(self.device)
label = label[:nrow].repeat(nrow)
x_sample = self.G(noise2, label)
sample_path = os.path.join(self.result_dir,
'{}-sample.png'.format(i + 1))
save_image(utils.denorm(x_sample), sample_path, nrow=nrow)
def sample_gaussian(size, truncate_std=None, gpu=None):
y = torch.randn(*size).float()
y = y if gpu is None else y.cuda(gpu)
if truncate_std is not None:
truncated_normal(y, mean=0, std=1, trunc_std=truncate_std)
return y
def train(self):
loss = {}
nrow = min(int(np.sqrt(self.batch_size)), 8)
n_samples = nrow * nrow
iter_per_epoch = len(self.train_loader.dataset) // self.batch_size
max_iteration = self.num_epoch * iter_per_epoch
lambda_l1 = 0.2
print('Start training...')
for epoch in tqdm(range(self.resume_epoch, self.num_epoch)):
for i, (x_real, noise,
label) in enumerate(tqdm(self.train_loader)):
# lr decay
if epoch * iter_per_epoch + i >= self.lr_decay_start:
utils.decay_lr(self.g_optimizer, max_iteration,
self.lr_decay_start, self.g_lr)
utils.decay_lr(self.d_optimizer, max_iteration,
self.lr_decay_start, self.d_lr)
if i % 1000 == 0:
print('d_lr / g_lr is updated to {:.8f} / {:.8f} !'.
format(self.d_optimizer.param_groups[0]['lr'],
self.g_optimizer.param_groups[0]['lr']))
x_real = x_real.to(self.device)
noise = noise.to(self.device)
label = label.to(self.device)
#'''
# =================================================================================== #
# 1. Train the discriminator #
# =================================================================================== #
for param in self.D.parameters():
param.requires_grad = True
dis_real, real_list = self.D(x_real, label)
real_list = [h.detach() for h in real_list]
x_fake = self.G(noise, label).detach()
dis_fake, _ = self.D(x_fake, label)
d_loss_real, d_loss_fake = self.dis_hinge(dis_real, dis_fake)
# sample
try:
x_real2, label2 = next(real_iter)
except:
real_iter = iter(self.real_loader)
x_real2, label2 = next(real_iter)
x_real2 = x_real2.to(self.device)
label2 = label2.to(self.device)
noise2 = torch.FloatTensor(utils.truncated_normal(self.batch_size*self.z_dim)) \
.view(self.batch_size, self.z_dim).to(self.device)
# noise2 = torch.randn(self.batch_size, self.z_dim).to(self.device)
dis_real2, _ = self.D(x_real2, label2)
x_fake2 = self.G(noise2, label2).detach()
dis_fake2, _ = self.D(x_fake2, label2)
d_loss_real2, d_loss_fake2 = self.dis_hinge(
dis_real2, dis_fake2)
# Backward and optimize.
d_loss = d_loss_real + d_loss_fake + 0.2 * (d_loss_real2 +
d_loss_fake2)
self.d_optimizer.zero_grad()
d_loss.backward()
self.d_optimizer.step()
# Logging.
loss['D/loss_real'] = d_loss_real.item()
loss['D/loss_fake'] = d_loss_fake.item()
loss['D/loss_real2'] = d_loss_real2.item()
loss['D/loss_fake2'] = d_loss_fake2.item()
# =================================================================================== #
# 2. Train the generator #
# =================================================================================== #
#'''
x_fake = self.G(noise, label)
for param in self.D.parameters():
param.requires_grad = False
dis_fake, fake_list = self.D(x_fake, label)
g_loss_feat = self.KDLoss(real_list, fake_list)
g_loss_pix = F.l1_loss(x_fake, x_real)
g_loss = g_loss_feat + lambda_l1 * g_loss_pix
loss['G/loss_ft'] = g_loss_feat.item()
loss['G/loss_l1'] = g_loss_pix.item()
if (i + 1) % self.n_critic == 0:
dis_fake, _ = self.D(x_fake, label)
g_loss_fake = self.gen_hinge(dis_fake)
g_loss += self.lambda_gan * g_loss_fake
# sample
noise2 = torch.FloatTensor(utils.truncated_normal(self.batch_size*self.z_dim)) \
.view(self.batch_size, self.z_dim).to(self.device)
# noise2 = torch.randn(self.batch_size, self.z_dim).to(self.device)
x_fake2 = self.G(noise2, label2)
dis_fake2, _ = self.D(x_fake2, label2)
g_loss_fake2 = self.gen_hinge(dis_fake2)
g_loss += 0.2 * self.lambda_gan * g_loss_fake2
loss['G/loss_fake'] = g_loss_fake.item()
loss['G/loss_fake2'] = g_loss_fake2.item()
self.g_optimizer.zero_grad()
g_loss.backward()
self.g_optimizer.step()
# =================================================================================== #
# 3. Miscellaneous #
# =================================================================================== #
# Print out training information.
if (i + 1) % self.log_step == 0:
log = "[{}/{}]".format(epoch, i)
for tag, value in loss.items():
log += ", {}: {:.4f}".format(tag, value)
print(log)
if self.use_tensorboard:
for tag, value in loss.items():
self.logger.scalar_summary(tag, value, i + 1)
if epoch == 0 or (epoch + 1) % self.sample_step == 0:
with torch.no_grad():
"""
# randomly sampled noise
noise = torch.FloatTensor(utils.truncated_normal(n_samples*self.z_dim)) \
.view(n_samples, self.z_dim).to(self.device)
label = label[:nrow].repeat(nrow)
#label = np.random.choice(1000, nrow, replace=False)
#label = torch.tensor(label).repeat(10).to(self.device)
x_sample = self.G(noise, label)
sample_path = os.path.join(self.sample_dir, '{}-sample.png'.format(epoch+1))
save_image(utils.denorm(x_sample.cpu()), sample_path, nrow=nrow, padding=0)
"""
# recons
n = min(x_real.size(0), 8)
comparison = torch.cat([x_real[:n], x_fake[:n]])
sample_path = os.path.join(
self.sample_dir, '{}-train.png'.format(epoch + 1))
save_image(utils.denorm(comparison.cpu()), sample_path)
print('Save fake images into {}...'.format(sample_path))
# noise2
comparison = torch.cat([x_real2[:n], x_fake2[:n]])
sample_path = os.path.join(
self.sample_dir, '{}-random.png'.format(epoch + 1))
save_image(utils.denorm(comparison.cpu()), sample_path)
print('Save fake images into {}...'.format(sample_path))
# noise sampled from BigGAN's test set
try:
x_real, noise, label = next(test_iter)
except:
test_iter = iter(self.test_loader)
x_real, noise, label = next(test_iter)
noise = noise.to(self.device)
label = label.to(self.device)
x_fake = self.G(noise, label).detach().cpu()
n = min(x_real.size(0), 8)
comparison = torch.cat([x_real[:n], x_fake[:n]])
sample_path = os.path.join(self.sample_dir,
'{}-test.png'.format(epoch + 1))
save_image(utils.denorm(comparison.cpu()), sample_path)
print('Save fake images into {}...'.format(sample_path))
lambda_l1 = max(0.00, lambda_l1 - 0.01)
# Save model checkpoints.
if (epoch + 1) % self.model_save_step == 0:
utils.save_model(self.model_save_dir, epoch + 1, self.G,
self.D, self.g_optimizer, self.d_optimizer)
def test_intra_fid_all(self):
test_num = 5000
assert test_num % self.batch_size == 0, "test_num mod batch_size != 0"
config = tf.ConfigProto()
config.gpu_options.per_process_gpu_memory_fraction = 0.5
self.build_fid_graph()
sess = tf.Session(config=config)
sess.run(tf.global_variables_initializer())
sess.graph.finalize()
# noises = np.load('noises.npy') # fixed noise from truncated normal
fake_act = np.zeros((test_num, 2048))
fid_scores = []
with torch.no_grad():
for c_id in tqdm(range(398)):
label = torch.LongTensor(self.batch_size).fill_(c_id).to(
self.device)
for i in tqdm(range(0, test_num, self.batch_size)):
# noise = torch.FloatTensor(noises[i:i+self.batch_size]).to(self.device)
# randomly sample
noise = torch.FloatTensor(utils.truncated_normal(self.batch_size*self.z_dim)) \
.view(self.batch_size, self.z_dim).to(self.device)
x_sample = self.G(noise, label)
if i == 0:
sample_path = os.path.join(self.result_dir,
'{}-test.png'.format(c_id))
save_image(utils.denorm(x_sample[:64]),
sample_path,
nrow=8)
x_sample = np.transpose(x_sample.cpu().numpy(),
(0, 2, 3, 1)) # [NCHW] -> [NHWC]
fake_act[i:i + self.batch_size] = sess.run(
self.activations, {self.images_holder: x_sample})
# np.save('fake_act_{}.npy'.format(c_id), fake_act)
real_act = np.load(
os.path.join(self.real_incep_stat_dir,
'act_{}.npy').format(c_id))
if self.use_numpy_fid:
real_mean, real_cov = np.mean(real_act,
axis=0), np.cov(real_act,
rowvar=False)
fake_mean, fake_cov = np.mean(fake_act,
axis=0), np.cov(fake_act,
rowvar=False)
fid = numpy_fid(fake_mean, fake_cov, real_mean, real_cov)
else:
fid = sess.run(self.fid, {
self.real_acts: real_act,
self.fake_acts: fake_act
})
print('[{}] FID: {:.3f}'.format(c_id, fid))
fid_scores.append(fid)
np.save(
os.path.join(self.model_save_dir,
'intra_fid_scores_real_small.npy'), fid_scores)
print('[TinyGAN] Intra-class FID: {:.3f}, std: {:.3f}'.format(
np.mean(fid_scores), np.std(fid_scores)))
sess.close()
def setup_weights(self, n_neurons, input_shape):
rad = 1 / np.sqrt(input_shape)
X = truncated_normal(mean=0, sd=1, low=-rad, upp=rad)
self.weights = np.array(X.rvs((n_neurons, input_shape)))