def sample_gaussian(shape, truncate_std=None, gpu=None): """ sample a gaussian(noise), capable to use truncated normal dist. """ y = torch.randn(shape) y = y.cuda(gpu) if gpu is not None else y if truncate_std is not None: truncated_normal(y, mean=0, std=1, truncate_std=truncate_std) return y
def __init__(self): super(Net, self).__init__() self.w1 = Parameter(truncated_normal(1, 40, std=1e-2), requires_grad=True) self.w2 = Parameter(truncated_normal(40, 40, std=1e-2), requires_grad=True) self.w3 = Parameter(truncated_normal(40, 1, std=1e-2), requires_grad=True) self.b1 = Parameter(torch.zeros(40), requires_grad=True) self.b2 = Parameter(torch.zeros(40), requires_grad=True) self.b3 = Parameter(torch.zeros(1), requires_grad=True) self.params = OrderedDict([('w1', self.w1), ('w2', self.w2), ('w3', self.w3), ('b1', self.b1), ('b2', self.b2), ('b3', self.b3)])
def find_distributions(self, station, clss, kind): if self.source(station, clss, kind) == 'NoArrivals': return lambda: float('Inf') if self.source(station, clss, kind)[0] == 'Uniform': return lambda: uniform( self.source(station, clss, kind)[1], self.source(station, clss, kind)[2]) if self.source(station, clss, kind)[0] == 'Deterministic': return lambda: self.source(station, clss, kind)[1] if self.source(station, clss, kind)[0] == 'Exponential': return lambda: expovariate(self.source(station, clss, kind)[1]) if self.source(station, clss, kind)[0] == 'Normal': return lambda: truncated_normal( self.source(station, clss, kind)[1], self.source(station, clss, kind)[2]) if self.source(station, clss, kind)[0] == 'Custom': return lambda: random_choice( array=self.source(station, clss, kind)[1], probs=self.source(station, clss, kind)[2]) if self.source(station, clss, kind)[0] == 'UserDefined': return lambda: self.check_userdef_dist( self.source(station, clss, kind)[1]) if self.source(station, clss, kind)[0] == 'TimeDependent': return lambda t: self.check_timedependent_dist( self.source(station, clss, kind)[1], kind, t) if self.source(station, clss, kind)[0] == 'Empirical': if isinstance(self.source(station, clss, kind)[1], str): return lambda: random_choice( self.import_empirical(self.source(station, clss, kind)[1])) return lambda: random_choice(self.source(station, clss, kind)[1])
def test_inception(self): test_num = 50000 assert test_num % self.batch_size == 0, "test_num mod batch_size != 0" imgs = np.zeros((test_num, 3, 128, 128)) with torch.no_grad(): for i in tqdm(range(0, test_num, self.batch_size)): # randomly sample from 1000 classes # [0. 398) label = torch.LongTensor(self.batch_size).random_(0, 398).to( self.device) noise = torch.FloatTensor(utils.truncated_normal(self.batch_size*self.z_dim))\ .view(self.batch_size, self.z_dim).to(self.device) imgs[i:i + self.batch_size] = self.G( noise, label).cpu().numpy() # [NCHW] IS_mean, IS_std = get_inception_score(imgs) print('IS_mean: {:.2f}, IS_std: {:.2f}'.format(IS_mean, IS_std))
def test_inter_fid(self): test_num = 50000 assert test_num % self.batch_size == 0, "test_num mod batch_size != 0" config = tf.ConfigProto() config.gpu_options.per_process_gpu_memory_fraction = 0.5 self.build_fid_graph() sess = tf.Session(config=config) sess.run(tf.global_variables_initializer()) sess.graph.finalize() # load dumped mean & cov from inception_activations real_mean = np.load('{}/stat_real_ani.npz'.format( self.real_fid_stat_dir))['mean'] real_cov = np.load('{}/stat_real_ani.npz'.format( self.real_fid_stat_dir))['cov'] fake_act = np.zeros((test_num, 2048)) with torch.no_grad(): for i in tqdm(range(0, test_num, self.batch_size)): # randomly sample from animals classes label = torch.LongTensor(self.batch_size).random_(0, 398).to( self.device) noise = torch.FloatTensor(utils.truncated_normal(self.batch_size*self.z_dim)) \ .view(self.batch_size, self.z_dim).to(self.device) x_sample = self.G(noise, label) if i == 0: sample_path = os.path.join(self.result_dir, 'test-{}.png'.format(i)) save_image(utils.denorm(x_sample[:64]), sample_path, nrow=8) x_sample = np.transpose(x_sample.cpu().numpy(), (0, 2, 3, 1)) # [NCHW] -> [NHWC] fake_act[i:i + self.batch_size] = sess.run( self.activations, {self.images_holder: x_sample}) fake_mean, fake_cov = np.mean(fake_act, axis=0), np.cov(fake_act, rowvar=False) fid = numpy_fid(fake_mean, fake_cov, real_mean, real_cov) print('[TinyGAN] Inter-class FID: {:.3f}'.format(fid)) sess.close()
def convolution(self, input, nOut, kH, kW, strides, padding='VALID', boolDecay=True, non_linearity=tf.nn.relu): self.layer_key += 1 with tf.name_scope('Convolution_{0}'.format(self.layer_key)) as scope: nIn = input.get_shape().dims[-1].value stddev = math.sqrt(2.0 / (kH * kW * nIn)) filterInitializer = tf.constant_initializer( truncated_normal(shape=(kH, kW, nIn, nOut), stddev=stddev, mean=0.)) kernel = tf.get_variable(initializer=filterInitializer, shape=(kH, kW, nIn, nOut), trainable=True, name='filter_{0}'.format(self.layer_key)) conv = tf.nn.conv2d(input=input, filter=kernel, strides=strides, padding=padding) output_shape = [dim.value for dim in conv.get_shape()] output_shape[0] = -1 biases = tf.get_variable( initializer=tf.constant_initializer(0.01 + np.zeros(nOut)), shape=nOut, trainable=True, name='biases_{0}'.format(self.layer_key)) conv_biases = conv + biases if non_linearity: nonlin = non_linearity(conv_biases) else: nonlin = conv_biases if boolDecay: self.variables += [kernel] self.variables += [biases] print(nonlin) return nonlin
def save_images(generator_model, sample_images_path, submission_dir, z_dim, num_of_classes=120, num_images=10000): sample_images_dir = Path(sample_images_path) sample_images_dir.mkdir(exist_ok=True) im_batch_size = 50 device = torch.device('cuda') for i_batch in range(0, num_images, im_batch_size): z = utils.truncated_normal((im_batch_size, z_dim), threshold=1) gen_z = torch.from_numpy(z).float().to(device) dog_labels = torch.squeeze(torch.randint(0, num_of_classes, (im_batch_size,), device=device)) gen_images = generator_model(gen_z, dog_labels) images = gen_images.to('cpu').clone().detach() images = images.numpy().transpose(0, 2, 3, 1) for i_image in range(gen_images.size(0)): save_image(utils.denorm(gen_images[i_image, :, :, :]), sample_images_dir / f'image_{i_batch + i_image:05d}.png') submission_dir = Path(submission_dir) submission_dir.mkdir(exist_ok=True) shutil.make_archive(os.path.join(submission_dir, 'images'), 'zip', sample_images_path)
def test(self): nrow = 8 n_samples = nrow * nrow with torch.no_grad(): for i, (x_real, noise, label) in enumerate(tqdm(self.test_loader)): if i == 10: break x_real = x_real.to(self.device) noise = noise.to(self.device) label = label.to(self.device) ''' test flop ''' if i == 0: from thop import profile flops, params = profile(self.G, inputs=(noise[0].unsqueeze(0), label[0].unsqueeze(0))) print( '=======================================================================' ) print('FLOPS: {:.2f} B, Params.: {:.1f} M'.format( flops / 10**9, params / 10**6)) print( '=======================================================================' ) # recon x_fake = self.G(noise, label) comparison = torch.cat([x_real[:nrow], x_fake[:nrow].float()]) sample_path = os.path.join(self.result_dir, '{}-rec.png'.format(i + 1)) save_image(utils.denorm(comparison.cpu()), sample_path) # sample noise2 = torch.FloatTensor(utils.truncated_normal(n_samples*self.z_dim)) \ .view(n_samples, self.z_dim).to(self.device) label = label[:nrow].repeat(nrow) x_sample = self.G(noise2, label) sample_path = os.path.join(self.result_dir, '{}-sample.png'.format(i + 1)) save_image(utils.denorm(x_sample), sample_path, nrow=nrow)
def sample_gaussian(size, truncate_std=None, gpu=None): y = torch.randn(*size).float() y = y if gpu is None else y.cuda(gpu) if truncate_std is not None: truncated_normal(y, mean=0, std=1, trunc_std=truncate_std) return y
def train(self): loss = {} nrow = min(int(np.sqrt(self.batch_size)), 8) n_samples = nrow * nrow iter_per_epoch = len(self.train_loader.dataset) // self.batch_size max_iteration = self.num_epoch * iter_per_epoch lambda_l1 = 0.2 print('Start training...') for epoch in tqdm(range(self.resume_epoch, self.num_epoch)): for i, (x_real, noise, label) in enumerate(tqdm(self.train_loader)): # lr decay if epoch * iter_per_epoch + i >= self.lr_decay_start: utils.decay_lr(self.g_optimizer, max_iteration, self.lr_decay_start, self.g_lr) utils.decay_lr(self.d_optimizer, max_iteration, self.lr_decay_start, self.d_lr) if i % 1000 == 0: print('d_lr / g_lr is updated to {:.8f} / {:.8f} !'. format(self.d_optimizer.param_groups[0]['lr'], self.g_optimizer.param_groups[0]['lr'])) x_real = x_real.to(self.device) noise = noise.to(self.device) label = label.to(self.device) #''' # =================================================================================== # # 1. Train the discriminator # # =================================================================================== # for param in self.D.parameters(): param.requires_grad = True dis_real, real_list = self.D(x_real, label) real_list = [h.detach() for h in real_list] x_fake = self.G(noise, label).detach() dis_fake, _ = self.D(x_fake, label) d_loss_real, d_loss_fake = self.dis_hinge(dis_real, dis_fake) # sample try: x_real2, label2 = next(real_iter) except: real_iter = iter(self.real_loader) x_real2, label2 = next(real_iter) x_real2 = x_real2.to(self.device) label2 = label2.to(self.device) noise2 = torch.FloatTensor(utils.truncated_normal(self.batch_size*self.z_dim)) \ .view(self.batch_size, self.z_dim).to(self.device) # noise2 = torch.randn(self.batch_size, self.z_dim).to(self.device) dis_real2, _ = self.D(x_real2, label2) x_fake2 = self.G(noise2, label2).detach() dis_fake2, _ = self.D(x_fake2, label2) d_loss_real2, d_loss_fake2 = self.dis_hinge( dis_real2, dis_fake2) # Backward and optimize. d_loss = d_loss_real + d_loss_fake + 0.2 * (d_loss_real2 + d_loss_fake2) self.d_optimizer.zero_grad() d_loss.backward() self.d_optimizer.step() # Logging. loss['D/loss_real'] = d_loss_real.item() loss['D/loss_fake'] = d_loss_fake.item() loss['D/loss_real2'] = d_loss_real2.item() loss['D/loss_fake2'] = d_loss_fake2.item() # =================================================================================== # # 2. Train the generator # # =================================================================================== # #''' x_fake = self.G(noise, label) for param in self.D.parameters(): param.requires_grad = False dis_fake, fake_list = self.D(x_fake, label) g_loss_feat = self.KDLoss(real_list, fake_list) g_loss_pix = F.l1_loss(x_fake, x_real) g_loss = g_loss_feat + lambda_l1 * g_loss_pix loss['G/loss_ft'] = g_loss_feat.item() loss['G/loss_l1'] = g_loss_pix.item() if (i + 1) % self.n_critic == 0: dis_fake, _ = self.D(x_fake, label) g_loss_fake = self.gen_hinge(dis_fake) g_loss += self.lambda_gan * g_loss_fake # sample noise2 = torch.FloatTensor(utils.truncated_normal(self.batch_size*self.z_dim)) \ .view(self.batch_size, self.z_dim).to(self.device) # noise2 = torch.randn(self.batch_size, self.z_dim).to(self.device) x_fake2 = self.G(noise2, label2) dis_fake2, _ = self.D(x_fake2, label2) g_loss_fake2 = self.gen_hinge(dis_fake2) g_loss += 0.2 * self.lambda_gan * g_loss_fake2 loss['G/loss_fake'] = g_loss_fake.item() loss['G/loss_fake2'] = g_loss_fake2.item() self.g_optimizer.zero_grad() g_loss.backward() self.g_optimizer.step() # =================================================================================== # # 3. Miscellaneous # # =================================================================================== # # Print out training information. if (i + 1) % self.log_step == 0: log = "[{}/{}]".format(epoch, i) for tag, value in loss.items(): log += ", {}: {:.4f}".format(tag, value) print(log) if self.use_tensorboard: for tag, value in loss.items(): self.logger.scalar_summary(tag, value, i + 1) if epoch == 0 or (epoch + 1) % self.sample_step == 0: with torch.no_grad(): """ # randomly sampled noise noise = torch.FloatTensor(utils.truncated_normal(n_samples*self.z_dim)) \ .view(n_samples, self.z_dim).to(self.device) label = label[:nrow].repeat(nrow) #label = np.random.choice(1000, nrow, replace=False) #label = torch.tensor(label).repeat(10).to(self.device) x_sample = self.G(noise, label) sample_path = os.path.join(self.sample_dir, '{}-sample.png'.format(epoch+1)) save_image(utils.denorm(x_sample.cpu()), sample_path, nrow=nrow, padding=0) """ # recons n = min(x_real.size(0), 8) comparison = torch.cat([x_real[:n], x_fake[:n]]) sample_path = os.path.join( self.sample_dir, '{}-train.png'.format(epoch + 1)) save_image(utils.denorm(comparison.cpu()), sample_path) print('Save fake images into {}...'.format(sample_path)) # noise2 comparison = torch.cat([x_real2[:n], x_fake2[:n]]) sample_path = os.path.join( self.sample_dir, '{}-random.png'.format(epoch + 1)) save_image(utils.denorm(comparison.cpu()), sample_path) print('Save fake images into {}...'.format(sample_path)) # noise sampled from BigGAN's test set try: x_real, noise, label = next(test_iter) except: test_iter = iter(self.test_loader) x_real, noise, label = next(test_iter) noise = noise.to(self.device) label = label.to(self.device) x_fake = self.G(noise, label).detach().cpu() n = min(x_real.size(0), 8) comparison = torch.cat([x_real[:n], x_fake[:n]]) sample_path = os.path.join(self.sample_dir, '{}-test.png'.format(epoch + 1)) save_image(utils.denorm(comparison.cpu()), sample_path) print('Save fake images into {}...'.format(sample_path)) lambda_l1 = max(0.00, lambda_l1 - 0.01) # Save model checkpoints. if (epoch + 1) % self.model_save_step == 0: utils.save_model(self.model_save_dir, epoch + 1, self.G, self.D, self.g_optimizer, self.d_optimizer)
def test_intra_fid_all(self): test_num = 5000 assert test_num % self.batch_size == 0, "test_num mod batch_size != 0" config = tf.ConfigProto() config.gpu_options.per_process_gpu_memory_fraction = 0.5 self.build_fid_graph() sess = tf.Session(config=config) sess.run(tf.global_variables_initializer()) sess.graph.finalize() # noises = np.load('noises.npy') # fixed noise from truncated normal fake_act = np.zeros((test_num, 2048)) fid_scores = [] with torch.no_grad(): for c_id in tqdm(range(398)): label = torch.LongTensor(self.batch_size).fill_(c_id).to( self.device) for i in tqdm(range(0, test_num, self.batch_size)): # noise = torch.FloatTensor(noises[i:i+self.batch_size]).to(self.device) # randomly sample noise = torch.FloatTensor(utils.truncated_normal(self.batch_size*self.z_dim)) \ .view(self.batch_size, self.z_dim).to(self.device) x_sample = self.G(noise, label) if i == 0: sample_path = os.path.join(self.result_dir, '{}-test.png'.format(c_id)) save_image(utils.denorm(x_sample[:64]), sample_path, nrow=8) x_sample = np.transpose(x_sample.cpu().numpy(), (0, 2, 3, 1)) # [NCHW] -> [NHWC] fake_act[i:i + self.batch_size] = sess.run( self.activations, {self.images_holder: x_sample}) # np.save('fake_act_{}.npy'.format(c_id), fake_act) real_act = np.load( os.path.join(self.real_incep_stat_dir, 'act_{}.npy').format(c_id)) if self.use_numpy_fid: real_mean, real_cov = np.mean(real_act, axis=0), np.cov(real_act, rowvar=False) fake_mean, fake_cov = np.mean(fake_act, axis=0), np.cov(fake_act, rowvar=False) fid = numpy_fid(fake_mean, fake_cov, real_mean, real_cov) else: fid = sess.run(self.fid, { self.real_acts: real_act, self.fake_acts: fake_act }) print('[{}] FID: {:.3f}'.format(c_id, fid)) fid_scores.append(fid) np.save( os.path.join(self.model_save_dir, 'intra_fid_scores_real_small.npy'), fid_scores) print('[TinyGAN] Intra-class FID: {:.3f}, std: {:.3f}'.format( np.mean(fid_scores), np.std(fid_scores))) sess.close()
def setup_weights(self, n_neurons, input_shape): rad = 1 / np.sqrt(input_shape) X = truncated_normal(mean=0, sd=1, low=-rad, upp=rad) self.weights = np.array(X.rvs((n_neurons, input_shape)))