def conceptcombineeval(model_list, select_idx): dataset = CelebAHQOverfit() dataloader = torch.utils.data.DataLoader(dataset, batch_size=64, shuffle=True, num_workers=4) n = 64 labels = [] for six in select_idx: label_ix = np.eye(2)[six] label_batch = np.tile(label_ix[None, :], (n, 1)) label = torch.Tensor(label_batch).cuda() labels.append(label) def get_color_distortion(s=1.0): # s is the strength of color distortion. color_jitter = transforms.ColorJitter(0.8 * s, 0.8 * s, 0.8 * s, 0.4 * s) rnd_color_jitter = transforms.RandomApply([color_jitter], p=0.8) rnd_gray = transforms.RandomGrayscale(p=0.2) color_distort = transforms.Compose([rnd_color_jitter, rnd_gray]) return color_distort color_transform = get_color_distortion() im_size = 128 transform = transforms.Compose([ transforms.RandomResizedCrop(im_size, scale=(0.4, 1.0)), transforms.RandomHorizontalFlip(), color_transform, transforms.ToTensor() ]) gt_ims = [] fake_ims = [] for data, label in tqdm(dataloader): gt_ims.extend( list((data.numpy().transpose( (0, 2, 3, 1)) * 255).astype(np.uint8))) im = torch.rand(n, 3, 128, 128).cuda() im_noise = torch.randn_like(im).detach() # First get good initializations for sampling # for i in range(10): # for i in range(20): # im_noise.normal_() # im = im + 0.001 * im_noise # # im.requires_grad = True # im.requires_grad_(requires_grad=True) # energy = 0 # for model, label in zip(model_list, labels): # energy = model.forward(im, label) + energy # # print("step: ", i, energy.mean()) # im_grad = torch.autograd.grad([energy.sum()], [im])[0] # im = im - FLAGS.step_lr * im_grad # im = im.detach() # im = torch.clamp(im, 0, 1) # im = im.detach().cpu().numpy().transpose((0, 2, 3, 1)) # im = (im * 255).astype(np.uint8) # ims = [] # for i in range(im.shape[0]): # im_i = np.array(transform(Image.fromarray(np.array(im[i])))) # ims.append(im_i) # im = torch.Tensor(np.array(ims)).cuda() # Then refine the images for i in range(FLAGS.num_steps): im_noise.normal_() im = im + 0.001 * im_noise # im.requires_grad = True im.requires_grad_(requires_grad=True) energy = 0 for model, label in zip(model_list, labels): energy = model.forward(im, label) + energy print("step: ", i, energy.mean()) im_grad = torch.autograd.grad([energy.sum()], [im])[0] im = im - FLAGS.step_lr * im_grad im = im.detach() im = torch.clamp(im, 0, 1) im = im.detach().cpu() fake_ims.extend( list((im.numpy().transpose((0, 2, 3, 1)) * 255).astype(np.uint8))) if len(gt_ims) > 10000: break get_fid_score(gt_ims, fake_ims) fake_ims = np.array(fake_ims) fake_ims_flat = fake_ims.reshape(fake_ims.shape[0], -1) std_im = np.std(fake_ims, axis=0).mean() print("standard deviation of image", std_im) import pdb pdb.set_trace() print("here")
def conceptcombineeval(model_list, select_idx): dataset = ImageNet() dataloader = torch.utils.data.DataLoader(dataset, batch_size=64, shuffle=True, num_workers=4) n = 64 labels = [] for six in select_idx: six = np.random.permutation(1000)[:n] print(six) label_batch = np.eye(1000)[six] # label_ix = np.eye(2)[six] # label_batch = np.tile(label_ix[None, :], (n, 1)) label = torch.Tensor(label_batch).cuda() labels.append(label) def get_color_distortion(s=1.0): # s is the strength of color distortion. color_jitter = transforms.ColorJitter(0.8 * s, 0.8 * s, 0.8 * s, 0.4 * s) rnd_color_jitter = transforms.RandomApply([color_jitter], p=0.8) rnd_gray = transforms.RandomGrayscale(p=0.2) color_distort = transforms.Compose([rnd_color_jitter, rnd_gray]) return color_distort color_transform = get_color_distortion(0.5) im_size = 128 transform = transforms.Compose([ transforms.RandomResizedCrop(im_size, scale=(0.3, 1.0)), transforms.RandomHorizontalFlip(), color_transform, transforms.ToTensor() ]) gt_ims = [] fake_ims = [] label_embed = torch.eye(1000).cuda() im = None for _, data, label in tqdm(dataloader): print(label) gt_ims.extend(list((data.numpy() * 255).astype(np.uint8))) if im is None: im = torch.rand(n, 3, 128, 128).cuda() im_noise = torch.randn_like(im).detach() # First get good initializations for sampling for i in range(5): for i in range(60): label = torch.randperm(1000).to(im.device)[:n] label = label_embed[label] im_noise.normal_() im = im + 0.001 * im_noise # im.requires_grad = True im.requires_grad_(requires_grad=True) energy = 0 for model, label in zip(model_list, labels): energy = model.forward(im, label) + energy # print("step: ", i, energy.mean()) im_grad = torch.autograd.grad([energy.sum()], [im])[0] im = im - FLAGS.step_lr * im_grad im = im.detach() im = torch.clamp(im, 0, 1) im = im.detach().cpu().numpy().transpose((0, 2, 3, 1)) im = (im * 255).astype(np.uint8) ims = [] for i in range(im.shape[0]): im_i = np.array(transform(Image.fromarray(np.array(im[i])))) ims.append(im_i) im = torch.Tensor(np.array(ims)).cuda() # Then refine the images for i in range(FLAGS.num_steps): im_noise.normal_() im = im + 0.001 * im_noise # im.requires_grad = True im.requires_grad_(requires_grad=True) energy = 0 label = torch.randperm(1000).to(im.device)[:n] label = label_embed[label] for model, label in zip(model_list, labels): energy = model.forward(im, label) + energy print("step: ", i, energy.mean()) im_grad = torch.autograd.grad([energy.sum()], [im])[0] im = im - FLAGS.step_lr * im_grad im = im.detach() im = torch.clamp(im, 0, 1) im_cpu = im.detach().cpu() ims = list((im_cpu.numpy().transpose( (0, 2, 3, 1)) * 255).astype(np.uint8)) fake_ims.extend(ims) if len(gt_ims) > 50000: break splits = max(1, len(fake_ims) // 5000) score, std = get_inception_score(fake_ims, splits=splits) print("inception score {}, with std {} ".format(score, std)) get_fid_score(gt_ims, fake_ims) import pdb pdb.set_trace() print("here")
def compute_inception(sess, target_vars): X_START = target_vars['X_START'] Y_GT = target_vars['Y_GT'] X_finals = target_vars['X_finals'] NOISE_SCALE = target_vars['NOISE_SCALE'] energy_noise = target_vars['energy_noise'] size = FLAGS.im_number num_steps = size // 1000 images = [] test_ims = [] test_images = [] if FLAGS.dataset == "cifar10": test_dataset = Cifar10(full=True, noise=False) elif FLAGS.dataset == "celeba": dataset = CelebA() elif FLAGS.dataset == "imagenet" or FLAGS.dataset == "imagenetfull": test_dataset = Imagenet(train=False) if FLAGS.dataset != "imagenetfull": test_dataloader = DataLoader(test_dataset, batch_size=FLAGS.batch_size, num_workers=4, shuffle=True, drop_last=False) else: test_dataloader = TFImagenetLoader('test', FLAGS.batch_size, 0, 1) for data_corrupt, data, label_gt in tqdm(test_dataloader): data = data.numpy() test_ims.extend(list(rescale_im(data))) if FLAGS.dataset == "imagenetfull" and len(test_ims) > 60000: test_ims = test_ims[:60000] break # n = min(len(images), len(test_ims)) print(len(test_ims)) # fid = get_fid_score(test_ims[:30000], test_ims[-30000:]) # print("Base FID of score {}".format(fid)) if FLAGS.dataset == "cifar10": classes = 10 else: classes = 1000 if FLAGS.dataset == "imagenetfull": n = 128 else: n = 32 for j in range(num_steps): itr = int(1000 / 500 * FLAGS.repeat_scale) data_buffer = InceptionReplayBuffer(1000) curr_index = 0 identity = np.eye(classes) test_steps = range(300, itr, 20) for i in tqdm(range(itr)): model_index = curr_index % len(X_finals) x_final = X_finals[model_index] noise_scale = [1] if len(data_buffer) < 1000: x_init = np.random.uniform(0, 1, (FLAGS.batch_size, n, n, 3)) label = np.random.randint(0, classes, (FLAGS.batch_size)) label = identity[label] x_new = sess.run([x_final], {X_START:x_init, Y_GT:label, NOISE_SCALE: noise_scale})[0] data_buffer.add(x_new, label) else: (x_init, label), idx = data_buffer.sample(FLAGS.batch_size) keep_mask = (np.random.uniform(0, 1, (FLAGS.batch_size)) > 0.99) label_keep_mask = (np.random.uniform(0, 1, (FLAGS.batch_size)) > 0.9) label_corrupt = np.random.randint(0, classes, (FLAGS.batch_size)) label_corrupt = identity[label_corrupt] x_init_corrupt = np.random.uniform(0, 1, (FLAGS.batch_size, n, n, 3)) if i < itr - FLAGS.nomix: x_init[keep_mask] = x_init_corrupt[keep_mask] label[label_keep_mask] = label_corrupt[label_keep_mask] # else: # noise_scale = [0.7] x_new, e_noise = sess.run([x_final, energy_noise], {X_START:x_init, Y_GT:label, NOISE_SCALE: noise_scale}) data_buffer.set_elms(idx, x_new, label) curr_index += 1 ims = np.array(data_buffer._storage[:1000]) ims = rescale_im(ims) test_images.extend(list(ims)) saveim = osp.join(FLAGS.logdir, FLAGS.exp, "test{}.png".format(FLAGS.resume_iter)) row = 15 col = 20 ims = ims[:row * col] if FLAGS.dataset != "imagenetfull": im_panel = ims.reshape((row, col, 32, 32, 3)).transpose((0, 2, 1, 3, 4)).reshape((32*row, 32*col, 3)) else: im_panel = ims.reshape((row, col, 128, 128, 3)).transpose((0, 2, 1, 3, 4)).reshape((128*row, 128*col, 3)) imsave(saveim, im_panel) splits = max(1, len(test_images) // 5000) score, std = get_inception_score(test_images, splits=splits) print("Inception score of {} with std of {}".format(score, std)) # FID score # n = min(len(images), len(test_ims)) fid = get_fid_score(test_images, test_ims) print("FID of score {}".format(fid))
def compute_inception(model): size = FLAGS.im_number num_steps = size // 1000 images = [] test_ims = [] if FLAGS.dataset == "cifar10": test_dataset = Cifar10(FLAGS) elif FLAGS.dataset == "celeba": test_dataset = CelebAHQ() elif FLAGS.dataset == "mnist": test_dataset = Mnist(train=True) test_dataloader = DataLoader(test_dataset, batch_size=FLAGS.batch_size, num_workers=4, shuffle=True, drop_last=False) if FLAGS.dataset == "cifar10": for data_corrupt, data, label_gt in tqdm(test_dataloader): data = data.numpy() test_ims.extend(list(rescale_im(data))) if len(test_ims) > 10000: break elif FLAGS.dataset == "mnist": for data_corrupt, data, label_gt in tqdm(test_dataloader): data = data.numpy() test_ims.extend(list(np.tile(rescale_im(data), (1, 1, 3)))) if len(test_ims) > 10000: break test_ims = test_ims[:10000] classes = 10 print(FLAGS.batch_size) data_buffer = None for j in range(num_steps): itr = int(1000 / 500 * FLAGS.repeat_scale) if data_buffer is None: data_buffer = InceptionReplayBuffer(1000) curr_index = 0 identity = np.eye(classes) if FLAGS.dataset == "celeba": n = 128 c = 3 elif FLAGS.dataset == "mnist": n = 28 c = 1 else: n = 32 c = 3 for i in tqdm(range(itr)): noise_scale = [1] if len(data_buffer) < 1000: x_init = np.random.uniform(0, 1, (FLAGS.batch_size, c, n, n)) label = np.random.randint(0, classes, (FLAGS.batch_size)) x_init = torch.Tensor(x_init).cuda() label = identity[label] label = torch.Tensor(label).cuda() x_new, _ = gen_image(label, FLAGS, model, x_init, FLAGS.num_steps) x_new = x_new.detach().cpu().numpy() label = label.detach().cpu().numpy() data_buffer.add(x_new, label) else: if i < itr - FLAGS.nomix: (x_init, label), idx = data_buffer.sample( FLAGS.batch_size, transform=FLAGS.transform) else: if FLAGS.dataset == "celeba": n = 20 else: n = 2 ix = i % n # for i in range(n): start_idx = (1000 // n) * ix end_idx = (1000 // n) * (ix + 1) (x_init, label) = data_buffer._encode_sample( list(range(start_idx, end_idx)), transform=False) idx = list(range(start_idx, end_idx)) x_init = torch.Tensor(x_init).cuda() label = torch.Tensor(label).cuda() x_new, energy = gen_image(label, FLAGS, model, x_init, FLAGS.num_steps) energy = energy.cpu().detach().numpy() x_new = x_new.cpu().detach().numpy() label = label.cpu().detach().numpy() data_buffer.set_elms(idx, x_new, label) if FLAGS.im_number != 50000: print(np.mean(energy), np.std(energy)) curr_index += 1 ims = np.array(data_buffer._storage[:1000]) ims = rescale_im(ims).transpose((0, 2, 3, 1)) if FLAGS.dataset == "mnist": ims = np.tile(ims, (1, 1, 1, 3)) images.extend(list(ims)) random.shuffle(images) saveim = osp.join('sandbox_cachedir', FLAGS.exp, "test{}.png".format(FLAGS.idx)) if FLAGS.dataset == "cifar10": rix = np.random.permutation(1000)[:100] ims = ims[rix] im_panel = ims.reshape((10, 10, 32, 32, 3)).transpose( (0, 2, 1, 3, 4)).reshape((320, 320, 3)) imsave(saveim, im_panel) print("Saved image!!!!") splits = max(1, len(images) // 5000) score, std = get_inception_score(images, splits=splits) print("Inception score of {} with std of {}".format(score, std)) # FID score n = min(len(images), len(test_ims)) fid = get_fid_score(images, test_ims) print("FID of score {}".format(fid)) elif FLAGS.dataset == "mnist": # ims = ims[:100] # im_panel = ims.reshape((10, 10, 32, 32, 3)).transpose((0, 2, 1, 3, 4)).reshape((320, 320, 3)) # imsave(saveim, im_panel) ims = ims[:100] im_panel = ims.reshape((10, 10, 28, 28, 3)).transpose( (0, 2, 1, 3, 4)).reshape((280, 280, 3)) imsave(saveim, im_panel) print("Saved image!!!!") splits = max(1, len(images) // 5000) # score, std = get_inception_score(images, splits=splits) # print("Inception score of {} with std of {}".format(score, std)) # FID score n = min(len(images), len(test_ims)) fid = get_fid_score(images, test_ims) print("FID of score {}".format(fid)) elif FLAGS.dataset == "celeba": ims = ims[:25] im_panel = ims.reshape((5, 5, 128, 128, 3)).transpose( (0, 2, 1, 3, 4)).reshape((5 * 128, 5 * 128, 3)) imsave(saveim, im_panel)