def __init__(self, image_size):
self.generator_g = Generator().cuda()
self.generator_f = Generator().cuda()
self.discriminator_c = Discriminator(image_size,
num_input_channels=3).cuda()
self.discriminator_t = Discriminator(image_size,
num_input_channels=1).cuda()
self.content_criterion = ContentLoss().cuda()
self.tv_criterion = TVLoss().cuda()
self.color_criterion = nn.BCEWithLogitsLoss().cuda()
self.texture_criterion = nn.BCEWithLogitsLoss().cuda()
betas = (0.5, 0.999)
self.g_optimizer = optim.Adam(lr=1e-4,
params=self.generator_g.parameters(),
betas=betas)
self.f_optimizer = optim.Adam(lr=1e-4,
params=self.generator_f.parameters(),
betas=betas)
self.c_optimizer = optim.Adam(lr=4e-4,
params=self.discriminator_c.parameters(),
betas=betas)
self.t_optimizer = optim.Adam(lr=4e-4,
params=self.discriminator_t.parameters(),
betas=betas)
self.blur = GaussianBlur().cuda()
self.gray = Grayscale().cuda()
def __init__(self, config):
'''keypoint order = thumb, index, middle, ring, little'''
self.root_dir = config['path']
self.augment = (bool(config['augment'])
and (config['scope'] == 'train'))
if config['scope'] == 'train':
(self.samples,
self.labels) = get_splitted_image_names(self.root_dir)['train']
elif config['scope'] == 'val':
(self.samples,
self.labels) = get_splitted_image_names(self.root_dir)['val']
else:
print('Validation set is not available..')
self.samples, self.labels3d = None, None
self.bboxes = pd.read_csv(
'dataset/dexter_labeling/bbox_dexter+object.csv')
color_intrisics = np.array([[587.45209, 0, 325], [0, 600.67456, 249],
[0, 0, 1]])
color_extrisics = np.array([[0.9999, 0.0034, 0.0161, 19.0473],
[-0.0033, 1.0000, -0.0079, -1.8514],
[-0.0162, 0.0079, 0.9998, -4.7501]])
self.M_color = np.matmul(color_intrisics, color_extrisics)
self.image_size = 128
if self.augment:
self.transform_image = transforms.Compose([
transforms.Resize((self.image_size, self.image_size)),
transforms.ColorJitter(brightness=.25,
contrast=.25,
saturation=.25),
TransformBlur(),
transforms.ToTensor(),
transforms.Normalize([0.485, 0.456, 0.406],
[0.229, 0.224, 0.225])
])
else:
self.transform_image = transforms.Compose([
transforms.Resize((self.image_size, self.image_size)),
transforms.ToTensor(),
transforms.Normalize([0.485, 0.456, 0.406],
[0.229, 0.224, 0.225])
])
self.transform_heatmap = transforms.Compose(
[GaussianBlur(), transforms.ToTensor()])
print('Images in Dexter+Object dataset: {}. Augmentation: {}'.format(
len(self.samples), self.augment))
def __init__(self, config):
self.root_dir = config['path']
self.augment = (bool(config['augment'])
and (config['scope'] == 'train'))
#self.augment = True
if config['scope'] == 'train':
self.samples = get_splitted_image_names(self.root_dir)['train']
elif config['scope'] == 'val':
self.samples = get_splitted_image_names(self.root_dir)['val']
else:
self.samples = get_splitted_image_names(self.root_dir)['test']
##===================
##Used only for model valuation - images with objects and without objects
self.with_objects = None
if self.with_objects == True:
self.samples = [
x for x in self.samples if x.find('/withObject/') >= 0
]
if self.with_objects == False:
self.samples = [
x for x in self.samples if x.find('/noObject/') >= 0
]
##===================
self.image_size = 128
if self.augment:
self.transform_image = transforms.Compose([
transforms.Resize((self.image_size, self.image_size)),
transforms.ColorJitter(brightness=.25,
contrast=.25,
saturation=.25),
TransformBlur(),
transforms.ToTensor(),
transforms.Normalize([0.485, 0.456, 0.406],
[0.229, 0.224, 0.225])
])
else:
self.transform_image = transforms.Compose([
transforms.Resize((self.image_size, self.image_size)),
transforms.ToTensor(),
transforms.Normalize([0.485, 0.456, 0.406],
[0.229, 0.224, 0.225])
])
self.transform_heatmap = transforms.Compose(
[GaussianBlur(), transforms.ToTensor()])
print('Images in GANerated dataset: {}. Augmentation: {}'.format(
len(self.samples), self.augment))
def __init__(self, config):
self.root_dir = config['path']
self.augment = (bool(config['augment']) and (config['scope'] == 'train'))
if config['scope'] == 'train':
(self.samples, self.labels3d) = get_splitted_image_names(self.root_dir)['train']
elif config['scope'] == 'val':
(self.samples, self.labels3d) = get_splitted_image_names(self.root_dir)['val']
else:
print('Validation set is not available..')
self.samples, self.labels3d = None, None
self.image_size = 128
self.intrinsic = np.array([[607.92271, 0, 314.78337],
[0, 607.88192, 236.42484],
[0, 0, 1]
])
if self.augment:
self.transform_image = transforms.Compose([
transforms.Resize((self.image_size, self.image_size)),
transforms.ColorJitter(brightness=.25, contrast=.25, saturation=.25),
TransformBlur(),
transforms.ToTensor(),
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
])
self.transform_rotatate = TransformRotate(rotation_type='2d')
else:
self.transform_image = transforms.Compose([
transforms.Resize((self.image_size, self.image_size)),
transforms.ToTensor(),
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
])
self.transform_heatmap = transforms.Compose([
GaussianBlur(),
transforms.ToTensor()
])
print('Images in Stereo dataset: {}. Augmentation: {}'.format(len(self.samples),self.augment))
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
import argparse
from sklearn.datasets import load_breast_cancer
from torch.utils.data import DataLoader, TensorDataset, Dataset
import torchvision
import numpy as np
import torchvision.transforms as transforms
from utils import GaussianBlur
data_augment = transforms.Compose([
transforms.ToPILImage(),
transforms.RandomHorizontalFlip(),
GaussianBlur(),
transforms.ToTensor()
])
class Net(nn.Module):
def __init__(self):
super(Net, self).__init__()
self.base = torchvision.models.densenet121(pretrained=False)
self.encoder = nn.Sequential(*list(self.base.children())[:-1])
self.linear = nn.Sequential(nn.Linear(1024, 512),
nn.ReLU(inplace=True), nn.Linear(512, 4))
self.avgpool = nn.AdaptiveAvgPool2d(1)
def forward(self, x):
def main_worker(gpu, ngpus_per_node, args):
args.gpu = gpu
if args.gpu is not None:
print("Use GPU: {} for training".format(args.gpu))
print('Model:', args.model)
model = models.__dict__[args.model](pretrained=False)
if args.model.startswith('res'):
model.fc = torch.nn.Linear(in_features=2048,
out_features=args.n_out,
bias=True)
else:
model.classifier = torch.nn.Linear(in_features=1280,
out_features=args.n_out,
bias=True)
# DataParallel will divide and allocate batch_size to all available GPUs
model = torch.nn.DataParallel(model).cuda()
# define loss function (criterion) and optimizer
criterion = nn.CrossEntropyLoss().cuda(args.gpu)
optimizer = torch.optim.Adam(model.parameters(),
args.lr,
weight_decay=args.weight_decay)
cudnn.benchmark = True
if args.resume:
if os.path.isfile(args.resume):
print(args.resume)
checkpoint = torch.load(args.resume)
model.load_state_dict(checkpoint['model_state_dict'])
optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
else:
print("=> no checkpoint found at '{}'".format(args.resume))
savefile_name = args.model + '_augmentstrong_batch256_' + str(
args.augmentation) + '_Y_5_288'
normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
if args.augmentation:
train_dataset = datasets.ImageFolder(
args.data,
transforms.Compose([
transforms.RandomResizedCrop(224, scale=(0.2, 1.)),
transforms.RandomApply(
[transforms.ColorJitter(0.9, 0.9, 0.9, 0.5)], p=0.9),
transforms.RandomGrayscale(p=0.2),
transforms.RandomApply([GaussianBlur([.1, 2.])], p=0.5),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(), normalize
]))
else:
train_dataset = datasets.ImageFolder(
args.data, transforms.Compose([transforms.ToTensor(), normalize]))
train_loader = torch.utils.data.DataLoader(train_dataset,
batch_size=args.batch_size,
shuffle=True,
num_workers=args.workers,
pin_memory=True,
sampler=None)
acc1_list = []
for epoch in range(args.start_epoch, args.epochs):
# train for one epoch
acc1 = train(train_loader, model, criterion, optimizer, epoch, args)
acc1_list.append(acc1)
torch.save(
{
'acc1_list': acc1_list,
'model_state_dict': model.state_dict(),
'optimizer_state_dict': optimizer.state_dict()
}, savefile_name + '_epoch_' + str(epoch) + '.tar')
def __init__(self, config):
self.root_dir = config['path']
self.root_dir_background = config['path_background']
self.augment = (bool(config['augment'])
and (config['scope'] == 'train'))
#self.augment = True
if config['scope'] == 'train':
self.samples = get_splitted_image_names(self.root_dir)['train']
self.backgrounds = get_background_images(
self.root_dir_background)['train']
elif config['scope'] == 'val':
self.samples = get_splitted_image_names(self.root_dir)['val']
self.backgrounds = get_background_images(
self.root_dir_background)['val']
else:
self.samples = get_splitted_image_names(self.root_dir)['test']
self.backgrounds = get_background_images(
self.root_dir_background)['test']
##===================
##Used only for model valuation - images with objects and without objects
self.with_objects = None
if self.with_objects == True:
self.samples = [x for x in self.samples if x.find('_object') >= 0]
if self.with_objects == False:
self.samples = [
x for x in self.samples if x.find('_noobject') >= 0
]
##===================
self.image_size = 128
if self.augment:
self.transform_image = transforms.Compose([
transforms.CenterCrop((480, 480)),
transforms.Resize((self.image_size, self.image_size)),
transforms.ColorJitter(brightness=.25,
contrast=.25,
saturation=.25),
TransformBlur(),
transforms.ToTensor(),
transforms.Normalize([0.485, 0.456, 0.406],
[0.229, 0.224, 0.225])
])
self.transform_scale = TransformScale()
self.transform_rotatate = TransformRotate()
else:
self.transform_image = transforms.Compose([
transforms.CenterCrop((480, 480)),
transforms.Resize((self.image_size, self.image_size)),
transforms.ToTensor(),
transforms.Normalize([0.485, 0.456, 0.406],
[0.229, 0.224, 0.225])
])
self.transform_heatmap = transforms.Compose(
[GaussianBlur(), transforms.ToTensor()])
# projecting 3D coordintated onto color image plane
self.color_intrisics_matrix = np.array([[617.173, 0, 315.453],
[0, 617.173, 242.259],
[0, 0, 1]])
self.color_extrisics_matrix = np.array([[1, 0, 0, 24.7],
[0, 1, 0, -0.0471401],
[0, 0, 1, 3.72045]])
print('Images in SynHands dataset: {}. Augmentation: {}'.format(
len(self.samples), self.augment))
def conceptcombine(model_list, select_idx):
n = 64
labels = []
for six in select_idx:
label_ix = np.eye(10)[six]
label_batch = np.tile(label_ix[None, :], (n, 1))
label = torch.Tensor(label_batch).cuda()
labels.append(label)
im = torch.rand(n, 3, 32, 32).cuda()
im_noise = torch.randn_like(im).detach()
def get_color_distortion(s=1.0):
# s is the strength of color distortion.
color_jitter = transforms.ColorJitter(0.8 * s, 0.8 * s, 0.8 * s,
0.4 * s)
rnd_color_jitter = transforms.RandomApply([color_jitter], p=0.8)
rnd_gray = transforms.RandomGrayscale(p=0.2)
color_distort = transforms.Compose([rnd_color_jitter, rnd_gray])
return color_distort
color_transform = get_color_distortion()
im_size = 32
transform = transforms.Compose([
transforms.RandomResizedCrop(im_size, scale=(0.02, 1.0)),
transforms.RandomHorizontalFlip(), color_transform,
GaussianBlur(kernel_size=5),
transforms.ToTensor()
])
# First get good initializations for sampling
for i in range(10):
for i in range(20):
im_noise.normal_()
im = im + 0.001 * im_noise
# im.requires_grad = True
im.requires_grad_(requires_grad=True)
energy = 0
for model, label in zip(model_list, labels):
energy = model.forward(im, label) + energy
# print("step: ", i, energy.mean())
im_grad = torch.autograd.grad([energy.sum()], [im])[0]
im = im - FLAGS.step_lr * im_grad
im = im.detach()
im = torch.clamp(im, 0, 1)
im = im.detach().cpu().numpy().transpose((0, 2, 3, 1))
im = (im * 255).astype(np.uint8)
ims = []
for i in range(im.shape[0]):
im_i = np.array(transform(Image.fromarray(np.array(im[i]))))
ims.append(im_i)
im = torch.Tensor(np.array(ims)).cuda()
# Then refine the images
for i in range(FLAGS.num_steps):
im_noise.normal_()
im = im + 0.001 * im_noise
# im.requires_grad = True
im.requires_grad_(requires_grad=True)
energy = 0
for model, label in zip(model_list, labels):
energy = model.forward(im, label) + energy
print("step: ", i, energy.mean())
im_grad = torch.autograd.grad([energy.sum()], [im])[0]
im = im - FLAGS.step_lr * im_grad
im = im.detach()
im = torch.clamp(im, 0, 1)
output = im.detach().cpu().numpy()
output = output.transpose((0, 2, 3, 1))
output = output.reshape((-1, 8, 32, 32, 3)).transpose(
(0, 2, 1, 3, 4)).reshape((-1, 32 * 8, 3))
imsave("debug.png", output)
def conceptcombineeval(model_list, select_idx):
dataset = CelebAHQ()
dataloader = torch.utils.data.DataLoader(dataset,
batch_size=64,
shuffle=True,
num_workers=4)
n = 64
labels = []
for six in select_idx:
label_ix = np.eye(2)[six]
label_batch = np.tile(label_ix[None, :], (n, 1))
label = torch.Tensor(label_batch).cuda()
labels.append(None)
def get_color_distortion(s=1.0):
# s is the strength of color distortion.
color_jitter = transforms.ColorJitter(0.8 * s, 0.8 * s, 0.8 * s,
0.4 * s)
rnd_color_jitter = transforms.RandomApply([color_jitter], p=0.8)
rnd_gray = transforms.RandomGrayscale(p=0.2)
color_distort = transforms.Compose([rnd_color_jitter, rnd_gray])
return color_distort
color_transform = get_color_distortion(0.5)
im_size = 128
transform = transforms.Compose([
transforms.RandomResizedCrop(im_size, scale=(0.3, 1.0)),
transforms.RandomHorizontalFlip(), color_transform,
GaussianBlur(kernel_size=5),
transforms.ToTensor()
])
gt_ims = []
fake_ims = []
im = None
for _, data, label in tqdm(dataloader):
gt_ims.extend(list((data.numpy() * 255).astype(np.uint8)))
if im is None:
im = torch.rand(n, 3, 128, 128).cuda()
im_noise = torch.randn_like(im).detach()
# First get good initializations for sampling
for i in range(20):
for i in range(15):
im_noise.normal_()
im = im + 0.001 * im_noise
# im.requires_grad = True
im.requires_grad_(requires_grad=True)
energy = 0
for model, label in zip(model_list, labels):
energy = model.forward(im, None) + energy
# print("step: ", i, energy.mean())
im_grad = torch.autograd.grad([energy.sum()], [im])[0]
im = im - FLAGS.step_lr * im_grad
im = im.detach()
im = torch.clamp(im, 0, 1)
im = im.detach().cpu().numpy().transpose((0, 2, 3, 1))
im = (im * 255).astype(np.uint8)
ims = []
for i in range(im.shape[0]):
im_i = np.array(transform(Image.fromarray(np.array(im[i]))))
ims.append(im_i)
im = torch.Tensor(np.array(ims)).cuda()
# Then refine the images
for i in range(FLAGS.num_steps):
im_noise.normal_()
im = im + 0.001 * im_noise
# im.requires_grad = True
im.requires_grad_(requires_grad=True)
energy = 0
for model, label in zip(model_list, labels):
energy = model.forward(im, None) + energy
im_grad = torch.autograd.grad([energy.sum()], [im])[0]
im = im - FLAGS.step_lr * im_grad
im = im.detach()
im = torch.clamp(im, 0, 1)
im_cpu = im.detach().cpu()
fake_ims.extend(
list((im_cpu.numpy().transpose(
(0, 2, 3, 1)) * 255).astype(np.uint8)))
if len(gt_ims) > 10000:
break
get_fid_score(gt_ims, fake_ims)
fake_ims = np.array(fake_ims)
fake_ims_flat = fake_ims.reshape(fake_ims.shape[0], -1)
std_im = np.std(fake_ims, axis=0).mean()
print("standard deviation of image", std_im)
import pdb
pdb.set_trace()
print("here")