def display(self, i):
input_tensor, output_tensor = self.__getitem__(i)
input_tensor = self.inverse_transform(input_tensor)
output_tensor = self.inverse_transform(output_tensor)
input_image = tensor2image(input_tensor)
output_image = tensor2image(output_tensor)
display_image_side(input_image, output_image, show=True)
def display(self, i):
input_tensor, output_tensor = self.__getitem__(i)
input_image = tensor2image(
self.inverse_transform(input_tensor)).astype(np.uint8)
output_image = tensor2image(
self.inverse_transform(output_tensor)).astype(np.uint8)
fig, ax = plt.subplots(1, 2)
ax[0].imshow(input_image)
ax[1].imshow(output_image)
plt.show()
def infer(netG_B2A, im_path):
transforms_ = [
transforms.Resize(args.size, Image.BICUBIC),
transforms.ToTensor(),
transforms.Normalize((0.5, ), (0.5, ))
]
transforms_ = transforms.Compose(transforms_)
real_im = Image.open(im_path)
data = transforms_(real_im)
if args.cuda:
data = data.cuda()
data = data.unsqueeze(0)
data = data[:, :args.nc, :, :]
# do the forward pass
fake_data = netG_B2A(data)
fake_im = tensor2image(
fake_data.detach(),
np.array([0.5 for _ in range(args.nc)], dtype='float32'),
np.array([0.5 for _ in range(args.nc)], dtype='float32'))
fake_im = np.transpose(fake_im, (1, 2, 0))
fake_im = fake_im[:, :, ::-1]
out_path = os.path.join(args.output_dir,
im_path[len(args.dataset_root_path) + 1:])
cv2.imwrite(out_path, fake_im)
return
def visualize_predictions(test_loader, G, step, writer, mask=False):
if mask:
n_samples = len(test_loader.dataset)
input_batch_numpy = []
prediction_batch_numpy = []
for j in range(10):
random_index = int(np.random.random() * n_samples)
inputs, _ = test_loader.dataset[random_index]
inputs = inputs.unsqueeze(dim=0)
predictions, mask, copied, synthesized, weighted_synthesized = G(
inputs)
images_numpy = [
255 * ((0.5 * utils.tensor2image(input_tensor.cpu())[0]) + 0.5)
for input_tensor in [
inputs, predictions, mask, copied, synthesized,
weighted_synthesized
]
]
figure = utils.display_images(images_numpy)
writer.add_figure(f'translations_{step}_{j}',
figure,
global_step=step)
else:
n_samples = len(test_loader.dataset)
input_batch_numpy = []
prediction_batch_numpy = []
for j in range(10):
random_index = int(np.random.random() * n_samples)
inputs, _ = test_loader.dataset[random_index]
inputs = inputs.unsqueeze(dim=0)
predictions = G(inputs).cpu()
inputs_numpy = utils.tensor2image(inputs.cpu())[0]
predictions_numpy = utils.tensor2image(predictions.cpu())[0]
inputs_numpy = (inputs_numpy * 0.5 + 0.5) * 255
predictions_numpy = (predictions_numpy * 0.5 + 0.5) * 255
figure = utils.display_image_side(inputs_numpy, predictions_numpy)
writer.add_figure(f'translations_{step}_{j}',
figure,
global_step=step)
def save_images(self, images):
for name, image in images.items():
print(self.image_name)
save_dir = os.path.join(self.args.save_dir, 'result_images')
if not os.path.exists(save_dir):
os.makedirs(save_dir)
save_filename = '%s%s_epoch%s.png' % (
name, self.image_name[0].split(".")[0], self.args.load_epoch
) # batch_size=1
save_path = os.path.join(save_dir, save_filename)
image = tensor2image(image)
pil_image = Image.fromarray(image.squeeze())
pil_image.save(save_path)
def test(self, filepath, generate_images=True):
# load training info
if filepath is not None:
self._load(filepath)
if self.args.eval_mode:
self.model.eval()
else:
self.model.train()
with torch.no_grad():
mse = 0.0
for iteration, batch in enumerate(self.test_dataloader):
real_a, real_b = batch[0].to(self.device), batch[1].to(self.device)
pred_b = self.model.generate(real_a)
mse += self.criterion_mse(pred_b, real_b).item()/len(self.test_dataloader)
if generate_images:
for i in range(pred_b.shape[0]):
im = tensor2image(pred_b[i])
im.save(self.args.results_dir+'/'+str(self.args.batch_size*iteration+i+1)+'.jpg')
print('Epoch[{0}] - MSE: {1}'.format(
self.epoch, mse
))
def gen_images(nets, dataloader, opt):
def _vbound(divider=4):
c, h, w = opt.input_nc, opt.size, opt.size
out_shape = c, int(h // divider), int(w * 10)
return torch.full(out_shape, fill_value=0.8)
def _hbound(divider=4):
"""
Args:
shape: Either size = 2 for gray image, or 3 for RGB image.
"""
c, h, w = opt.input_nc, opt.size, opt.size
out_shape = c, h, int(w // divider)
return torch.full(out_shape, fill_value=0.8)
netG_A2B, netG_B2A = nets
# Inputs & targets memory allocation
Tensor = torch.cuda.FloatTensor if opt.cuda else torch.Tensor
input_A = Tensor(opt.batchSize, opt.input_nc, opt.size, opt.size)
input_B = Tensor(opt.batchSize, opt.output_nc, opt.size, opt.size)
mask_scales = [str(scale)
for scale in get_mask_scales(opt)] # ['0.5', '0.8', '1.0']
total = min(len(dataloader), opt.max_n)
for i, batch in tqdm(enumerate(dataloader), total=total):
if i >= opt.max_n:
break
# Set model input
real_A = Variable(input_A.copy_(batch['A']))
real_B = Variable(input_B.copy_(batch['B']))
if not opt.use_mask:
# fake_B = 0.5 * (netG_A2B(real_A, mask=mask).data + 1.0)
# fake_A = 0.5 * (netG_B2A(real_B, mask=mask).data + 1.0)
fake_B = tensor2image(netG_A2B(real_A, mask=None))
fake_A = tensor2image(netG_B2A(real_B, mask=None))
save_image(fake_A, f'output/{opt.run_id}/A/{i+1:04d}.png')
save_image(fake_B, f'output/{opt.run_id}/B/{i+1:04d}.png')
continue
masks = [
gen_random_mask(size, shape=real_A.shape) for size in mask_scales
]
for scale, mask in zip(mask_scales, masks):
fake_B = tensor2image(netG_A2B(real_A, mask=mask))
fake_A = tensor2image(netG_B2A(real_B, mask=mask))
save_image(
fake_A,
f'output/{opt.run_id}/A/scale={float(scale):.2f}/{i+1:04d}.png'
)
save_image(
fake_B,
f'output/{opt.run_id}/B/scale={float(scale):.2f}/{i+1:04d}.png'
)
if not opt.grid:
continue
grid = [_vbound()]
for mask in masks:
# Generate output
fake_B = netG_A2B(real_A, mask=mask)
fake_A = netG_B2A(real_B, mask=mask)
recovered_A = netG_B2A(fake_B, mask=mask)
recovered_B = netG_A2B(fake_A, mask=mask)
same_A = netG_B2A(real_A, mask=mask)
same_B = netG_A2B(real_B, mask=mask)
images = [
_hbound(),
mask[0],
# torch.ones_like(real_A[0]), # separator
_hbound(),
tensor2image(real_A[0]),
tensor2image(fake_B[0]),
tensor2image(recovered_A[0]),
tensor2image(same_A[0]),
_hbound(),
tensor2image(real_B[0]),
tensor2image(fake_A[0]),
tensor2image(recovered_B[0]),
tensor2image(same_B[0]),
_hbound(),
]
grid.append(torch.cat(images, dim=2)) # cat on width
grid.append(_vbound())
grid = torch.cat(grid, dim=1) # cat on height
# Save image files
save_image(grid, f'output/{opt.run_id}/grid/{i+1:04d}.png')
recovered_ABA_test = netG_B2A(fake_AB_test)
recovered_BAB_test = netG_A2B(fake_BA_test)
test_product_name_A = batch_['img_A'][0]
res_A = test_product_name_A.split("/")[-1]
res_A = res_A.split(".")[0]
# fn = os.path.join(output_path_, str(j))
fn_A = os.path.join(output_path_, res_A)
test_product_name_B = batch_['img_B'][0]
res_B = test_product_name_B.split("/")[-1]
res_B = res_B.split(".")[0]
# fn = os.path.join(output_path_, str(j))
fn_B = os.path.join(output_path_, res_B)
# A_test = np.hstack([tensor2image(real_A_test[0]), tensor2image(fake_AB_test[0]),
# tensor2image(recovered_ABA_test[0])])
# B_test = np.hstack([tensor2image(real_B_test[0]), tensor2image(fake_BA_test[0]),
# tensor2image(recovered_BAB_test[0])])
A_test = np.hstack([tensor2image(fake_AB_test[0])])
B_test = np.hstack([tensor2image(fake_BA_test[0])])
imageio.imwrite(fn_A + '_A.jpg', A_test)
imageio.imwrite(fn_B + '_B.jpg', B_test)
else:
break
for j, batch_ in enumerate(dataloader_test):
if j < 60:
real_A_test = Variable(input_A.copy_(batch_['A']))
real_B_test = Variable(input_B.copy_(batch_['B']))
fake_AB_test = netG_A2B(real_A_test)
fake_BA_test = netG_B2A(real_B_test)
recovered_ABA_test = netG_B2A(fake_AB_test)
recovered_BAB_test = netG_A2B(fake_BA_test)
fn = os.path.join(output_path_, str(j))
A_test = np.hstack([
tensor2image(real_A_test[0]),
tensor2image(fake_AB_test[0]),
tensor2image(recovered_ABA_test[0])
])
B_test = np.hstack([
tensor2image(real_B_test[0]),
tensor2image(fake_BA_test[0]),
tensor2image(recovered_BAB_test[0])
])
imageio.imwrite(fn + '_A.jpg', A_test)
imageio.imwrite(fn + '_B.jpg', B_test)
# imageio.imwrite(fn + '.A.jpg', tensor2image(real_A_test[0]))
# imageio.imwrite(fn + '.B.jpg', tensor2image(real_B_test[0]))
# imageio.imwrite(fn + '.BA.jpg', tensor2image(fake_BA_test[0]))
# imageio.imwrite(fn + '.AB.jpg', tensor2image(fake_AB_test[0]))
safe_mkdirs(output_path_)
for j, batch_ in enumerate(dataloader_test):
if j < 60:
real_A_test = Variable(input_A.copy_(batch_['A']))
real_B_test = Variable(input_B.copy_(batch_['B']))
fake_AB_test = netG_A2B(real_A_test)
fake_BA_test = netG_B2A(real_B_test)
recovered_ABA_test = netG_B2A(fake_AB_test)
recovered_BAB_test = netG_A2B(fake_BA_test)
fn = os.path.join(output_path_, str(j))
A_test = np.hstack([tensor2image(real_A_test[0]), tensor2image(fake_AB_test[0]),
tensor2image(recovered_ABA_test[0])])
B_test = np.hstack([tensor2image(real_B_test[0]), tensor2image(fake_BA_test[0]),
tensor2image(recovered_BAB_test[0])])
imageio.imwrite(fn + '_A.jpg', A_test)
imageio.imwrite(fn + '_B.jpg', B_test)
# imageio.imwrite(fn + '.A.jpg', tensor2image(real_A_test[0]))
# imageio.imwrite(fn + '.B.jpg', tensor2image(real_B_test[0]))
# imageio.imwrite(fn + '.BA.jpg', tensor2image(fake_BA_test[0]))
# imageio.imwrite(fn + '.AB.jpg', tensor2image(fake_AB_test[0]))
# imageio.imwrite(fn + '.ABA.jpg', tensor2image(recovered_ABA_test[0]))
# imageio.imwrite(fn + '.BAB.jpg', tensor2image(recovered_BAB_test[0]))
else:
break
def training_mode(model_name):
batch_size = 25
train_loader = torch.utils.data.DataLoader(datasets.MNIST(
'../data',
train=True,
download=True,
transform=transforms.Compose(
[transforms.ToTensor(),
transforms.Normalize((0.5, ), (0.5, ))])),
batch_size=batch_size,
shuffle=True)
test_loader = torch.utils.data.DataLoader(datasets.MNIST(
'../data',
train=False,
transform=transforms.Compose(
[transforms.ToTensor(),
transforms.Normalize((0.5, ), (0.5, ))])),
batch_size=batch_size,
shuffle=True)
# Prepare unpaired image translation dataset
X = np.array(
[utils.tensor2image(image_batch) for image_batch, _ in train_loader])
X = X.reshape(
(X.shape[0] * X.shape[1], X.shape[2], X.shape[3], X.shape[4]))
Y = np.array(X) * -1
np.random.shuffle(Y)
# utils.display_image(X[0])
# utils.display_image(Y[0])
X_train, X_test, Y_train, Y_test = train_test_split(X, Y)
unsupervised_train_loader = data.DataLoader(data.TensorDataset(
utils.image2tensor(X_train), utils.image2tensor(Y_train)),
batch_size=1)
unsupervised_test_loader = data.DataLoader(data.TensorDataset(
utils.image2tensor(X_test), utils.image2tensor(Y_test)),
batch_size=1)
x, y = next(iter(unsupervised_train_loader))
c = utils.tensor2image(x.cpu()[0:1])
utils.display_image(c[0])
# Initialize network and optimizers
G = Generator(3)
F = Generator(3)
D_X = Discriminator(3)
D_Y = Discriminator(3)
G_optimizer = optim.Adam(G.parameters(), lr=1e-3)
F_optimizer = optim.Adam(F.parameters(), lr=1e-3)
D_X_optimizer = optim.Adam(D_X.parameters(), lr=1e-3)
D_Y_optimizer = optim.Adam(D_Y.parameters(), lr=1e-3)
writer = SummaryWriter(model_name)
# Train network
for epoch in range(1):
train(epoch, G, F, D_X, D_Y, G_optimizer, F_optimizer, D_X_optimizer,
D_Y_optimizer, unsupervised_train_loader, writer, test_loader)
save_model(f'../models/{model_name}_G.pt', G, F,
f'../models/{model_name}_F.pt')
def test(netG_B2A, netGaze):
if netG_B2A is not None:
netG_B2A.eval()
netGaze.eval()
out_vid = None
pred_all = np.array([], dtype='int64')
target_all = np.array([], dtype='int64')
for idx, (data, target) in enumerate(test_loader):
if args.cuda:
data, target = data[:, :args.nc, :, :].cuda(), target.cuda()
data, target = Variable(data), Variable(target)
# do the forward pass
if netG_B2A is not None:
data = gaze2gan(data, test_loader.dataset.mean[0:args.nc],
test_loader.dataset.std[0:args.nc])
fake_data = netG_B2A(data)
fake_data = gan2gaze(fake_data,
test_loader.dataset.mean[0:args.nc],
test_loader.dataset.std[0:args.nc])
scores, masks = netGaze(fake_data.repeat(1, int(3 / args.nc), 1,
1))
else:
data = gaze2gan(data, test_loader.dataset.mean[0:args.nc],
test_loader.dataset.std[0:args.nc])
fake_data = gan2gaze(data, test_loader.dataset.mean[0:args.nc],
test_loader.dataset.std[0:args.nc])
scores, masks = netGaze(fake_data.repeat(1, int(3 / args.nc), 1,
1))
if args.save_viz:
im = tensor2image(
data.repeat(1, int(3 / args.nc), 1, 1).detach(),
np.array([0.5 for _ in range(args.nc)], dtype='float32'),
np.array([0.5 for _ in range(args.nc)], dtype='float32'))
im = np.transpose(im, (1, 2, 0))
im = im[:, :, ::-1]
im_wo = tensor2image(
fake_data.repeat(1, int(3 / args.nc), 1, 1).detach(),
test_loader.dataset.mean, test_loader.dataset.std)
im_wo = np.transpose(im_wo, (1, 2, 0))
im_wo = im_wo[:, :, ::-1]
im_out = create_viz(im, im_wo, scores, masks, activity_classes)
if out_vid is None:
out_vid = cv2.VideoWriter(
os.path.join(args.output_dir, 'out.avi'),
cv2.VideoWriter_fourcc(*'XVID'), 5,
(im_out.shape[1], im_out.shape[0]))
out_vid.write(im_out)
scores = scores.view(-1, args.num_classes)
pred = scores.data.max(1)[
1] # got the indices of the maximum, match them
print('Done with image {} out {}...'.format(
min(args.batch_size * (idx + 1), len(test_loader.dataset)),
len(test_loader.dataset)))
pred_all = np.append(pred_all, pred.cpu().numpy())
target_all = np.append(target_all, target.cpu().numpy())
micro_acc, macro_acc = plot_confusion_matrix(target_all, pred_all,
merged_activity_classes,
args.output_dir)
print("\n------------------------")
print("Micro-average accuracy = {:.2f}%".format(micro_acc))
print("Macro-average accuracy = {:.2f}%\n------------------------".format(
macro_acc))
with open(os.path.join(args.output_dir, "logs.txt"), "a") as f:
f.write("\n------------------------\n")
f.write("Micro-average accuracy = {:.2f}%\n".format(micro_acc))
f.write("Macro-average accuracy = {:.2f}%\n------------------------\n".
format(macro_acc))
if args.save_viz:
out_vid.release()
return micro_acc, macro_acc