def forward(self, predicted_img_batch, target_img_batch):
"""Forward function for the DeepLPF loss
:param predicted_img_batch:
:param target_img_batch: Tensor of shape BxCxWxH
:returns: value of loss function
:rtype: float
"""
if predicted_img_batch.shape[2] != target_img_batch.shape[2]:
target_img_batch = target_img_batch.transpose(2, 3)
num_images = target_img_batch.shape[0]
target_img_batch = target_img_batch
ssim_loss_value = Variable(
torch.cuda.FloatTensor(torch.zeros(1, 1).cuda()))
l1_loss_value = Variable(
torch.cuda.FloatTensor(torch.zeros(1, 1).cuda()))
for i in range(0, num_images):
target_img = target_img_batch[i, :, :, :].cuda()
predicted_img = predicted_img_batch[i, :, :, :].cuda()
predicted_img_lab = ImageProcessing.rgb_to_lab(
predicted_img.squeeze(0))
target_img_lab = ImageProcessing.rgb_to_lab(target_img.squeeze(0))
target_img_L_ssim = target_img_lab[0, :, :].unsqueeze(0)
predicted_img_L_ssim = predicted_img_lab[0, :, :].unsqueeze(0)
target_img_L_ssim = target_img_L_ssim.unsqueeze(0)
predicted_img_L_ssim = predicted_img_L_ssim.unsqueeze(0)
ssim_value = self.compute_msssim(predicted_img_L_ssim,
target_img_L_ssim)
ssim_loss_value += (1.0 - ssim_value)
l1_loss_value += F.l1_loss(predicted_img_lab, target_img_lab)
l1_loss_value = l1_loss_value / num_images
ssim_loss_value = ssim_loss_value / num_images
deeplpf_loss = l1_loss_value + 1e-3 * ssim_loss_value
return deeplpf_loss
def upload(request):
color_dict = {9:"r", 8:"rp", 7:"p", 6:"pb", 5:"b", 4:"bg", 3:"g", 2:"gy", 1:"y", 0:"yr"}
if request.method == 'GET':
form = LookUploadForm()
return render_to_response('looks/upload.html',
{'form': form },
context_instance=RequestContext(request))
elif request.method == 'POST':
form = LookUploadForm(request.POST, request.FILES)
if form.is_valid():
lookfile = request.FILES['look_photo']
filename = settings.MEDIA_ROOT + '%s/%s' % ( 'looks', lookfile.name)
#resize file
resizedfile = resize_image(lookfile)
#save to db
look = form.save(commit=False)
if len(form.data['look_description']) > 100:
look.look_short_desc = form.data['look_description'][:97] + '...'
else:
look.look_short_desc = form.data['look_description']
look.look_photo_path = filename
look.look_photo_upload_IP_addr = get_client_ip(request)
look.look_photo.save(filename, resizedfile)
look.save()
#upload file
fd = open(filename, 'wb')
for chunk in resizedfile.chunks():
fd.write(chunk)
fd.close()
#detect img color
colors = ImageProcessing.best_matched_colors(filename)
for c in colors:
color = LookColor()
color.belong_to_look = look
color.color_category = color_dict[c]
color.red = 0
color.green = 0
color.blue = 0
color.hue = 0
color.save()
#(red, green, blue) = average_image_color(filename)
#color = LookColor()
#color.belong_to_look = look
#color.red = red
#color.green = green
#color.blue = blue
#color.hue = hue
#color.color_category = 'b' #choice(LookColor.COLOR_LIST)[0]
#color.save()
return HttpResponseRedirect(reverse('Look_v3_1.look.views.detail', args=(look.id,)))
else:
return render_to_response('looks/upload.html',
{'form': form},
context_instance=RequestContext(request))
def forward(self, predicted_img_batch, target_img_batch, gradient_regulariser):
"""Forward function for the CURL loss
:param predicted_img_batch_high_res:
:param predicted_img_batch_high_res_rgb:
:param target_img_batch: Tensor of shape BxCxWxH
:returns: value of loss function
:rtype: float
"""
num_images = target_img_batch.shape[0]
target_img_batch = target_img_batch
ssim_loss_value = Variable(
torch.cuda.FloatTensor(torch.zeros(1, 1).cuda()))
l1_loss_value = Variable(
torch.cuda.FloatTensor(torch.zeros(1, 1).cuda()))
cosine_rgb_loss_value = Variable(
torch.cuda.FloatTensor(torch.zeros(1, 1).cuda()))
sat_loss_value = Variable(
torch.cuda.FloatTensor(torch.zeros(1, 1).cuda()))
hue_loss_value = Variable(
torch.cuda.FloatTensor(torch.zeros(1, 1).cuda()))
value_loss_value = Variable(
torch.cuda.FloatTensor(torch.zeros(1, 1).cuda()))
a_loss_value = Variable(
torch.cuda.FloatTensor(torch.zeros(1, 1).cuda()))
b_loss_value = Variable(
torch.cuda.FloatTensor(torch.zeros(1, 1).cuda()))
hsv_loss_value = Variable(
torch.cuda.FloatTensor(torch.zeros(1, 1).cuda()))
deep_isp_loss = Variable(
torch.cuda.FloatTensor(torch.zeros(1, 1).cuda()))
rgb_loss_value = Variable(
torch.cuda.FloatTensor(torch.zeros(1, 1).cuda()))
for i in range(0, num_images):
target_img = target_img_batch[i, :, :, :].permute(2, 0, 1).cuda()
predicted_img = predicted_img_batch[i, :, :, :].cuda()
predicted_img_lab = torch.clamp(
ImageProcessing.rgb_to_lab(predicted_img.squeeze(0)), 0, 1)
target_img_lab = torch.clamp(
ImageProcessing.rgb_to_lab(target_img.squeeze(0)), 0, 1)
target_img_hsv = torch.clamp(ImageProcessing.rgb_to_hsv(
target_img.squeeze(0)), 0, 1)
predicted_img_hsv = torch.clamp(ImageProcessing.rgb_to_hsv(
predicted_img.squeeze(0)), 0, 1)
predicted_img_hue = (predicted_img_hsv[0, :, :]*2*math.pi)
predicted_img_val = predicted_img_hsv[2, :, :]
predicted_img_sat = predicted_img_hsv[1, :, :]
target_img_hue = (target_img_hsv[0, :, :]*2*math.pi)
target_img_val = target_img_hsv[2, :, :]
target_img_sat = target_img_hsv[1, :, :]
target_img_L_ssim = target_img_lab[0, :, :].unsqueeze(0)
predicted_img_L_ssim = predicted_img_lab[0, :, :].unsqueeze(0)
target_img_L_ssim = target_img_L_ssim.unsqueeze(0)
predicted_img_L_ssim = predicted_img_L_ssim.unsqueeze(0)
ssim_value = self.compute_msssim(
predicted_img_L_ssim, target_img_L_ssim)
ssim_loss_value += (1.0 - ssim_value)
predicted_img_1 = predicted_img_val * \
predicted_img_sat*torch.cos(predicted_img_hue)
predicted_img_2 = predicted_img_val * \
predicted_img_sat*torch.sin(predicted_img_hue)
target_img_1 = target_img_val * \
target_img_sat*torch.cos(target_img_hue)
target_img_2 = target_img_val * \
target_img_sat*torch.sin(target_img_hue)
p = torch.stack(
(predicted_img_1, predicted_img_2, predicted_img_val), 2)
d = torch.stack((target_img_1, target_img_2, target_img_val), 2)
l1_loss_value += F.l1_loss(predicted_img_lab, target_img_lab)
rgb_loss_value += F.l1_loss(predicted_img, target_img)
hsv_loss_value += F.l1_loss(p, d)
cosine_rgb_loss_value += (1-torch.mean(
torch.nn.functional.cosine_similarity(predicted_img, target_img, dim=0)))
l1_loss_value = l1_loss_value/num_images
rgb_loss_value_hsv = rgb_loss_value/num_images
ssim_loss_value = ssim_loss_value/num_images
cosine_rgb_loss_value = cosine_rgb_loss_value/num_images
hsv_loss_value = hsv_loss_value/num_images
'''
Note the hyperparameters 1e-3, 1e-6 below work well for SamsungS7. They
may need changed for other datasets.
'''
curl_loss = (rgb_loss_value + cosine_rgb_loss_value + l1_loss_value +
hsv_loss_value + 1e-3*ssim_loss_value + 1e-6*gradient_regulariser)/6
return curl_loss
def forward(self, x):
"""Forward function for the CURL layer
:param x: forward the data x through the network
:returns: Tensor representing the predicted image
:rtype: Tensor
"""
'''
This function is where the magic happens :)
'''
x.contiguous() # remove memory holes
feat = x[:, 3:64, :, :]
img = x[:, 0:3, :, :]
torch.cuda.empty_cache()
shape = x.shape
img_clamped = torch.clamp(img, 0, 1)
img_lab = torch.clamp(ImageProcessing.rgb_to_lab(
img_clamped.squeeze(0)), 0, 1)
feat_lab = torch.cat((feat, img_lab.unsqueeze(0)), 1)
x = self.lab_layer1(feat_lab)
del feat_lab
x = self.lab_layer2(x)
x = self.lab_layer3(x)
x = self.lab_layer4(x)
x = self.lab_layer5(x)
x = self.lab_layer6(x)
x = self.lab_layer7(x)
x = self.lab_layer8(x)
x = x.view(x.size()[0], -1)
x = self.dropout1(x)
L = self.fc_lab(x)
img_lab, gradient_regulariser_lab = ImageProcessing.adjust_lab(
img_lab.squeeze(0), L[0, 0:48])
img_rgb = ImageProcessing.lab_to_rgb(img_lab.squeeze(0))
img_rgb = torch.clamp(img_rgb, 0, 1)
feat_rgb = torch.cat((feat, img_rgb.unsqueeze(0)), 1)
x = self.rgb_layer1(feat_rgb)
x = self.rgb_layer2(x)
x = self.rgb_layer3(x)
x = self.rgb_layer4(x)
x = self.rgb_layer5(x)
x = self.rgb_layer6(x)
x = self.rgb_layer7(x)
x = self.rgb_layer8(x)
x = x.view(x.size()[0], -1)
x = self.dropout2(x)
R = self.fc_rgb(x)
img_rgb, gradient_regulariser_rgb = ImageProcessing.adjust_rgb(
img_rgb.squeeze(0), R[0, 0:48])
img_rgb = torch.clamp(img_rgb, 0, 1)
img_hsv = ImageProcessing.rgb_to_hsv(img_rgb.squeeze(0))
img_hsv = torch.clamp(img_hsv, 0, 1)
feat_hsv = torch.cat((feat, img_hsv.unsqueeze(0)), 1)
x = self.hsv_layer1(feat_hsv)
del feat_hsv
x = self.hsv_layer2(x)
x = self.hsv_layer3(x)
x = self.hsv_layer4(x)
x = self.hsv_layer5(x)
x = self.hsv_layer6(x)
x = self.hsv_layer7(x)
x = self.hsv_layer8(x)
x = x.view(x.size()[0], -1)
x = self.dropout3(x)
H = self.fc_hsv(x)
img_hsv, gradient_regulariser_hsv = ImageProcessing.adjust_hsv(
img_hsv, H[0, 0:64])
img_hsv = torch.clamp(img_hsv, 0, 1)
img_residual = torch.clamp(ImageProcessing.hsv_to_rgb(
img_hsv.squeeze(0)), 0, 1)
img = torch.clamp(img + img_residual.unsqueeze(0), 0, 1)
gradient_regulariser = gradient_regulariser_rgb + \
gradient_regulariser_lab+gradient_regulariser_hsv
return img, gradient_regulariser
def evaluate(self, net, epoch=0):
"""Evaluates a network on a specified split of a dataset e.g. test, validation
:param net: PyTorch neural network data structure
:param data_loader: an instance of the DataLoader class for the dataset of interest
:param split_name: name of the split e.g. "test", "validation"
:param log_dirpath: logging directory
:returns: average loss, average PSNR
:rtype: float, float
"""
psnr_avg = 0.0
ssim_avg = 0.0
examples = 0
running_loss = 0
num_batches = 0
batch_size = 1
out_dirpath = self.log_dirpath + "/" + self.split_name.lower()
if not os.path.isdir(out_dirpath):
os.mkdir(out_dirpath)
# switch model to evaluation mode
net.eval()
net.cuda()
for batch_num, data in enumerate(self.data_loader, 0):
input_img_batch, output_img_batch, name = Variable(data['input_img'], requires_grad=False,
volatile=True).cuda(), Variable(data['output_img'],
requires_grad=False,
volatile=True).cuda(), \
data['name']
input_img_batch = input_img_batch.unsqueeze(0)
for i in range(0, input_img_batch.shape[0]):
img = input_img_batch[i, :, :, :]
img = torch.clamp(img, 0, 1)
net_output_img_example = net(img)
loss = self.criterion(net_output_img_example[:, 0:3, :, :],
output_img_batch[:, 0:3, :, :])
input_img_example = (
input_img_batch.cpu().data[0, 0:3, :, :].numpy() *
255).astype('uint8')
output_img_batch_numpy = output_img_batch.squeeze(
0).data.cpu().numpy()
output_img_batch_numpy = ImageProcessing.swapimdims_3HW_HW3(
output_img_batch_numpy)
output_img_batch_rgb = output_img_batch_numpy
output_img_batch_rgb = ImageProcessing.swapimdims_HW3_3HW(
output_img_batch_rgb)
output_img_batch_rgb = np.expand_dims(output_img_batch_rgb,
axis=0)
net_output_img_example_numpy = net_output_img_example.squeeze(
0).data.cpu().numpy()
net_output_img_example_numpy = ImageProcessing.swapimdims_3HW_HW3(
net_output_img_example_numpy)
net_output_img_example_rgb = net_output_img_example_numpy
net_output_img_example_rgb = ImageProcessing.swapimdims_HW3_3HW(
net_output_img_example_rgb)
net_output_img_example_rgb = np.expand_dims(
net_output_img_example_rgb, axis=0)
net_output_img_example_rgb = np.clip(
net_output_img_example_rgb, 0, 1)
running_loss += loss.data[0]
examples += batch_size
num_batches += 1
psnr_example = ImageProcessing.compute_psnr(
output_img_batch_rgb.astype(np.float32),
net_output_img_example_rgb.astype(np.float32), 1.0)
ssim_example = ImageProcessing.compute_ssim(
output_img_batch_rgb.astype(np.float32),
net_output_img_example_rgb.astype(np.float32))
psnr_avg += psnr_example
ssim_avg += ssim_example
if batch_num > 30:
'''
We save only the first 30 images down for time saving
purposes
'''
continue
else:
output_img_example = (output_img_batch_rgb[0, 0:3, :, :] *
255).astype('uint8')
net_output_img_example = (
net_output_img_example_rgb[0, 0:3, :, :] *
255).astype('uint8')
plt.imsave(
out_dirpath + "/" + name[0].split(".")[0] + "_" +
self.split_name.upper() + "_" + str(epoch + 1) + "_" +
str(examples) + "_PSNR_" +
str("{0:.3f}".format(psnr_example)) + "_SSIM_" +
str("{0:.3f}".format(ssim_example)) + ".jpg",
ImageProcessing.swapimdims_3HW_HW3(
net_output_img_example))
del net_output_img_example_numpy
del net_output_img_example_rgb
del output_img_batch_rgb
del output_img_batch_numpy
del input_img_example
del output_img_batch
psnr_avg = psnr_avg / num_batches
ssim_avg = ssim_avg / num_batches
logging.info('loss_%s: %.5f psnr_%s: %.3f ssim_%s: %.3f' %
(self.split_name, (running_loss / examples),
self.split_name, psnr_avg, self.split_name, ssim_avg))
loss = (running_loss / examples)
return loss, psnr_avg, ssim_avg