def webcam(style_transform_path, width=1280, height=720):
"""
Captures and saves an image, perform style transfer, and again saves the styled image.
Reads the styled image and show in window.
Saving and loading SHOULD BE eliminated, however this produces too much whitening in
the "generated styled image". This may be caused by the async nature of VideoCapture,
and I don't know how to fix it.
"""
# Device
device = ("cuda" if torch.cuda.is_available() else "cpu")
# Load Transformer Network
print("Loading Transformer Network")
net = transformer.TransformerNetwork()
net.load_state_dict(torch.load(style_transform_path))
net = net.to(device)
print("Done Loading Transformer Network")
# Set webcam settings
cam = cv2.VideoCapture(0)
cam.set(3, width)
cam.set(4, height)
# Main loop
with torch.no_grad():
count = 1
while True:
# Get webcam input
ret_val, img = cam.read()
# Mirror
img = cv2.flip(img, 1)
utils.saveimg(img, str(count) + ".png")
# Free-up unneeded cuda memory
torch.cuda.empty_cache()
# Generate image
content_image = utils.load_image(str(count) + ".png")
content_tensor = utils.itot(content_image).to(device)
generated_tensor = net(content_tensor)
generated_image = utils.ttoi(generated_tensor.detach())
if (PRESERVE_COLOR):
generated_image = utils.transfer_color(content_image,
generated_image)
utils.saveimg(generated_image, str(count + 1) + ".png")
img2 = cv2.imread(str(count + 1) + ".png")
count += 2
# Show webcam
cv2.imshow('Demo webcam', img2)
if cv2.waitKey(1) == 27:
break # esc to quit
# Free-up memories
cam.release()
cv2.destroyAllWindows()
def stylize_folder(style_path, folder_containing_the_content_folder, save_folder, batch_size=1):
"""Stylizes images in a folder by batch
If the images are of different dimensions, use transform.resize() or use a batch size of 1
IMPORTANT: Put content_folder inside another folder folder_containing_the_content_folder
folder_containing_the_content_folder
content_folder
pic1.ext
pic2.ext
pic3.ext
...
and saves as the styled images in save_folder as follow:
save_folder
pic1.ext
pic2.ext
pic3.ext
...
"""
# Device
device = ("cuda" if torch.cuda.is_available() else "cpu")
# Image loader
transform = transforms.Compose([
transforms.ToTensor(),
transforms.Lambda(lambda x: x.mul(255))
])
image_dataset = utils.ImageFolderWithPaths(folder_containing_the_content_folder, transform=transform)
image_loader = torch.utils.data.DataLoader(image_dataset, batch_size=batch_size)
# Load Transformer Network
net = transformer.TransformerNetwork()
net.load_state_dict(torch.load(style_path))
net = net.to(device)
# Stylize batches of images
with torch.no_grad():
for content_batch, _, path in image_loader:
# Free-up unneeded cuda memory
torch.cuda.empty_cache()
# Generate image
generated_tensor = net(content_batch.to(device)).detach()
# Save images
for i in range(len(path)):
generated_image = utils.ttoi(generated_tensor[i])
if (PRESERVE_COLOR):
generated_image = utils.transfer_color(content_image, generated_image)
image_name = os.path.basename(path[i])
utils.saveimg(generated_image, save_folder + image_name)
#stylize()
def __getitem__(self, idx):
inputs = {}
do_flip = False
if self.is_train and random.random() > 0.5:
do_flip = True
# load from disk
left_image, background_image = self.load_images(idx, do_flip=do_flip)
loaded_disparity = self.load_disparity(idx, do_flip=do_flip)
inputs['left_image'] = left_image
inputs['background'] = background_image
inputs['loaded_disparity'] = loaded_disparity
# resize and/or crop
inputs = self.prepare_sizes(inputs)
# match color in background image
inputs['background'] = transfer_color(np.array(inputs['background']),
np.array(inputs['left_image']))
# convert scaleless disparity to pixel disparity
inputs['disparity'] = \
self.process_disparity(inputs['loaded_disparity'],
max_disparity_range=(50, self.max_disparity))
# now generate synthetic stereo image
projection_disparity = inputs['disparity']
right_image = self.project_image(inputs['left_image'],
projection_disparity,
inputs['background'])
# augmentation
right_image = self.augment_synthetic_image(right_image)
# only keep required keys and prepare for network
inputs = {
'image': inputs['left_image'],
'stereo_image': right_image,
'disparity': projection_disparity.astype(float),
'mono_disparity': inputs['loaded_disparity'].astype(float),
}
# finally crop to feed width
for key in ['image', 'stereo_image']:
inputs[key] = inputs[key].crop(
(0, 0, self.feed_width, self.feed_height))
for key in ['disparity', 'mono_disparity']:
inputs[key] = inputs[key][:, :self.feed_width]
self.preprocess(inputs)
return inputs
def webcam(style_transform_path, width=1280, height=720):
"""
Captures and saves an image, perform style transfer, and again saves the styled image.
Reads the styled image and show in window.
"""
# Device
device = "cuda" if torch.cuda.is_available() else "cpu"
# Load Transformer Network
print("Loading Transformer Network")
net = transformer.TransformerNetwork()
net.load_state_dict(torch.load(style_transform_path))
net = net.to(device)
print("Done Loading Transformer Network")
# Set webcam settings
cam = cv2.VideoCapture(0)
cam.set(3, width)
cam.set(4, height)
# Main loop
with torch.no_grad():
while True:
# Get webcam input
ret_val, img = cam.read()
# Mirror
img = cv2.flip(img, 1)
# Free-up unneeded cuda memory
torch.cuda.empty_cache()
# Generate image
content_tensor = utils.itot(img).to(device)
generated_tensor = net(content_tensor)
generated_image = utils.ttoi(generated_tensor.detach())
if PRESERVE_COLOR:
generated_image = utils.transfer_color(img, generated_image)
generated_image = generated_image / 255
# Show webcam
cv2.imshow("Demo webcam", generated_image)
if cv2.waitKey(1) == 27:
break # esc to quit
# Free-up memories
cam.release()
cv2.destroyAllWindows()
def webcam(style_transform_path, width=1280, height=720):
# Device
device = ("cuda" if torch.cuda.is_available() else "cpu")
# Load Transformer Network
print("Loading Transformer Network")
net = transformer.TransformerNetwork()
net.load_state_dict(torch.load(style_transform_path, map_location=device))
net = net.to(device)
print("Done Loading Transformer Network")
# Set webcam settings
cam = cv2.VideoCapture(0)
cam.set(3, width)
cam.set(4, height)
# Main loop
with torch.no_grad():
count = 1
while True:
# Get webcam input
ret_val, img = cam.read()
# AI Mirror
img = cv2.flip(img, 1)
# Free-up unneeded cuda memory
torch.cuda.empty_cache()
# Generate image
content_tensor = utils.itot(img).to(device)
generated_tensor = net(content_tensor)
generated_image = utils.ttoi(generated_tensor.detach())
if (PRESERVE_COLOR):
generated_image = utils.transfer_color(content_image,
generated_image)
img2 = cv2.imdecode(
cv2.imencode(".png", generated_image)[1], cv2.IMREAD_UNCHANGED)
count += 2
# Show webcam
cv2.imshow('Demo webcam', img2)
if cv2.waitKey(1) == 27:
break # esc to quit
# Free-up memories
cam.release()
cv2.destroyAllWindows()
def save(rgb_in, args):
output_path = args.output_dir + args.output_image
utils.save_img_rgb(output_path + args.output_ext, rgb_in)
results = {
'path': output_path + args.output_ext,
}
#preserve original color
if args.preserve_color:
generated_img = utils.load_image_bgr(output_path + args.output_ext)
orig_img = utils.load_image_bgr(args.content_image)
new_img = utils.transfer_color(orig_img, generated_img)
utils.save_img_bgr(output_path + "-color_preserved" + args.output_ext,
new_img)
results[
'color_preserved_path'] = output_path + "-color_preserved" + args.output_ext
return results
def stylize_folder_single(style_path, content_folder, save_folder):
"""
Reads frames/pictures as follows:
content_folder
pic1.ext
pic2.ext
pic3.ext
...
and saves as the styled images in save_folder as follow:
save_folder
pic1.ext
pic2.ext
pic3.ext
...
"""
# Device
device = ("cuda" if torch.cuda.is_available() else "cpu")
# Load Transformer Network
net = transformer.TransformerNetwork()
net.load_state_dict(torch.load(style_path))
net = net.to(device)
# Stylize every frame
images = [img for img in os.listdir(content_folder) if img.endswith(".jpg")]
with torch.no_grad():
for image_name in images:
# Free-up unneeded cuda memory
torch.cuda.empty_cache()
# Load content image
content_image = utils.load_image(content_folder + image_name)
content_tensor = utils.itot(content_image).to(device)
# Generate image
generated_tensor = net(content_tensor)
generated_image = utils.ttoi(generated_tensor.detach())
if (PRESERVE_COLOR):
generated_image = utils.transfer_color(content_image, generated_image)
# Save image
utils.saveimg(generated_image, save_folder + image_name)
def stylize():
# Device
device = ("cuda" if torch.cuda.is_available() else "cpu")
# Load Transformer Network
net = transformer.TransformerNetwork()
net.load_state_dict(torch.load(STYLE_TRANSFORM_PATH))
net = net.to(device)
with torch.no_grad():
while(1):
torch.cuda.empty_cache()
print("Stylize Image~ Press Ctrl+C and Enter to close the program")
content_image_path = input("Enter the image path: ")
content_image = utils.load_image(content_image_path)
starttime = time.time()
content_tensor = utils.itot(content_image).to(device)
generated_tensor = net(content_tensor)
generated_image = utils.ttoi(generated_tensor.detach())
if (PRESERVE_COLOR):
generated_image = utils.transfer_color(content_image, generated_image)
print("Transfer Time: {}".format(time.time() - starttime))
utils.show(generated_image)
utils.saveimg(generated_image, "helloworld.jpg")
def stylize(args):
"""Run the style transfer."""
device = torch.device("cuda" if args.cuda else "cpu")
content_img, style_img = get_eval_images(args)
content_img = content_img.to(device, torch.float)
style_img = style_img.to(device, torch.float)
cnn = models.vgg19(pretrained=True).features.to(device).eval()
cnn_normalization_mean = torch.tensor([0.485, 0.456, 0.406]).to(device)
cnn_normalization_std = torch.tensor([0.229, 0.224, 0.225]).to(device)
print('Building the style transfer model..')
model, style_losses, content_losses = get_style_model_and_losses(
cnn, device, cnn_normalization_mean, cnn_normalization_std, style_img,
content_img)
print(model)
start = time.time()
# input_img = content_img.clone()
# input_img = style_img.clone()
input_img = torch.randn_like(content_img).to(device)
optimizer = get_optimizer(input_img, args)
print('Optimizing..')
run = [0]
losses = []
while run[0] <= args.num_steps:
def closure():
# correct the values of updated input image
input_img.data.clamp_(0, 1)
optimizer.zero_grad()
model(input_img)
style_score = 0
content_score = 0
for sl in style_losses:
style_score += sl.loss
for cl in content_losses:
content_score += cl.loss
style_score *= args.style_weight / 5
content_score *= args.content_weight
if args.reg:
if args.reg.lower() == "l2":
regularizer = args.reg_weight * 0.5 * torch.sum(
torch.norm(input_img))
elif args.reg.lower() == "tv":
tv = TVLoss()
regularizer = args.reg_weight * tv(input_img)
else:
regularizer = 0.0
if args.lap_weight:
lap_loss = args.lap_weight * LapStyleLoss(
device, content_img)(input_img)
else:
lap_loss = 0.0
loss = style_score + content_score + regularizer + lap_loss
losses.append(loss.item())
loss.backward()
run[0] += 1
if run[0] % 50 == 0:
print("run {}:".format(run))
print('Style Loss : {:4f} Content Loss: {:4f}'.format(
style_score.item(), content_score.item()))
print()
return loss
optimizer.step(closure)
# a last correction...
input_img.data.clamp_(0, 1)
print(f"running time {time.time() - start:.2f}s")
image = input_img.cpu().clone(
) # we clone the tensor to not do changes on it
image = image.squeeze(0) # remove the fake batch dimension
image = image.detach().numpy().transpose(1, 2, 0)
image = (image * 255).astype('uint8')
if args.color_prev:
content_img = content_img.cpu().clone().squeeze(
0).detach().numpy().transpose(1, 2, 0)
content_img = (content_img * 255).astype('uint8')
image = transfer_color(content_img, image)
imsave(
f"{args.output_image}/{args.content_image.split('/')[-1].replace('.jpg', '')}_"
f"{args.style_image.split('/')[-1].replace('.jpg', '')}_{args.optimizer}"
f"{'_' + args.reg if args.reg else ''}"
f"{'_lap' if args.lap_weight else ''}{'_color_prev' if args.color_prev else ''}_stylize.jpg",
image)
plt.plot(losses)
plt.title("Losses along iterations")
plt.savefig(
f"{args.output_image}/{args.content_image.split('/')[-1].replace('.jpg', '')}_"
f"{args.style_image.split('/')[-1].replace('.jpg', '')}_{args.optimizer}"
f"{'_' + args.reg if args.reg else ''}"
f"{'_lap' if args.lap_weight else ''}{'_color_prev' if args.color_prev else ''}_losses.jpg"
)
