def frame_format(self, paths, hs, he, ws, we):
"""
NOTE THAT WE NEED DIFFERENT SIZE FRAME AT DIFFERENT PYRAMID LEVEL SO WE RESIZE HERE
WE ARE GOING TO USE RANDOM CROP TECHNIQUES
"""
t0 = transforms.Compose([ToTensor()])
images = [*map(lambda x:Image.open(x),paths)]
R = lambda x, t: t(x).unsqueeze(1)[:,:,hs:he, ws:we]
frames = [*map(lambda img: R(img, t0),images)]
final_frame = torch.cat(frames, 1)
if not self.is_test:
f1 = []
f2 = []
f3 = []
for img in frames:
tenim = ToPILImage()(img[:,0,:])
f1im = ToTensor()(tenim.resize((256, 108)))
f2im = ToTensor()(tenim.resize((128, 54)))
f3im = ToTensor()(tenim.resize((64, 27)))
f1.append(f1im.unsqueeze(1))
f2.append(f2im.unsqueeze(1))
f3.append(f3im.unsqueeze(1))
pyra1_frame = torch.cat(f1, 1)
pyra2_frame = torch.cat(f2, 1)
laten_frame = torch.cat(f3, 1)
return final_frame, pyra1_frame, pyra2_frame, laten_frame
else:
return final_frame
def compute_metrics(net, valLoader, upscaling, multiscale):
net.eval()
psnr_low = 0
psnr_super = 0
ssim_low = 0
ssim_super = 0
cpu = torch.device("cpu")
cuda = torch.device("cuda:0")
with torch.no_grad():
for i, batch in enumerate(valLoader):
highres, lowres = batch
lowres = lowres.to(cuda)
highres = highres.to(cuda)
if (multiscale):
superres = net(lowres, upscaling)
else:
superres = net(lowres)
lowres = lowres.to(cpu)
lowres = ToImage()(lowres.view(lowres.size()[1:]))
lowres = lowres.resize(
(lowres.size[0] * upscaling, lowres.size[1] * upscaling),
Image.BICUBIC)
lowres = ToTensor()(lowres)
lowres = lowres.view([1] + list(lowres.size()))
lowres = lowres.to(cuda)
psnr_super += compute_psnr(superres, highres)
psnr_low += compute_psnr(lowres, highres)
ssim_super += compute_msssim(superres, highres)
print("|")
ssim_low += compute_msssim(lowres, highres)
return psnr_low / len(valLoader), psnr_super / len(valLoader), \
ssim_low / len(valLoader), ssim_super / len(valLoader)
def visualize(self, layer, filt, lr=1e-2, opt_steps=36):
sz = self.size
img = Image.fromarray(
np.uint8(np.random.uniform(150, 180, (sz, sz, 3))))
activations = SaveFeatures(list(self.target.children())[layer])
gaussian_filter = get_gaussian_kernel()
self.model.zero_grad()
for outer in tqdm(range(self.upscaling_steps), leave=False):
img_var = torch.unsqueeze(ToTensor()(img),
0).cuda(ORDINAL).requires_grad_(True)
img_var.requires_grad_(True).cuda(ORDINAL)
optimizer = torch.optim.Adam([img_var], lr=lr, weight_decay=1e-6)
pbar = tqdm(range(opt_steps), leave=False)
for n in pbar:
optimizer.zero_grad()
self.model(img_var)
loss = -activations.features[
0, filt].mean() + 0.00 * torch.norm(img_var)
loss.backward()
pbar.set_description(f'Loss: {loss.item()}')
optimizer.step()
sz = int(sz * self.upscaling_factor)
img = ToPILImage()(img_var.squeeze(0))
if outer != self.upscaling_steps:
img = img.resize((sz, sz))
img = img.filter(ImageFilter.BoxBlur(2))
self.output = img.copy()
self.save(layer, filt)
activations.close()
def ConvertPhoneImageToSLMImage(image_phone, SLM_to_phone_zoom_ratio):
"""
Resize the phone display image to be displayed on SLM, such that the phone
display imaged onto SLM matches exactly the size and position of the one on
the SLM. Returns a PIL image to be saved.
image_phone: 2D numpy array with dtype uint8
SLM_to_phone_zoom_ratio: the number of pixels on the SLM divided the number
of pixels on the phone such that the pattern of their image are of the same
size
"""
r = SLM_to_phone_zoom_ratio # equals the number of pixels of a line on SLM divided the that of the same line on the phone.
(h, w) = (1152, 1920) # SLM size
#im = image_phone.transpose() # transpose the image from 1920x1080 to 1080x1920
#im = im[::-1, :] # flip left and right
# Pad the phone image to 1920x1600 to give enough margin for cropping later
if isinstance(image_phone, (np.ndarray, )):
#im = np.zeros((1920, 1600), dtype="uint8")
#im[:, 260:(260+1080)] = image_phone
im = np.zeros((h, w), dtype="uint8")
im[36:(36 + 1080), :] = image_phone.transpose()
imPIL = PIL.Image.fromarray(im) # Convert from numpy to PIL image
elif isinstance(image_phone, (torch.Tensor, )):
#im = torch.zeros(1920, 1600, dtype=torch.uint8)
#im[:, 260:(260+1080)] = image_phone
im = torch.zeros(h, w, dtype=torch.uint8)
im[36:(36 + 1080), :] = image_phone.t()
imPIL = ToPILImage()(im) # Convert from numpy to PIL image
# Adjust pixel values according to SLM LUT
# im[im == 0] = 155 # pixel value for the max transmission intensity on SLM
# im[im == 255] = 77 # pixel value for the min transmission intensity on SLM
# Resize the SLM image
new_width = round(
float(im.shape[1]) * r / 2
) * 2 # Calculate the resized image size, round to even number of pixels.
new_height = round(float(im.shape[0]) * r / 2) * 2
imPIL = imPIL.resize((new_width, new_height)) # Resize the image
# Transform the image on the SLM to have the same spatial orientation as that on the phone.
imPIL = PIL.ImageOps.flip(imPIL)
imPIL = PIL.ImageOps.mirror(imPIL)
# Create a PIL image of the exactly same size as required by the SLM.
rowOffset = int(
(imPIL.size[1] - h) /
2) # Calculate the boundary coordinate for cropping the image
colOffset = int((imPIL.size[0] - w) /
2) # PIL.size sequence is reversed from numpy.shape!!!!!!!
image_SLM = imPIL.crop(
(colOffset, rowOffset, colOffset + w,
rowOffset + h)) # Crop the image to fit SLM size (1152x1920)
return image_SLM
def VideoDetection(model, img, size=(416,416), threshold=.25, IoU=.5, dataset='coco'):
image = ToPILImage()(img)
x = ToTensor()(image.resize(size))
preds = model.predict(x, threshold)[0]
preds = NonMaxSuppression(preds, IoU)
if len(preds) != 0:
scores, boxes, classes = YoloHead(preds)
image = DrawBoxes(image, scores, boxes, classes, dataset)
return image
def generate_image(self):
noise = self.get_noise()
image_tensor = self.model(noise)
image_tensor = (image_tensor.cpu() + 1) / 2
image = ToPILImage()(image_tensor[0])
image = image.resize((128, 128), resample=0)
with io.BytesIO() as f:
image.save(f, format='JPEG')
img_byte_arr = f.getvalue()
return img_byte_arr
def PictureDetection(model, img, size=(416,416), threshold=.25, IoU=.5, dataset='coco'):
if type(img) is str:
image = Image.open(img).convert('RGB')
x = toTensor(image, size)
else:
image = ToPILImage()(img)
x = ToTensor()(image.resize(size))
preds = model.predict(x, threshold)[0]
if len(preds) != 0:
preds = NonMaxSuppression(preds, IoU)
scores, boxes, classes = YoloHead(preds)
image = DrawBoxes(image, scores, boxes, classes, dataset)
return image
def evaluate_model(config, model_folder, image_folder, output_path, save_json=True):
"""
Evaluates a model by comparing input images with output images
:param config: the model configuration
:param model_folder: folder of the saved model
:param image_folder: path images used to evaluate the model
:param output_path: path where anonymized images should be stored
:return: list of distances
"""
image_folder = Path(image_folder)
output_path = Path(output_path)
model = config.model(**config.model_params)
model.load_model(Path(model_folder))
extractor = FaceExtractor(margin=0.05, mask_factor=10)
print("The authors of the package recommend 0.6 as max distance for the same person.")
scores = {}
for image_file in image_folder.iterdir():
if image_file.is_dir():
continue
print('#' * 10)
print('Processing image:', image_file.name)
input_image = Image.open(image_file)
extracted_face, extracted_info = extractor(input_image)
if extracted_face is None:
print('Face could not be extracted')
continue
face_out = model.anonymize(extracted_face, extracted_info).squeeze(0)
face_out = ToPILImage()(face_out.cpu().detach())
face_out = face_out.resize(extracted_face.size, resample=BICUBIC)
try:
face_out.save(output_path / ('anonymized_' + image_file.name.__str__()))
score, sim, emo = Evaluator.evaluate_image_pair(extracted_face, face_out)
scores[image_file.name] = {'score': score, 'sim': sim, 'emo': emo}
except Exception as ex:
print(ex)
continue
print('Current image score:', scores[image_file.name])
if save_json:
with open(output_path / 'scores.json', 'w') as f:
json.dump(scores, f)
return scores
def callback(self, oimg):
try:
#if you want to save images and labels ,please uncomment following codes(No.1 to No.4).
#NO.1 #write_image_name = "image_" + str(self.count) + ".jpg"
#No.2 #write_label_name = "label_" + str(self.count) + ".jpg"
oimg_b = bytes(oimg.data)
np_arr = np.fromstring(oimg_b, np.uint8)
img = cv2.imdecode(np_arr, cv2.IMREAD_COLOR)
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
#No.3 #cv2.imwrite("/home/amsl/images/output/seg_pub/image/" + write_image_name, img)
img_size = img.shape
image = PIL_Image.fromarray(img)
image = image.resize((1024,512),PIL_Image.NEAREST)
image = ToTensor()(image)
image = torch.Tensor(np.array([image.numpy()]))
image = image.cuda()
input_image = Variable(image)
with torch.no_grad():
output_image = self.model(input_image)
label = output_image[0].max(0)[1].byte().cpu().data
label_color = Colorize()(label.unsqueeze(0))
label_pub = ToPILImage()(label_color)
label_pub = label_pub.resize((img_size[1],img_size[0]),PIL_Image.NEAREST)
label_pub = np.asarray(label_pub)
#show label.
#plt.imshow(label_pub)
#plt.pause(0.001)
#No.4 #cv2.imwrite("/home/amsl/images/output/seg_pub/label/" + write_label_name, label_pub)
#self.pub_seg.publish(self.bridge.cv2_to_imgmsg(label_pub, "bgr8"))
self.pub_seg.publish(self.bridge.cv2_to_imgmsg(label_pub, "rgb8"))
print("published")
self.count += 1
except CvBridgeError as e:
print(e)
def visualize_components(epoch, model, save_path):
if not os.path.exists(save_path):
os.makedirs(save_path)
images = []
for idx, component in enumerate(model.components):
image = ToPILImage()(component.cpu()).convert("RGB")
image = image.resize((64, 64))
# image.save(f"{save_path}/{epoch}_{idx}.png")
image = ImageOps.expand(image, border=4, fill='black')
images.append(image)
result = get_concat_h(images)
result.save(f"{save_path}/{epoch}.png")
def augment_images(images, targets, preds):
augmented_images = []
for img, target, pred in zip(images, targets, preds):
img = ToPILImage()(img.cpu())
img1 = ImageDraw.Draw(img)
for bb in target['boxes']:
bb = bb.cpu().numpy()
img1.rectangle([bb[0], bb[1], bb[2], bb[3]],
fill=None,
outline="green",
width=5)
for bb in pred['boxes']:
img1.rectangle([bb[0], bb[1], bb[2], bb[3]],
fill=None,
outline="blue",
width=5)
#img1.rectangle([10, 10, 40, 40], fill =None, outline ="green", width=5)
augmented_images.append(ToTensor()(img.resize((256, 256))))
return augmented_images
def __call__(self, image):
"""
Merges an anonymized face on the scene
:param image: PIL image
:return: PIL image
"""
constructed_image = None
# Extract face
extracted_face, extracted_information = self.extractor(image)
if extracted_face is not None:
face_out = self.model.anonymize(extracted_face,
extracted_information).squeeze(0)
# get it back to the cpu and get the data
face_out = ToPILImage()(face_out.cpu().detach())
# scale to original resolution
face_out = face_out.resize(extracted_face.size, resample=BICUBIC)
# Constructed scene with new face
constructed_image = self.reconstructor(face_out,
extracted_information)
return constructed_image
def create_image(net, loader, name, upscaling=2, multiscale=False):
patch_size = 30
patches = np.array(
[[85, 90, 85 + patch_size, 90 + patch_size],
[160, 140, 160 + patch_size, 140 + patch_size],
[350, 80, 350 + patch_size, 80 + patch_size]])
fig = plt.figure(figsize=(15, 18), dpi=100)
gs = fig.add_gridspec(4, 3, wspace=0.01, hspace=0.3, left=0.05,
top=0.95, bottom=0.02, right=0.95)
highres, lowres = loader[0]
lowres = lowres.to(torch.device("cuda:0"))
lowres = lowres.view([1] + list(lowres.size()))
net.eval()
with torch.no_grad():
if (multiscale):
superres = net(lowres, upscaling)
else:
superres = net(lowres)
lowres = ToImage()(lowres.cpu().view(lowres.size()[1:]))
superres = ToImage()(superres.cpu().view(superres.size()[1:]))
highres = ToImage()(highres.cpu().view(highres.size()[1:]))
lowres_draw = lowres.copy()
draw = ImageDraw.Draw(lowres_draw)
draw.rectangle(list(patches[0]), outline='white')
draw.rectangle(list(patches[1]), outline='white')
draw.rectangle(list(patches[2]), outline='white')
lowres = lowres.resize((lowres.size[0] * upscaling,
lowres.size[1] * upscaling),
Image.BICUBIC)
psnr_low, psnr_super, ssim_low, ssim_super = compute_metrics(
net, DataLoader(loader), upscaling, multiscale)
lowres_title = "Low-resolution image" +\
"\nAverage PSNR over the dataset: {:.2f}\n".format(psnr_low) +\
"Average SSIM over the dataset: {:.4f}".format(ssim_low)
superres_title = "Reconstructed image\n" +\
"Average PSNR over the dataset: {:.2f}\n".format(psnr_super) +\
"Average SSIM over the dataset: {:.4f}".format(ssim_super)
fig.add_subplot(gs[0, 0], xticks=[], yticks=[],
ylabel=f"Image", title=lowres_title)
plt.imshow(np.array(lowres_draw))
fig.add_subplot(gs[0, 1], xticks=[], yticks=[],
title=superres_title)
plt.imshow(np.array(superres))
fig.add_subplot(gs[0, 2], xticks=[], yticks=[],
title="High-resolution image")
plt.imshow(np.array(highres))
ylabels = ["Patch 1", "Patch 2", "Patch 3"]
for i in range(3):
print(lowres.size, highres.size, superres.size)
lowres_patch = lowres.crop(patches[i] * upscaling)
highres_patch = highres.crop(patches[i] * upscaling)
superres_patch = superres.crop(patches[i] * upscaling)
fig.add_subplot(gs[1 + i, 0], xticks=[], yticks=[],
ylabel=ylabels[i])
plt.imshow(np.array(lowres_patch))
fig.add_subplot(gs[1 + i, 1], xticks=[], yticks=[])
plt.imshow(np.array(superres_patch))
fig.add_subplot(gs[1 + i, 2], xticks=[], yticks=[])
plt.imshow(np.array(highres_patch))
plt.savefig(name)
"description": description,
"ckpt": "stylegan2-ffhq-config-f.pt",
"stylegan_size": 1024,
"lr_rampup": 0.05,
"lr": 0.1,
"step": optimization_steps,
"mode": experiment_type,
"l2_lambda": l2_lambda,
"latent_path": latent_path,
"truncation": 0.7,
"save_intermediate_image_every": 1 if create_video else 20,
"results_dir": "results"
}
from optimization.run_optimization import main
from argparse import Namespace
result = main(Namespace(**args))
#@title Visualize Result
from torchvision.utils import make_grid
from torchvision.transforms import ToPILImage
result_image = ToPILImage()(make_grid(result.detach().cpu(),
normalize=True,
scale_each=True,
range=(-1, 1),
padding=0))
h, w = result_image.size
result_image.resize((h // 2, w // 2))
result_image.save("result.png")
print("save successully")
def __getitem__(self, index):
img_path, pid, camid = self.dataset[index]
img = read_image(img_path)
seed = random.randint(0, 2**32)
radint = random.randint(0, 1)
# print('radint',radint)
# if self.transform is not None:
# random.seed(seed)
# img = self.transform(img)
# area = img.size()[1] * img.size()[2]
area = 256 * 128
i = random.randint(0, 20)
j = random.randint(0, 20)
# print('&&&*****')
# print('i',i)
# print('j',j)
# print('####')
target_area = random.uniform(0.02, 0.4) * area
aspect_ratio = random.uniform(0.3, 1 / (0.3))
img_transform = build_transforms(self.cfg, radint, i, j, is_train=True)
mask_transform = mask_transforms(self.cfg, radint, i, j, is_train=True)
img = img_transform(img)
x1, y1, h, w = get_x1y1(img)
# print(img.size())
# import pdb
# pdb.set_trace()
REA = RandomErasing(radint,
x1,
y1,
h,
w,
probability=self.cfg.INPUT.RE_PROB,
mean=self.cfg.INPUT.PIXEL_MEAN)
img = REA(img)
MREA = MaskRandomErasing(radint,
x1,
y1,
h,
w,
probability=self.cfg.INPUT.RE_PROB,
mean=self.cfg.INPUT.PIXEL_MEAN)
if self.use_salience and not self.use_parsing:
salience_path = osp.join(self.salience_base_path,
get_image_name(img_path) + '.npy')
if self.transform_salience_parsing == None:
salience_img = preprocess_salience(
read_numpy_file(salience_path))
else:
random.seed(seed)
# salience_img = self.transform_salience_parsing(Image.fromarray(read_numpy_file(salience_path)))
salience_img = mask_transform(
Image.fromarray(read_numpy_file(salience_path)))
salience_img = MREA(salience_img)
salience_img = ToPILImage()(salience_img)
salience_img = T.Resize((64, 32))(salience_img)
salience_img = T.ToTensor()(salience_img)
# print(salience_img.size())
# import pdb
# pdb.set_trace()
# salience_img = salience_img.resize((32, 64), Image.BILINEAR)
# salience_img = np.array(salience_img)
# salience_img =torch.Tensor(salience_img)
return img, pid, camid, salience_img, img_path
elif not self.use_salience and self.use_parsing:
parsing_path = osp.join(self.parsing_base_path,
get_image_name(img_path) + '.npy')
zerovalue = get_value(parsing_path)
# parsing_img, label_mark = decode_parsing(torch.tensor(read_numpy_file(parsing_path)))
parsing_img = decode_parsing(
torch.tensor(read_numpy_file(parsing_path)))
parsing_img = new_decode_parsing(
torch.tensor(read_numpy_file(parsing_path)), zerovalue)
body_parsing_img = body_decode_parsing(
torch.tensor(read_numpy_file(parsing_path)))
if self.transform_salience_parsing != None:
new_parsing_img = []
body_new_parsing_img = []
for slide in parsing_img:
random.seed(seed)
# img_i = self.transform_salience_parsing(Image.fromarray(slide))
img_i = mask_transform(Image.fromarray(slide))
img_i = MREA(img_i)
img_i = ToPILImage()(img_i)
img_i = T.Resize((64, 32))(img_i)
# img_i = T.ToTensor()(img_i)
# img_i = img_i.resize((64, 192), Image.BILINEAR)
# img_i = img_i.resize((32, 64), Image.BILINEAR)
img_i = np.array(img_i)
new_parsing_img.append(img_i)
for slide in body_parsing_img:
random.seed(seed)
img_i = self.transform_salience_parsing(
Image.fromarray(slide))
# img_i = img_i.resize((64, 192), Image.BILINEAR)
# img_i = img_i.resize((32, 64), Image.BILINEAR)
img_i = np.array(img_i)
body_new_parsing_img.append(img_i)
# parsing_img = np.array(new_parsing_img)
parsing_img = torch.Tensor(new_parsing_img)
# import pdb
# pdb.set_trace()
# body_new_parsing_img = np.array(body_new_parsing_img)
body_new_parsing_img = torch.Tensor(body_new_parsing_img)
# return img, pid, camid, parsing_img,body_new_parsing_img, img_path
return img, pid, camid, parsing_img, img_path
elif self.use_parsing and self.use_salience:
parsing_path = osp.join(self.parsing_base_path,
get_image_name(img_path) + '.npy')
salience_path = osp.join(self.salience_base_path,
get_image_name(img_path) + '.npy')
if self.transform_salience_parsing == None:
salience_img = preprocess_salience(
read_numpy_file(salience_path))
parsing_img = decode_parsing(
torch.tensor(read_numpy_file(parsing_path)))
else:
random.seed(seed)
salience_img = self.transform_salience(
Image.fromarray(read_numpy_file(salience_path)))
salience_img = salience_img.resize((64, 128), Image.BILINEAR)
salience_img = np.array(salience_img)
parsing_img = decode_parsing(
torch.tensor(read_numpy_file(parsing_path)))
new_parsing_img = []
for slide in parsing_img:
random.seed(seed)
img_i = self.transform_salience_parsing(
Image.fromarray(slide))
img_i = img_i.resize((64, 128), Image.BILINEAR)
img_i = np.array(img_i)
new_parsing_img.append(img_i)
parsing_img = np.array(new_parsing_img)
return img, pid, camid, salience_img, parsing_img, img_path
else:
return img, pid, camid, img_path
def main(args):
modelpath = args.loadDir + args.loadModel
weightspath = args.loadDir + args.loadWeights
print("Loading model: " + modelpath)
print("Loading weights: " + weightspath)
model = Net(NUM_CLASSES)
model = torch.nn.DataParallel(model)
if (not args.cpu):
model = model.cuda()
#model.load_state_dict(torch.load(args.state))
#model.load_state_dict(torch.load(weightspath)) #not working if missing key
def load_my_state_dict(
model, state_dict
): #custom function to load model when not all dict elements
own_state = model.state_dict()
for name, param in state_dict.items():
if name not in own_state:
continue
own_state[name].copy_(param)
return model
model = load_my_state_dict(model, torch.load(weightspath))
print("Model and weights LOADED successfully")
model.eval()
if (not os.path.exists(args.datadir)):
print("Error: datadir could not be loaded")
loader = DataLoader(KITTI(args.datadir,
input_transform_cityscapes,
target_transform_cityscapes,
subset=args.subset),
num_workers=args.num_workers,
batch_size=args.batch_size,
shuffle=False)
# input_transform_cityscapes = Compose([ Resize((512, 1024), Image.BILINEAR), ToTensor(),
# Normalize([.485, .456, .406], [.229, .224, .225]),])
# with open(image_path_city('/home/gongyiqun', '4.png'), 'rb') as f:
# images = load_image(f).convert('RGB')
# images = input_transform_cityscapes(images)
# For visualizer:
# must launch in other window "python3.6 -m visdom.server -port 8097"
# and access localhost:8097 to see it
if (args.visualize):
vis = visdom.Visdom()
for step, (images, filename) in enumerate(loader):
if (not args.cpu):
images = images.cuda()
#labels = labels.cuda()
inputs = Variable(images)
#targets = Variable(labels)
with torch.no_grad():
outputs = model(inputs)
label = outputs[0].max(0)[1].byte().cpu().data
#label_cityscapes = cityscapes_trainIds2labelIds(label.unsqueeze(0))
label_color = Colorize()(label.unsqueeze(0))
filenameSave = "./save_color(KITTI)/" + filename[0].split(
"leftImg8bit/")[1]
# filenameSave = "./save_color/"+"Others"
os.makedirs(os.path.dirname(filenameSave), exist_ok=True)
#image_transform(label.byte()).save(filenameSave)
label_save = ToPILImage()(label_color)
label_save = label_save.resize((1242, 375),
Image.BILINEAR) # For KITTI only
label_save.save(filenameSave)
if (args.visualize):
vis.image(label_color.numpy())
print(step, filenameSave)
