def infer(data_path, model):
psnr = utils.AvgrageMeter()
ssim = utils.AvgrageMeter()
model.eval()
transforms = torchvision.transforms.Compose(
[torchvision.transforms.ToTensor()])
with torch.no_grad():
for step, pt in enumerate(glob.glob(data_path)):
image = np.array(Image.open(pt))
clear_image = utils.crop_img(image[:, :image.shape[1] // 2, :],
base=args.patch_size)
rain_image = utils.crop_img(image[:, image.shape[1] // 2:, :],
base=args.patch_size)
# # Test on whole image
# input = transforms(rain_image).unsqueeze(dim=0).cuda()
# target = transforms(clear_image).unsqueeze(dim=0).cuda(async=True)
# logits = model(input)
# n = input.size(0)
# Test on whole image with data augmentation
target = transforms(clear_image).unsqueeze(dim=0).cuda()
for i in range(8):
im = utils.data_augmentation(rain_image, i)
input = transforms(im.copy()).unsqueeze(dim=0).cuda()
begin_time = time.time()
if i == 0:
logits = utils.inverse_augmentation(
model(input).cpu().numpy().transpose(0, 2, 3, 1)[0], i)
else:
logits = logits + utils.inverse_augmentation(
model(input).cpu().numpy().transpose(0, 2, 3, 1)[0], i)
end_time = time.time()
n = input.size(0)
logits = transforms(logits / 8).unsqueeze(dim=0).cuda()
# # Test on patches2patches
# noise_patches = utils.slice_image2patches(rain_image, patch_size=args.patch_size)
# image_patches = utils.slice_image2patches(clear_image, patch_size=args.patch_size)
# input = torch.tensor(noise_patches.transpose(0,3,1,2)/255.0, dtype=torch.float32).cuda()
# target = torch.tensor(image_patches.transpose(0,3,1,2)/255.0, dtype=torch.float32).cuda()
# logits = model(input)
# n = input.size(0)
s = pytorch_ssim.ssim(torch.clamp(logits, 0, 1), target)
p = utils.compute_psnr(
np.clip(logits.detach().cpu().numpy(), 0, 1),
target.detach().cpu().numpy())
psnr.update(p, n)
ssim.update(s, n)
print('psnr:%6f ssim:%6f' % (p, s))
# Image.fromarray(rain_image).save(args.save+'/'+str(step)+'_noise.png')
# Image.fromarray(np.clip(logits[0].cpu().numpy().transpose(1,2,0)*255, 0, 255).astype(np.uint8)).save(args.save+'/'+str(step)+'_denoised.png')
return psnr.avg, ssim.avg
def camera_facerda():
cap = cv2.VideoCapture(0)
success, frame = cap.read()
h, w = frame.shape[:2]
centerface = CenterFace(h, w, landmarks=True)
model_path = "../model/frda_sim.onnx"
facerda = FaceRDA(model_path, True)
while success:
success, frame = cap.read()
dets, _ = centerface(frame, threshold=0.5) # 3. forward
if dets.shape[0] == 0:
continue
for det in dets:
boxes, score = det[:4].astype("int32"), det[4]
roi_box = centerface.get_crop_box(boxes[0], boxes[1],
boxes[2] - boxes[0],
boxes[3] - boxes[1], 1.4)
face, ret_roi = crop_img(frame, roi_box)
vertices = facerda(face, roi_box)
frame = plot_vertices(frame, vertices)
cv2.imshow('frame', frame)
cv2.waitKey(30)
def ffwd(data_in, paths_out, checkpoint_dir, device_t='/gpu:0', batch_size=4):
assert len(paths_out) > 0
is_paths = type(data_in[0]) == str
if is_paths:
assert len(data_in) == len(paths_out)
img_shape, borders = get_img_eval(data_in[0]) # shape with border
img_shape = img_shape.shape
else:
assert data_in.size[0] == len(paths_out)
img_shape = X[0].shape
g = tf.Graph()
batch_size = min(len(paths_out), batch_size)
curr_num = 0
soft_config = tf.ConfigProto(allow_soft_placement=True)
soft_config.gpu_options.allow_growth = True
with g.as_default(), g.device(device_t), \
tf.Session(config=soft_config) as sess:
batch_shape = (batch_size,) + img_shape
img_placeholder = tf.placeholder(tf.float32, shape=batch_shape,
name='img_placeholder')
preds = transform.net(img_placeholder)
saver = tf.train.Saver()
if os.path.isdir(checkpoint_dir):
ckpt = tf.train.get_checkpoint_state(checkpoint_dir)
if ckpt and ckpt.model_checkpoint_path:
saver.restore(sess, ckpt.model_checkpoint_path)
else:
raise Exception("No checkpoint found...")
else:
saver.restore(sess, checkpoint_dir)
num_iters = int(len(paths_out)/batch_size)
for i in range(num_iters):
pos = i * batch_size
curr_batch_out = paths_out[pos:pos+batch_size]
if is_paths:
curr_batch_in = data_in[pos:pos+batch_size]
X = np.zeros(batch_shape, dtype=np.float32)
for j, path_in in enumerate(curr_batch_in):
img, _ = get_img_eval(path_in)
assert img.shape == img_shape, \
'Images have different dimensions. ' + \
'Resize images or use --allow-different-dimensions.'
X[j] = img
else:
X = data_in[pos:pos+batch_size]
_preds = sess.run(preds, feed_dict={img_placeholder:X})
for j, path_out in enumerate(curr_batch_out):
save_img(path_out, crop_img(_preds[j], borders))
remaining_in = data_in[num_iters*batch_size:]
remaining_out = paths_out[num_iters*batch_size:]
if len(remaining_in) > 0:
ffwd(remaining_in, remaining_out, checkpoint_dir,
device_t=device_t, batch_size=1)
def get_train_batch(self, iterator):
data, *_ = iterator.sample(batch_size=self._batch)
in_data = data[:, :self._in_seq, :, :, :]
if c.IN_CHANEL == 3:
gt_data = data[:,
self._in_seq:self._in_seq + self._out_seq, :, :, :]
elif c.IN_CHANEL == 1:
gt_data = data[:,
self._in_seq:self._in_seq + self._out_seq, :, :, :]
else:
raise NotImplementedError
if c.NORMALIZE:
in_data = normalize_frames(in_data)
gt_data = normalize_frames(gt_data)
in_data = crop_img(in_data)
gt_data = crop_img(gt_data)
return in_data, gt_data
def process_batch(
self, batch_sample
): # batch_sample: a list containing the paths to the center/left/right images and steering angle.
'''
Samples in each batch are processed.
The center, left, right, and horizontally flipped images are used and processed.
'''
# New augmented images and steering angles
images = []
steering_angles = []
# original measured steering_angle
steering_angle = np.float32(batch_sample[3]) # 4th column in csv
for image_directory_index in range(3):
image_file_name = batch_sample[image_directory_index].split(
'/'
)[-1] # split using '/' and [-1] indicates the last element in the list
# addition from center/left/right images
image = cv2.imread(
self.image_directory +
image_file_name) # opencv reads images in BGR color
cropped_image = utils.crop_img(image)
blurred_image = utils.blur_img(cropped_image) # gaussian blurring
resized_image = utils.resize_img(blurred_image)
yuv_image = utils.bgr2yuv(
resized_image) # bgr to yuv (as nVidia suggested)
images.append(yuv_image)
# steer angle correction on left/right images to balance the dataset with non-zero steering angles
if image_directory_index == 1: # left camera
steering_angles.append(steering_angle +
self.steering_recovery_factor)
elif image_directory_index == 2: # right camera
steering_angles.append(steering_angle -
self.steering_recovery_factor)
else: # center camera
steering_angles.append(steering_angle)
# if the magnitude of steering angle is bigger than 0.2, flipped image of center is added.
if image_directory_index == 0 and abs(steering_angle) > 0.2:
# if image_directory_index == 0:
images.append(utils.flip_img(yuv_image))
# images.append(utils.flip_img(cropped_resized))
steering_angles.append(steering_angle * (-1.0))
return images, steering_angles
def pipeline(img_name):
path = f'./static/uploaded_images/{img_name}.jpeg'
try:
img = Image.open(path)
except:
return ValueError
# preprocessing image
img = rescale_image(img)
thresh = threshold(img)
cnts, bbox = find_bbox(thresh)
img = crop_img(img, bbox)
final_img = remove_noise_and_smooth(img)
# finding text in image
text = pytesseract.image_to_string(final_img)
# searching for dates
date = find_date(text)
if date is None:
return date
return date.strftime("%Y-%m-%d")
def test_facerda():
frame = cv2.imread("00.jpg")
cv2.imshow('frame', frame)
h, w = frame.shape[:2]
centerface = CenterFace(h, w, landmarks=True)
model_path = "../model/frda_all_sim.onnx"
facerda = FaceRDA(model_path)
dets, _ = centerface(frame, threshold=0.5) # 3. forward
if dets.shape[0] == 0:
return
for det in dets:
boxes, score = det[:4].astype("int32"), det[4]
roi_box = centerface.get_crop_box(boxes[0], boxes[1], boxes[2] - boxes[0], boxes[3] - boxes[1], 1.4)
face, ret_roi = crop_img(frame, roi_box)
vertices = facerda(face, roi_box)
frame = plot_vertices(frame, vertices)
cv2.imshow('image', frame)
cv2.waitKey(0)
def load_data(args):
images = []
labels = []
index = 0
path = "Chars74k/English/Img/GoodImg/"
for i in os.listdir(path + "Bmp"):
if i != ".DS_Store":
for j in os.listdir(path + "Bmp" + "/" + i):
if j != ".DS_Store":
img = (Image.open(path + "Bmp" + "/" + i + "/" +
j).convert("RGB").resize(
(args.IMG_SIZE, args.IMG_SIZE)))
images.append(img)
labels.append(1)
index += 1
for i in os.listdir("Background"):
if i != ".DS_Store":
img = Image.open("Background" + "/" + i).convert("RGB")
cropped_imgs = crop_img(img, args.IMG_SIZE)
for i in cropped_imgs:
images.append(i.resize((args.IMG_SIZE, args.IMG_SIZE)))
labels.append(0)
return images, labels
def telemetry(sid, data):
if data:
# The current steering angle of the car
steering_angle = data["steering_angle"]
# The current throttle of the car
throttle = data["throttle"]
# The current speed of the car
speed = data["speed"]
# The current image from the center camera of the car
imgString = data["image"]
image = Image.open(BytesIO(base64.b64decode(imgString)))
image_array = np.asarray(image)
## modified lines ##
image_array = utils.crop_img(image_array)
image_array = utils.blur_img(image_array)
image_array = utils.resize_img(image_array)
image_array = utils.rgb2yuv(image_array)
# This model currently assumes that the features of the model are just the images. Feel free to change this.
steering_angle = float(
model.predict(image_array[None, :, :, :], batch_size=1))
# Speed control using PI Controller
throttle = controller.update(float(speed))
print(steering_angle, throttle)
send_control(steering_angle, throttle)
# save frame
if args.image_folder != '':
timestamp = datetime.utcnow().strftime('%Y_%m_%d_%H_%M_%S_%f')[:-3]
image_filename = os.path.join(args.image_folder, timestamp)
image.save('{}.jpg'.format(image_filename))
else:
# NOTE: DON'T EDIT THIS.
sio.emit('manual', data={}, skip_sid=True)
def main(video_path,
out_dir,
fa_model,
euler_model,
ver_model,
device,
min_size=200,
frame_sample_ratio=0.1,
min_samples=10,
sample_limit=500,
min_res=720,
id_limit=7):
# cache_file = os.path.join(out_dir, os.path.splitext(os.path.basename(video_path))[0] + '.pkl')
# cache_file = os.path.splitext(video_path)[0] + '.pkl'
# Validate video resolution
cap = cv2.VideoCapture(video_path)
if not cap.isOpened():
raise RuntimeError('Failed to read video: ' + video_path)
height = cap.get(cv2.CAP_PROP_FRAME_HEIGHT)
width = cap.get(cv2.CAP_PROP_FRAME_WIDTH)
if width < min_res or height < min_res:
return
cap.release()
# Extract landmarks and bounding boxes
frame_indexes, landmarks, bboxes, eulers = utils.extract_landmarks_bboxes_from_video(
video_path, fa_model, euler_model, device=device, min_size=min_size)
if len(frame_indexes) == 0:
return
# Check if there is only 1 ID per frame
check = [len(x) == 1 for x in bboxes]
one_id = sum(check) == len(bboxes)
if one_id:
frame_indexes = [np.array(frame_indexes)]
landmarks = [np.array(landmarks)]
bboxes = [np.array(bboxes)]
eulers = [np.array(eulers)]
else:
frame_indexes, landmarks, bboxes, eulers = multi_id_processing(
video_path, frame_indexes, landmarks, bboxes, eulers, ver_model,
id_limit)
id_num = 0
for face_id in range(len(frame_indexes)):
frame_indexes_id = frame_indexes[face_id]
landmarks_id = landmarks[face_id]
bboxes_id = bboxes[face_id]
eulers_id = eulers[face_id]
# filter by min_samples
sample_size = min(
int(np.round(len(frame_indexes_id) * frame_sample_ratio)),
sample_limit)
if sample_size < min_samples:
continue
id_num += 1
# EULER ANGLES METHOD
best_sample_indexes = utils.fuse_clusters(
eulers_id, 0.3) # <<- changes from 0.15 Yuval argues for 0.25 min
if len(best_sample_indexes) > 500:
# OLD VARIANCE METHOD
# Normalize landmarks and convert them to the descriptor vectors
landmark_descs = utils.norm_landmarks(landmarks_id)
best_sample_indexes = utils.landmarks_var_indexes(
frame_indexes_id, landmark_descs, sample_size)
selected_frame_map = dict(
zip(frame_indexes_id[best_sample_indexes],
best_sample_indexes))
else:
selected_frame_map = dict(
zip(frame_indexes_id[best_sample_indexes],
best_sample_indexes))
# Write frames
cap = cv2.VideoCapture(video_path)
total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
for i in tqdm(range(total_frames)):
ret, frame = cap.read()
if i in selected_frame_map:
# Crop frame
scaled_bbox = utils.scale_bbox(
bboxes_id[selected_frame_map[i]], scale=2.0, square=True)
cropped_frame = utils.crop_img(frame, scaled_bbox)
# Adjust output landmarks and bounding boxes
landmarks_id[selected_frame_map[i]] -= scaled_bbox[:2]
bboxes_id[selected_frame_map[i]][:2] -= scaled_bbox[:2]
# Write frame to file
save_path = os.path.join(out_dir, str(id_num))
if not os.path.exists(save_path):
os.makedirs(save_path)
cv2.imwrite(os.path.join(save_path, 'frame_%04d.jpg' % i),
cropped_frame)
cap.release()
# Write landmarks and bounding boxes
np.save(save_path + '_landmarks.npy',
landmarks_id[best_sample_indexes])
np.save(save_path + '_bboxes.npy', bboxes_id[best_sample_indexes])
np.save(save_path + '_eulers.npy', eulers_id[best_sample_indexes])
def multi_id_processing(video_path,
frame_indexes,
landmarks,
bboxes,
eulers,
verification_model,
id_limit=7):
""" Iterate each frame and split detected identities information"""
cap = cv2.VideoCapture(video_path)
total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
last_frame_features = []
frame_id = 0
unique_ids = 0
structured_frame_indexes = {}
structured_landmarks = {}
structured_bboxes = {}
structured_eulers = {}
structured_features = {}
for i in tqdm(range(total_frames)):
ret, frame = cap.read()
if i in frame_indexes:
curr_landmarks = landmarks[frame_id]
curr_bboxes = bboxes[frame_id]
curr_eulers = eulers[frame_id]
frame_rgb = frame[:, :, ::-1]
for detection in range(len(curr_bboxes)):
scaled_bbox = utils.scale_bbox(curr_bboxes[detection],
scale=0.8)
cropped_frame = utils.crop_img(frame_rgb, scaled_bbox)
features = verification_model(cropped_frame)
if type(features) == int:
continue
if frame_id == 0 and unique_ids == 0:
last_frame_features.append(features)
unique_ids += 1
structured_landmarks[0] = [curr_landmarks[detection]]
structured_bboxes[0] = [curr_bboxes[detection]]
structured_eulers[0] = [curr_eulers[detection]]
structured_frame_indexes[0] = [i]
structured_features[0] = [features]
else:
conf, index = utils.verificate_frame(last_frame_features,
features,
threshold=1e-3)
if conf > 0.65:
structured_landmarks[index].append(
curr_landmarks[detection])
structured_bboxes[index].append(curr_bboxes[detection])
structured_eulers[index].append(curr_eulers[detection])
structured_frame_indexes[index].append(i)
structured_features[index].append(features)
# update last_frame of the subject
last_frame_features[index] = features
else:
# set limitation on number of unique ids, the last one will be dropped out
if unique_ids != id_limit:
last_frame_features.append(features)
unique_ids += 1
else:
last_frame_features[-1] = features
idx = unique_ids - 1
structured_landmarks[idx] = [curr_landmarks[detection]]
structured_bboxes[idx] = [curr_bboxes[detection]]
structured_eulers[idx] = [curr_eulers[detection]]
structured_frame_indexes[idx] = [i]
structured_features[idx] = [features]
frame_id += 1
cap.release()
# create new arrays of id-related information
frame_indexes = [[]] * id_limit
landmarks = [[]] * id_limit
bboxes = [[]] * id_limit
eulers = [[]] * id_limit
features_mean = []
for key in structured_frame_indexes.keys():
conf = 0
feats = torch.cat(structured_features[key])
mean_feature = torch.mean(feats, 0).unsqueeze(0)
# for second identity do verification of similarity
if key != 0:
conf, index = utils.verificate_frame(features_mean,
mean_feature,
threshold=1e-3)
if conf > 0.7: # 0.8 was working
landmarks[index] = np.concatenate(
[landmarks[index],
np.array(structured_landmarks[key])])
bboxes[index] = np.concatenate(
[bboxes[index],
np.array(structured_bboxes[key])])
eulers[index] = np.concatenate(
[eulers[index],
np.array(structured_eulers[key])])
frame_indexes[index] = np.concatenate([
frame_indexes[index],
np.array(structured_frame_indexes[key])
])
else:
frame_indexes[key] = np.array(structured_frame_indexes[key])
landmarks[key] = np.array(structured_landmarks[key])
bboxes[key] = np.array(structured_bboxes[key])
eulers[key] = np.array(structured_eulers[key])
features_mean.append(mean_feature)
return frame_indexes, landmarks, bboxes, eulers
def predict(self, img_path):
img_name = img_path.split('/')[-1].split('.')[0]
img_origin = cv2.imread(img_path)
rects = self.face_detector(img_origin, 1)
pts_res = []
Ps = [] # Camera matrix collection
poses = [] # pose collection
vertices_list = [] # store multiple face vertices
params_list = []
roi_box_list = []
colors_list = []
for ind, rect in enumerate(rects):
if self.opt.dlib_landmark:
# - use landmark for roi box cropping
pts = self.face_regressor(img_origin, rect).parts()
pts = np.array([[pt.x, pt.y] for pt in pts]).T
roi_box = self._parse_roi_box_from_landmark(pts)
else:
# - use detected face bbox
bbox = [rect.left(), rect.top(), rect.right(), rect.bottom()]
roi_box = self._parse_roi_box_from_landmark(bbox)
roi_box_list.append(roi_box)
# step one
img = crop_img(img_origin, roi_box)
img = cv2.resize(img,
dsize=(self.opt.std_size, self.opt.std_size),
interpolation=cv2.INTER_LINEAR)
img = self.transform(img).unsqueeze(0)
with torch.no_grad():
img = img.cuda()
params = self.model(img)
params = params.squeeze().cpu().numpy().flatten().astype(
np.float32)
pts68 = self.morphable_model.predict_68pts(params, roi_box)
# two-step for more acccurate bbox to crop face
if self.opt.bbox_init == 'two':
roi_box = self._parse_roi_box_from_landmark(pts68)
img_step2 = crop_img(img_origin, roi_box)
img_step2 = cv2.resize(img_step2,
dsize=(self.opt.std_size,
self.opt.std_size),
interpolation=cv2.INTER_LINEAR)
_img_step2 = img_step2.copy()
img_step2 = self.transform(img_step2).unsqueeze(0)
with torch.no_grad():
img_step2 = img_step2.cuda()
params = self.model(img_step2)
params = params.squeeze().cpu().numpy().flatten().astype(
np.float32)
pts68 = self.morphable_model.predict_68pts(params, roi_box)
params_list.append(params)
vertices = self.morphable_model.predict_dense(params, roi_box)
if self.opt.dump_obj:
path = os.path.join(self.out_dir,
'{}_{}.obj'.format(img_name, ind))
colors = mesh.transform.get_colors_from_image(
img_origin, vertices) / 255.
colors_list.append(colors)
#tp = self.morphable_model.get_tex_params(_type='random')
#colors = self.morphable_model.generate_colors(tp)
#colors = np.minimum(np.maximum(colors, 0), 1)
mesh.interact.write_obj_with_colors(
path, vertices.T, self.morphable_model.model['tri'],
colors)
print(self.morphable_model.model['tri'])
h = img_origin.shape[0]
w = img_origin.shape[1]
image_vertices = vertices.copy().T
#image_vertices[:, 1] = h - image_vertices[:, 1] - 1
fitted_image = mesh.render.render_colors(
image_vertices, self.morphable_model.triangles, colors, h,
w) * 255.
print(fitted_image.shape, image_vertices.shape,
self.morphable_model.triangles.shape, colors.shape)
cv2.imwrite(path.replace('obj', 'jpg'),
fitted_image.astype('uint8'))
def orb_detection(rectified_paintings, paintings, bbc, frame_img, orb, bf,
dict, frm):
room = check_room(dict, frame_img)
for j, r in enumerate(rectified_paintings):
score_list = []
r, alpha, beta = automatic_brightness_and_contrast(r)
r = cv2.cvtColor(r, cv2.COLOR_BGR2GRAY)
r = crop_img(r, 0.15)
kp1, des1 = orb.detectAndCompute(r, None)
for i, paint in enumerate(paintings):
matches = bf.knnMatch(des1, paint.descriptors, k=2)
good = []
for m, n in matches:
if m.distance < 0.75 * n.distance:
good.append([m])
if len(good) > 10:
score_list.append((len(good), paint))
if len(score_list) > 0:
score_list = sorted(score_list, key=lambda x: x[0], reverse=True)
if dict.get(str(score_list[0][1].room)):
dict[str(str(score_list[0][1].room))] += 1
else:
dict[str(str(score_list[0][1].room))] = 1
if frm == 0 and j == len(rectified_paintings) - 1:
room = check_room(dict, frame_img)
if frm == 0:
cv2.rectangle(frame_img, (bbc[j][0], bbc[j][1]),
(bbc[j][0] + bbc[j][2], bbc[j][1] + bbc[j][3]),
(36, 255, 12), 1)
cv2.putText(frame_img, (str(score_list[0][1].filename + " " +
score_list[0][1].title)),
(bbc[j][0], bbc[j][1] + 15),
cv2.FONT_HERSHEY_SIMPLEX, 0.5, (36, 255, 12), 2)
else:
if str(score_list[0][1].room) == room:
cv2.rectangle(
frame_img, (bbc[j][0], bbc[j][1]),
(bbc[j][0] + bbc[j][2], bbc[j][1] + bbc[j][3]),
(36, 255, 12), 1)
cv2.putText(frame_img, (str(score_list[0][1].filename +
" " + score_list[0][1].title)),
(bbc[j][0], bbc[j][1] + 15),
cv2.FONT_HERSHEY_SIMPLEX, 0.5, (36, 255, 12),
2)
else:
cv2.rectangle(
frame_img, (bbc[j][0], bbc[j][1]),
(bbc[j][0] + bbc[j][2], bbc[j][1] + bbc[j][3]),
(0, 0, 255), 1)
else:
cv2.rectangle(frame_img, (bbc[j][0], bbc[j][1]),
(bbc[j][0] + bbc[j][2], bbc[j][1] + bbc[j][3]),
(0, 0, 255), 1)
return room
def run_benchmark(self, iter, mode="Test"):
if mode == "Valid":
time_interval = c.RAINY_VALID
stride = 5
else:
time_interval = c.RAINY_TEST
stride = 1
test_iter = Iterator(time_interval=time_interval,
sample_mode="sequent",
seq_len=self._in_seq + self._out_seq,
stride=1)
evaluator = Evaluator(iter, length=self._out_seq, mode=mode)
i = 1
while not test_iter.use_up:
data, date_clip, *_ = test_iter.sample(batch_size=self._batch)
in_data = np.zeros(shape=(self._batch, self._in_seq, self._h,
self._w, c.IN_CHANEL))
gt_data = np.zeros(shape=(self._batch, self._out_seq, self._h,
self._w, 1))
if type(data) == type([]):
break
in_data[...] = data[:, :self._in_seq, :, :, :]
if c.IN_CHANEL == 3:
gt_data[...] = data[:, self._in_seq:self._in_seq +
self._out_seq, :, :, :]
elif c.IN_CHANEL == 1:
gt_data[...] = data[:, self._in_seq:self._in_seq +
self._out_seq, :, :, :]
else:
raise NotImplementedError
# in_date = date_clip[0][:c.IN_SEQ]
if c.NORMALIZE:
in_data = normalize_frames(in_data)
gt_data = normalize_frames(gt_data)
in_data = crop_img(in_data)
gt_data = crop_img(gt_data)
mse, mae, gdl, pred = self.g_model.valid_step(in_data, gt_data)
evaluator.evaluate(gt_data, pred)
self.logger.info(
f"Iter {iter} {i}: \n\t mse:{mse} \n\t mae:{mae} \n\t gdl:{gdl}"
)
i += 1
if i % stride == 0:
if c.IN_CHANEL == 3:
in_data = in_data[:, :, :, :, 1:-1]
for b in range(self._batch):
predict_date = date_clip[b][self._in_seq - 1]
self.logger.info(f"Save {predict_date} results")
if mode == "Valid":
save_path = os.path.join(
c.SAVE_VALID, str(iter),
predict_date.strftime("%Y%m%d%H%M"))
else:
save_path = os.path.join(
c.SAVE_TEST, str(iter),
predict_date.strftime("%Y%m%d%H%M"))
path = os.path.join(save_path, "in")
save_png(in_data[b], path)
path = os.path.join(save_path, "pred")
save_png(pred[b], path)
path = os.path.join(save_path, "out")
save_png(gt_data[b], path)
evaluator.done()
self.notifier.eval(iter, evaluator.result_path)
def __getitem__(self, idx):
ops = self.ops
if th_rand() < 0.2:
imf, gt, emo = self.get_raf()
else:
imf, gt, emo = self.get_af()
img = imread_to_rgb(imf)
xcen = gt[0] + gt[2]*0.5
ycen = gt[1] + gt[3]*0.5
margin = (gt[2]+gt[3])*ops.aug_marg
wh_mean = np.sqrt((gt[2]+margin)*(gt[2]+margin))
width = wh_mean
height = wh_mean
# data augmentation part ==
aug_prob = ops.aug_prob
aug_crop = ops.aug_crop
# random crops (always)
xmod = th_choice([1,-1])*th_rand_rng(0., width*aug_crop)
ymod = th_choice([1,-1])*th_rand_rng(0., height*aug_crop)
whmod = th_choice([1,-1])*th_rand_rng(0., aug_crop)
wmod = 1.0 + whmod
hmod = 1.0 + whmod
if img.shape[0]==0 or img.shape[1]==0:
print 'b: ',sel_dict['img'], gt
if wmod <= 0 or hmod <= 0:
print 'b: ',wmod,hmod
batch_img = crop_img(img, int(xcen+xmod), int(ycen+ymod), int(width*wmod), int(height*hmod), True, True)
batch_img = cv2.resize(batch_img, (ops.img_sz, ops.img_sz))
# flip horizontally
if th_rand() < 0.5:
batch_img = np.flip(batch_img, axis=1)
# add noise / change level
if th_rand() < aug_prob:
noise_std = th_rand()*0.05
if th_rand() < 0.5:
noise_img = np.repeat( (noise_std)*torch.randn(ops.img_sz, ops.img_sz, 1), 3, 2)
else:
noise_img = (noise_std)*torch.randn(ops.img_sz, ops.img_sz, 3)
batch_img += noise_img.contiguous().numpy()
if th_rand() < aug_prob:
batch_img += (th_rand()-0.5)/50.
# gaussian
if th_rand() < aug_prob:
k_sz = th_randint(1,5,2)*2 +1
batch_img = cv2.GaussianBlur(batch_img, (k_sz[0], k_sz[1]), 0)
# end of data augmentation ==
# reset range [0,1]
if batch_img.min() < 0.:
batch_img[ batch_img < 0. ] = 0.
if batch_img.max() > 1.:
batch_img[ batch_img > 1. ] = 1.
# returns
batch_img = torch.Tensor( batch_img.transpose(2,0,1).copy() )
batch_emo = torch.Tensor( emo.copy() )
#batch_var = torch.Tensor( [sel_dict['val'], sel_dict['aro']] )
return batch_img, batch_emo #, batch_var
try:
frame1, bike_np_roi, _, _, _, _ = infer_image(
net, layer_names, height, width, frame.copy(), colors,
labels, confidence, threshold, poly)
except:
print(
"Skipping................................................."
)
continue
if bike_np_roi != []:
wait_flag = 0
for k in bike_np_roi:
#print(f'Bike co-ordinates {k[0]} , number plate co-ordinates {k[1]}')
bike = k[0]
bike_img = crop_img(frame.copy(), bike)
np = k[1]
if np != ():
wait_flag += 1
np_img = crop_img(frame.copy(), np)
np_img_text = np_img.copy()
if wait_flag > 1:
time.sleep(1)
read_score = plate_reader(np_img)
if (read_score == "Can't read"):
np_chars = "Can't read"
score = 0
else:
np_chars = read_score[0]
score = float(read_score[1])
cv.putText(np_img_text, str(np_chars), (10, 15),
def infer(data_path, model):
psnr = utils.AvgrageMeter()
ssim = utils.AvgrageMeter()
times = utils.AvgrageMeter()
model.eval()
rgb2gray = torchvision.transforms.Grayscale(1)
transforms = torchvision.transforms.Compose([
torchvision.transforms.ToTensor()
])
with torch.no_grad():
for step, pt in enumerate(glob.glob(data_path)):
image = utils.crop_img(np.array(rgb2gray(Image.open(pt)))[..., np.newaxis])
noise_map = np.random.randn(*(image.shape))*args.sigma
noise_img = np.clip(image+noise_map,0,255).astype(np.uint8)
# # Test on whole image
# input = transforms(noise_img).unsqueeze(dim=0).cuda()
# target = transforms(image).unsqueeze(dim=0).cuda()
# begin_time = time.time()
# logits = model(input)
# end_time = time.time()
# n = input.size(0)
# Test on whole image with data augmentation
target = transforms(image).unsqueeze(dim=0).cuda()
for i in range(8):
im = utils.data_augmentation(noise_img,i)
input = transforms(im.copy()).unsqueeze(dim=0).cuda()
begin_time = time.time()
if i == 0:
logits = utils.inverse_augmentation(model(input).cpu().numpy().transpose(0,2,3,1)[0],i)
else:
logits = logits + utils.inverse_augmentation(model(input).cpu().numpy().transpose(0,2,3,1)[0],i)
end_time = time.time()
n = input.size(0)
logits = transforms(logits/8).unsqueeze(dim=0).cuda()
# # Test on patches2patches
# noise_patches = utils.slice_image2patches(noise_img, patch_size=64)
# image_patches = utils.slice_image2patches(image, patch_size=64)
# input = torch.tensor(noise_patches.transpose(0,3,1,2)/255.0, dtype=torch.float32).cuda()
# target = torch.tensor(image_patches.transpose(0,3,1,2)/255.0, dtype=torch.float32).cuda()
# begin_time = time.time()
# logits = model(input)
# end_time = time.time()
# n = input.size(0)
s = pytorch_ssim.ssim(torch.clamp(logits,0,1), target)
p = utils.compute_psnr(np.clip(logits.detach().cpu().numpy(),0,1), target.detach().cpu().numpy())
t = end_time-begin_time
psnr.update(p, n)
ssim.update(s, n)
times.update(t,n)
print('psnr:%6f ssim:%6f time:%6f' % (p, s, t))
# Image.fromarray(noise_img[...,0]).save(args.save+'/'+str(step)+'_noise.png')
# Image.fromarray(np.clip(logits[0,0].cpu().numpy()*255, 0, 255).astype(np.uint8)).save(args.save+'/'+str(step)+'_denoised.png')
return psnr.avg, ssim.avg, times.avg
