def __init__(self, calibration_data_file, smooth_frames = 5, debug = False, failed_frames_dir = 'failed_frames'):
self.img_processor = ImageProcessor(calibration_data_file)
self.lane_tracker = LaneDetector(smooth_frames = smooth_frames)
self.frame_count = 0
self.fail_count = 0
self.debug = debug
self.failed_frames_dir = failed_frames_dir
self.last_frame = None
self.processed_frames = None
class VideoProcessor:
"""
Class used to process a video file and that produces an annotated version with the detected lanes.
"""
def __init__(self, calibration_data_file, smooth_frames = 5, debug = False, failed_frames_dir = 'failed_frames'):
self.img_processor = ImageProcessor(calibration_data_file)
self.lane_tracker = LaneDetector(smooth_frames = smooth_frames)
self.frame_count = 0
self.fail_count = 0
self.debug = debug
self.failed_frames_dir = failed_frames_dir
self.last_frame = None
self.processed_frames = None
def process_frame(self, img):
img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)
undistorted_img, thresholded_img, warped_img = self.img_processor.process_image(img)
_, polyfit, curvature, deviation, fail_code = self.lane_tracker.detect_lanes(warped_img)
fill_color = (0, 255, 0) if fail_code == 0 else (0, 255, 255)
lane_img = self.lane_tracker.draw_lanes(undistorted_img, polyfit, fill_color = fill_color)
lane_img = self.img_processor.unwarp_image(lane_img)
out_image = cv2.addWeighted(undistorted_img, 1.0, lane_img, 1.0, 0)
font = cv2.FONT_HERSHEY_DUPLEX
font_color = (0, 255, 0)
curvature_text = 'Left Curvature: {:.1f}, Right Curvature: {:.1f}'.format(curvature[0], curvature[1])
offset_text = 'Center Offset: {:.2f} m'.format(deviation)
cv2.putText(out_image, curvature_text, (30, 60), font, 1, font_color, 2)
cv2.putText(out_image, offset_text, (30, 90), font, 1, font_color, 2)
if fail_code > 0:
self.fail_count += 1
failed_text = 'Detection Failed: {}'.format(LaneDetector.FAIL_CODES[fail_code])
cv2.putText(out_image, failed_text, (30, 120), font, 1, (0, 0, 255), 2)
if self.debug:
print(failed_text)
cv2.imwrite(os.path.join(self.failed_frames_dir, 'frame' + str(self.frame_count) + '_failed_' + str(fail_code) + '.png'), img)
cv2.imwrite(os.path.join(self.failed_frames_dir, 'frame' + str(self.frame_count) + '_failed_' + str(fail_code) + '_lanes.png'), out_image)
if self.last_frame is not None: # Saves also the previous frame for comparison
cv2.imwrite(os.path.join(self.failed_frames_dir, 'frame' + str(self.frame_count) + '_failed_' + str(fail_code) + '_prev.png'), self.last_frame)
self.frame_count += 1
self.last_frame = img
out_image = cv2.cvtColor(out_image, cv2.COLOR_BGR2RGB)
return out_image
def process_frame_split(self, img):
img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)
undistorted_img, thresholded_img, processed_img = self.img_processor.process_image(img)
lanes_centroids, polyfit, curvature, deviation, fail_code = self.lane_tracker.detect_lanes(processed_img)
fill_color = (0, 255, 0) if fail_code == 0 else (0, 255, 255)
lane_img = self.lane_tracker.draw_lanes(undistorted_img, polyfit, fill_color = fill_color)
lane_img = self.img_processor.unwarp_image(lane_img)
out_image = cv2.addWeighted(undistorted_img, 1.0, lane_img, 1.0, 0)
color_img = self.img_processor.color_thresh(undistorted_img) * 255
gradient_img = self.img_processor.gradient_thresh(undistorted_img) * 255
processed_src = np.array(cv2.merge((thresholded_img, thresholded_img, thresholded_img)), np.uint8)
src, _ = self.img_processor._warp_coordinates(img)
src = np.array(src, np.int32)
cv2.polylines(processed_src, [src], True, (0,0,255), 2)
window_img = np.copy(processed_img)
window_img = self.lane_tracker.draw_windows(window_img, lanes_centroids, polyfit, blend = True)
lanes_img = np.copy(out_image)
font = cv2.FONT_HERSHEY_DUPLEX
font_color = (0, 255, 0)
curvature_text = 'Left Curvature: {:.1f}, Right Curvature: {:.1f}'.format(curvature[0], curvature[1])
offset_text = 'Center Offset: {:.2f} m'.format(deviation)
cv2.putText(out_image, curvature_text, (30, 60), font, 1, font_color, 2)
cv2.putText(out_image, offset_text, (30, 90), font, 1, font_color, 2)
if fail_code > 0:
self.fail_count += 1
failed_text = 'Detection Failed: {}'.format(LaneDetector.FAIL_CODES[fail_code])
cv2.putText(out_image, failed_text, (30, 120), font, 1, (0, 0, 255), 2)
if self.debug:
print(failed_text)
cv2.imwrite(os.path.join(self.failed_frames_dir, 'frame' + str(self.frame_count) + '_failed_' + str(fail_code) + '.png'), img)
cv2.imwrite(os.path.join(self.failed_frames_dir, 'frame' + str(self.frame_count) + '_failed_' + str(fail_code) + '_lanes.png'), out_image)
if self.last_frame is not None: # Saves also the previous frame for comparison
cv2.imwrite(os.path.join(self.failed_frames_dir, 'frame' + str(self.frame_count) + '_failed_' + str(fail_code) + '_prev.png'), self.last_frame)
self.frame_count += 1
undistorted_img = cv2.cvtColor(undistorted_img, cv2.COLOR_BGR2RGB)
color_img = cv2.cvtColor(color_img, cv2.COLOR_GRAY2RGB)
gradient_img = cv2.cvtColor(gradient_img, cv2.COLOR_GRAY2RGB)
thresholded_img= cv2.cvtColor(thresholded_img, cv2.COLOR_GRAY2RGB)
processed_src= cv2.cvtColor(processed_src, cv2.COLOR_BGR2RGB)
processed_img= cv2.cvtColor(processed_img, cv2.COLOR_GRAY2RGB)
window_img= cv2.cvtColor(window_img, cv2.COLOR_BGR2RGB)
lanes_img= cv2.cvtColor(lanes_img, cv2.COLOR_BGR2RGB)
out_image= cv2.cvtColor(out_image, cv2.COLOR_BGR2RGB)
return (undistorted_img, color_img, gradient_img, thresholded_img, processed_src, processed_img, window_img, lanes_img, out_image)
def process_video(self, file_path, output, t_start = None, t_end = None):
input_clip = VideoFileClip(file_path)
if t_start is not None:
input_clip = input_clip.subclip(t_start = t_start, t_end = t_end)
output_clip = input_clip.fl_image(self.process_frame)
output_clip.write_videofile(output, audio = False)
def process_frame_stage(self, img, idx):
if self.processed_frames is None:
self.processed_frames = []
if idx == 0:
result = self.process_frame_split(img)
self.processed_frames.append(result)
else:
self.frame_count += 1
return self.processed_frames[self.frame_count - 1][idx]
def process_video_split(self, file_path, output, t_start = None, t_end = None):
input_clip = VideoFileClip(file_path)
if t_start is not None:
input_clip = input_clip.subclip(t_start = t_start, t_end = t_end)
out_file_prefix = os.path.split(output)[1][:-4]
idx = 0
output_clip = input_clip.fl_image(lambda img: self.process_frame_stage(img, idx))
output_clip.write_videofile(out_file_prefix + '_undistoreted.mp4', audio = False)
idx += 1
self.frame_count = 0
output_clip.write_videofile(out_file_prefix + '_color.mp4', audio = False)
idx += 1
self.frame_count = 0
output_clip.write_videofile(out_file_prefix + '_gradient.mp4', audio = False)
idx += 1
self.frame_count = 0
output_clip.write_videofile(out_file_prefix + '_thresholded.mp4', audio = False)
idx += 1
self.frame_count = 0
output_clip.write_videofile(out_file_prefix + '_processed_src.mp4', audio = False)
idx += 1
self.frame_count = 0
output_clip.write_videofile(out_file_prefix + '_processed_dst.mp4', audio = False)
idx += 1
self.frame_count = 0
output_clip.write_videofile(out_file_prefix + '_window.mp4', audio = False)
idx += 1
self.frame_count = 0
output_clip.write_videofile(out_file_prefix + '_lanes.mp4', audio = False)
idx += 1
self.frame_count = 0
output_clip.write_videofile(out_file_prefix + '_final.mp4', audio = False)
print('Number of failed detection: {}'.format(self.fail_count))
def upload(filename):
""" Get the default Notebook ID and process the passed in file"""
basefile = os.path.basename(filename)
title, ext = os.path.splitext(basefile)
body = f"{basefile} uploaded from {platform.node()}\n"
datatype = mimetypes.guess_type(filename)[0]
if datatype is None:
# avoid subscript exception if datatype is None
if ext in (".url", ".lnk"):
datatype = "text/plain"
else:
datatype = ""
if datatype == "text/plain":
body += read_text_note(filename)
values = set_json_string(title, NOTEBOOK_ID, body)
if datatype == "text/csv":
table = read_csv(filename)
body += tabulate(table,
headers="keys",
numalign="right",
tablefmt="pipe")
values = set_json_string(title, NOTEBOOK_ID, body)
elif datatype[:5] == "image":
img_processor = ImageProcessor(LANGUAGE)
body += "\n<!---\n"
try:
body += img_processor.extract_text_from_image(
filename, autorotate=AUTOROTATION)
except TypeError:
print("Unable to perform OCR on this file.")
except OSError:
print(f"Invalid or incomplete file - {filename}")
return -1
body += "\n-->\n"
img = img_processor.encode_image(filename, datatype)
del img_processor
values = set_json_string(title, NOTEBOOK_ID, body, img)
else:
response = create_resource(filename)
body += f"[{basefile}](:/{response['id']})"
values = set_json_string(title, NOTEBOOK_ID, body)
if response["file_extension"] == "pdf":
img_processor = ImageProcessor(LANGUAGE)
if img_processor.pdf_valid(filename):
# Special handling for PDFs
body += "\n<!---\n"
body += img_processor.extract_text_from_pdf(filename)
body += "\n-->\n"
previewfile = img_processor.PREVIEWFILE
if not os.path.exists(previewfile):
previewfile = img_processor.pdf_page_to_image(filename)
img = img_processor.encode_image(previewfile, "image/png")
os.remove(previewfile)
values = set_json_string(title, NOTEBOOK_ID, body, img)
response = requests.post(ENDPOINT + "/notes" + TOKEN, data=values)
# print(response)
# print(response.text)
if response.status_code == 200:
if AUTOTAG:
apply_tags(body, response.json().get("id"))
print(f"Placed note into notebook {NOTEBOOK_ID}: {NOTEBOOK_NAME}")
if os.path.isdir(MOVETO):
moveto_filename = os.path.join(MOVETO, basefile)
print(moveto_filename)
if os.path.exists(moveto_filename):
print(f"{basefile} exists in moveto dir, not moving!")
else:
try:
# Give it a few seconds to release file lock
time.sleep(3)
shutil.move(filename, MOVETO)
except IOError:
print(f"File Locked-unable to move {filename}")
return 0
else:
print("ERROR! NOTE NOT CREATED")
print("Something went wrong corrupt file or note > 10MB?")
return -1
default=80,
help='Sliding window height')
parser.add_argument('--w_margin',
type=int,
default=35,
help='Sliding window margin from center')
args = parser.parse_args()
img_files = []
for ext in ('*.png', '*.jpg'):
img_files.extend(glob(os.path.join(args.dir, ext)))
img_processor = ImageProcessor(args.calibration_data_file)
lane_tracker = LaneDetector(window_width=args.w_width,
window_height=args.w_height,
margin=args.w_margin,
smooth_frames=0)
for img_file in tqdm(img_files, unit=' images', desc='Image processing'):
print(img_file)
input_img = cv2.imread(img_file)
undistorted_img, thresholded_img, processed_img = img_processor.process_image(
input_img)
out_file_prefix = os.path.join(args.output,
os.path.split(img_file)[1][:-4])
def init_vid_process(self, source, pipe, array, mode, cap):
mode = self.check_mode_availability(source, mode)
self.cam_process = ImageProcessor(source, mode, pipe, array, cap)
self.vid_process = Process(target=self.cam_process.run_vid, args=())
self.vid_process.start()
def init_process(self, source, pipe, array, mode, cap):
mode = self.check_mode_availability(source, mode)
self.cam_process = ImageProcessor(source, mode, pipe, array, cap)
self.cam_process.start()
class EyeCamera(camera.Camera):
def __init__(self, name=None, mode=(192,192,120)):
super().__init__(name)
self.mode = mode
self.cam_process = None
self.vid_process = None
self.shared_array = self.create_shared_array(mode)
self.detector = Detector3D()
self.bbox = None
self.pos = None
self.detector.update_properties({'2d':{'pupil_size_max':180}})
self.detector.update_properties({'2d':{'pupil_size_min':10}})
self.countdown = 5
def init_process(self, source, pipe, array, mode, cap):
mode = self.check_mode_availability(source, mode)
self.cam_process = ImageProcessor(source, mode, pipe, array, cap)
self.cam_process.start()
def init_vid_process(self, source, pipe, array, mode, cap):
mode = self.check_mode_availability(source, mode)
self.cam_process = ImageProcessor(source, mode, pipe, array, cap)
self.vid_process = Process(target=self.cam_process.run_vid, args=())
self.vid_process.start()
def join_process(self):
self.cam_process.join(10)
def join_vid_process(self):
self.vid_process.join(3)
def create_shared_array(self, mode):
w = mode[0]
h = mode[1]
return Array(ctypes.c_uint8, h*w*3, lock=False)
def check_mode_availability(self, source, mode):
dev_list = uvc.device_list()
cap = uvc.Capture(dev_list[source]['uid'])
if mode not in cap.avaible_modes:
m = cap.avaible_modes[0]
mode = (m[1], m[0], m[2])
self.shared_array = self.create_shared_array(mode)
self.mode = mode
return mode
def process(self, img):
if img is None:
return
height, width = img.shape[0], img.shape[1]
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
timestamp = uvc.get_time_monotonic()
roi = None
if self.bbox is not None:
xmin, ymin, w, h = self.bbox
roi = Roi(xmin, ymin, xmin+w, ymin+h)
result = self.detector.detect(gray, timestamp, roi=roi)
#print(result)
if result["model_confidence"] > 0.25:
sphere = result["projected_sphere"]
self.__draw_ellipse(sphere, img, (255,120,120), 1)
if result["confidence"] > 0.5:
n = np.array(result['circle_3d']['normal'])
self.bbox = self.__get_bbox(result, img)
self.__draw_tracking_info(result, img)
# cv2.imshow("testando", img)
# cv2.waitKey(1)
self.pos = np.array([n[0], n[1], n[2], time.monotonic()])
self.countdown = 5
else:
self.countdown -= 1
if self.countdown <= 0:
self.pos = None
self.bbox = None
return img
def freeze_model(self):
self.detector.update_properties({
"3d": {"model_is_frozen": True}
})
def unfreeze_model(self):
self.detector.update_properties({
"3d": {"model_is_frozen": False}
})
def __get_bbox(self, result, img):
r = result['diameter']
point = result['ellipse']['center']
x1 = point[0]-r*0.8
y1 = point[1]-r*0.8
x2 = point[0]+r*0.8
y2 = point[1]+r*0.8
x1 = self.__test_boundaries(x1, img.shape[1])
y1 = self.__test_boundaries(y1, img.shape[0])
x2 = self.__test_boundaries(x2, img.shape[1])
y2 = self.__test_boundaries(y2, img.shape[0])
w = x2-x1
h = y2-y1
cv2.rectangle(img, self.bbox, (125,80,80), 2, 1)
return int(x1),int(y1),int(w),int(h)
def __test_boundaries(self, x, lim):
if x < 0:
return 0
if x >= lim:
return lim-1
return x
def __draw_tracking_info(self, result, img, color=(255,120,120)):
ellipse = result["ellipse"]
normal = result["circle_3d"]["normal"]
center = tuple(int(v) for v in ellipse["center"])
cv2.drawMarker(img, center, (0,255,0), cv2.MARKER_CROSS, 12, 1)
self.__draw_ellipse(ellipse, img, (0,0,255))
dest_pos = (int(center[0]+normal[0]*60), int(center[1]+normal[1]*60))
cv2.line(img, center, dest_pos, (85,175,20),2)
# if self.bbox is not None:
# cv2.rectangle(img, self.bbox, (120,255,120), 2, 1)
def __draw_ellipse(self, ellipse, img, color, thickness=2):
center = tuple(int(v) for v in ellipse["center"])
axes = tuple(int(v/2) for v in ellipse["axes"])
rad = ellipse["angle"]
cv2.ellipse(img, center, axes, rad, 0, 360, color, 2)
def reset_model(self):
self.detector.reset_model()
def get_processed_data(self):
return self.pos