def __init__(self, video_clip, is_search_laneline=True, is_search_car=True,
camera_cali_file='', perspective_trans_params=None,
thresh_params=None,
car_classifier_file='', car_search_params=None):
"""Initialization.
:param video_clip: string
File name of the input video.
:param camera_cali_file: string
Name of the pickle file storing the camera calibration
:param perspective_trans_params: tuple
Source and destination points for perspective transformation
For example, (src, dst) with
src = np.float32([[0, 720], [570, 450], [708, 450], [1280, 720]])
dst = np.float32([[0, 720], [0, 0], [1280, 0], [1280, 720]])
:param thresh_params: list of dictionary
Parameters for the gradient and color threshhold.
:param car_classifier_file: String
Pickle file for CarClassifier object.
:param car_search_params: dictionary
Parameters for performing car search in an image.
:param is_search_car: Boolean
True for search cars in the video/image.
:param is_search_laneline: Boolean
True for search lanelines in the video/image.
"""
self.clip = VideoFileClip(video_clip)
self.shape = self.get_video_image(0).shape
assert(len(self.shape) == 3) # colored image
self.frame = 0 # the current frame index
if is_search_car is False and is_search_laneline is False:
raise SystemExit("Nothing needs to be done!:(")
# Load camera calibration information
with open(camera_cali_file, "rb") as fp:
camera_cali = pickle.load(fp)
# Get required parameters for image undistortion
_, self.camera_matrix, self.dist_coeffs, _, _ = cv2.calibrateCamera(
camera_cali["obj_points"], camera_cali["img_points"],
self.shape[0:2], None, None)
self._is_search_laneline = is_search_laneline
self.thresh_params = thresh_params # threshold parameter in lane line search
self.ppt_trans_params = perspective_trans_params
# perspective and inverse perspective transformation matrices
self.ppt_trans_matrix, self.inv_ppt_trans_matrix = \
get_perspective_trans_matrix(self.ppt_trans_params[0],
self.ppt_trans_params[1])
y_fit = np.arange(self.shape[0], 0, -10)
self.left_line = LaneLine(y_fit)
self.right_line = LaneLine(y_fit)
self._is_search_car = is_search_car
with open(car_classifier_file, "rb") as fp:
self.car_classifier = pickle.load(fp)
self.car_search_params = car_search_params
self.car_boxes = [] # square boxes classified as cars
def __init__(self, image, left: LaneLine, right: LaneLine):
self.left_lane = left
self.right_lane = right
if left.is_valid() and right.is_valid():
# print(self.left_lane.curvature_m, self.right_lane.curvature_m)
self.curvature_m = (self.left_lane.curvature_m + self.right_lane.curvature_m) / 2
self.lane_width_start_m = self.right_lane.offset_start_m - self.left_lane.offset_start_m
# self.offset_m = -(self.left_lane.offset_start_m + self.right_lane.offset_start_m)
self.offset_m = ((image.shape[1] / 2) - ((self.left_lane.fit_x[-1] + self.right_lane.fit_x[-1]) / 2)) * self.left_lane.xscale
def __init__(self, camera):
"""
Initializer.
"""
self._camera = camera
source_points = np.float32([[170, 720], [566, 450], [716, 450], [1120, 720]])
dest_points = np.float32([[350, 720], [200, 0], [1080, 0], [930, 720]])
self._transformation_matrix = cv2.getPerspectiveTransform(source_points, dest_points)
self._inverse_transformation_matrix = cv2.getPerspectiveTransform(dest_points, source_points)
sliding_windows_nr = 9
margin = 100
recenter_threshold = 50
self._left_lane_line = LaneLine(sliding_windows_nr, margin, recenter_threshold)
self._right_lane_line = LaneLine(sliding_windows_nr, margin, recenter_threshold)
def get_plottable_curves(height,
left_fit,
right_fit,
xm_per_pix,
ym_per_pix,
steps=20):
ploty = np.linspace(0, height, steps)
left_curve = np.int32(
list(
zip(
LaneLine.evaluate_poly2(left_fit, ploty * ym_per_pix) /
xm_per_pix, ploty)))
right_curve = np.int32(
list(
zip(
LaneLine.evaluate_poly2(right_fit, ploty * ym_per_pix) /
xm_per_pix, ploty)))
return left_curve, right_curve
def find_lines_from_previous(binary_warped, left_line: LaneLine,
right_lane: LaneLine):
nonzero = binary_warped.nonzero()
nonzeroy = np.array(nonzero[0])
nonzerox = np.array(nonzero[1])
left_fit = left_line.poly
right_fit = right_lane.poly
margin = 100
left_lane_inds = (
(nonzerox >
(left_fit[0] *
(nonzeroy**2) + left_fit[1] * nonzeroy + left_fit[2] - margin)) &
(nonzerox <
(left_fit[0] *
(nonzeroy**2) + left_fit[1] * nonzeroy + left_fit[2] + margin)))
right_lane_inds = (
(nonzerox >
(right_fit[0] *
(nonzeroy**2) + right_fit[1] * nonzeroy + right_fit[2] - margin)) &
(nonzerox <
(right_fit[0] *
(nonzeroy**2) + right_fit[1] * nonzeroy + right_fit[2] + margin)))
# Again, extract left and right line pixel positions
leftx = nonzerox[left_lane_inds]
lefty = nonzeroy[left_lane_inds]
rightx = nonzerox[right_lane_inds]
righty = nonzeroy[right_lane_inds]
# Fit a second order polynomial to each
left_fit = np.polyfit(lefty, leftx, 2)
right_fit = np.polyfit(righty, rightx, 2)
return LaneLine(left_fit, left_lane_inds, binary_warped.shape), \
LaneLine(right_fit, right_lane_inds, binary_warped.shape)
def _lane_lines_average(self):
"""
Averages the lane lines over (a part of) the available frame history
:return: the fitted x coordinates for left, right (or None, None if history is not available)
"""
if len(self.history) == 0:
return None, None
elif len(self.history) == 1:
return self.history[0].left_lane.fit_x, self.history[0].right_lane.fit_x
# else:
# return self.history[0].left_lane.fit_x, self.history[0].right_lane.fit_x
else:
end_index = min(len(self.history), 5)
weights = [100, 10] if end_index == 2 else [100, 10, 1]
left_poly = np.average([frame.left_lane.poly for frame in islice(self.history, end_index)], axis=0)
right_poly = np.average([frame.right_lane.poly for frame in islice(self.history, end_index)], axis=0,)
# weights = [100, 10] if end_index == 2 else [100, 10, 1]
# left_poly = np.average([frame.left_lane.poly for frame in islice(self.history, end_index)], axis=0,
# weights=weights)
# right_poly = np.average([frame.right_lane.poly for frame in islice(self.history, end_index)], axis=0,
# weights=weights)
fit_y = self.history[0].left_lane.fit_y
return LaneLine.calculate_points_along_line(left_poly, fit_y), LaneLine.calculate_points_along_line(
right_poly, fit_y)
class LaneDetector:
"""
Detects lane lines.
:param Camera camera: Calibrated camera responsible for removing distortions in images.
"""
def __init__(self, camera):
"""
Initializer.
"""
self._camera = camera
source_points = np.float32([[170, 720], [566, 450], [716, 450], [1120, 720]])
dest_points = np.float32([[350, 720], [200, 0], [1080, 0], [930, 720]])
self._transformation_matrix = cv2.getPerspectiveTransform(source_points, dest_points)
self._inverse_transformation_matrix = cv2.getPerspectiveTransform(dest_points, source_points)
sliding_windows_nr = 9
margin = 100
recenter_threshold = 50
self._left_lane_line = LaneLine(sliding_windows_nr, margin, recenter_threshold)
self._right_lane_line = LaneLine(sliding_windows_nr, margin, recenter_threshold)
def detect_lane_lines_in_video(self, video_path, output_file_name):
"""
Detects lane lines in an entire video clip.
:param str video_path : Path to a source video file.
:param str output_file_name: Name of the output file.
"""
source_clip = VideoFileClip(video_path)
converted_clip = source_clip.fl_image(self.detect_lane_lines_in_image)
converted_clip.write_videofile(output_file_name, audio=False)
def detect_lane_lines_in_image(self, image):
"""
Detects lane lines in single image,
:param np.ndarray image: An image to detect lines on.
:rtype: np.ndarray
"""
undistorted_image = self._camera.correct_distortions(image)
thresholded_image = self.threshold(undistorted_image)
birdseye_view_image = self.transform_perspective(thresholded_image)
self.update_lane_lines_positions(birdseye_view_image)
lanes_detected = self.draw_lanes_on_image(undistorted_image, birdseye_view_image)
return lanes_detected
def correct_distortions(self, image):
"""
Corrects image distortions.
:param np.ndarray image: Image with distortions.
:rtype: np.ndarray
"""
return self._camera.correct_distortions(image)
def threshold(self, image):
"""
Thresholds an image in order to find lines.
:param np.ndarray image: Image to put thresholds on.
:rtype: np.ndarray
"""
hls = cv2.cvtColor(image, cv2.COLOR_BGR2HLS).astype(np.float)
s_channel = hls[:, :, 2]
s_thresh = (130, 255)
s_binary = np.zeros_like(s_channel)
s_binary[(s_channel >= s_thresh[0]) & (s_channel <= s_thresh[1])] = 1
l_channel = hls[:, :, 1]
sobel_on_l_thresh = (40, 255)
sobel_l = cv2.Sobel(l_channel, cv2.CV_64F, 1, 0, ksize=3)
abs_sobel_l = np.absolute(sobel_l)
scaled_sobel_l = np.uint8(255 * abs_sobel_l / np.max(abs_sobel_l))
sobel_on_l_binary = np.zeros_like(scaled_sobel_l)
sobel_on_l_binary[(scaled_sobel_l >= sobel_on_l_thresh[0]) & (scaled_sobel_l <= sobel_on_l_thresh[1])] = 1
sobel_s_thresh = (30, 255)
sobel_s = cv2.Sobel(s_channel, cv2.CV_64F, 1, 0, ksize=3)
abs_sobel_s = np.absolute(sobel_s)
scaled_sobel_s = np.uint8(255 * abs_sobel_s / np.max(abs_sobel_s))
sobel_on_s_binary = np.zeros_like(scaled_sobel_s)
sobel_on_s_binary[(scaled_sobel_s >= sobel_s_thresh[0]) & (scaled_sobel_s <= sobel_s_thresh[1])] = 1
combined = np.zeros_like(s_binary)
combined[((sobel_on_l_binary == 1) | ((sobel_on_s_binary == 1) & (s_binary == 1)))] = 255
return combined
def transform_perspective(self, image):
"""
Transforms image perspective.
:param np.ndarray image: Image to transform.
:rtype: np.ndarray
"""
warped = cv2.warpPerspective(image, self._transformation_matrix, (1200, 720), flags=cv2.INTER_LINEAR)
return warped
def update_lane_lines_positions(self, binary_warped_image):
"""
Looks for lane lines in a binary warped image.
:param np.ndarray binary_warped_image: Binary warped image
"""
histogram = np.sum(binary_warped_image[binary_warped_image.shape[0] / 2:, :], axis=0)
midpoint = np.int(histogram.shape[0] / 2)
left_lane_base_x_position = np.argmax(histogram[:midpoint])
self._left_lane_line.find_in(binary_warped_image, left_lane_base_x_position)
right_lane_base_x_position = np.argmax(histogram[midpoint:]) + midpoint
self._right_lane_line.find_in(binary_warped_image, right_lane_base_x_position)
left_current_fit = self._left_lane_line.current_polynomial_fit()
right_current_fit = self._right_lane_line.current_polynomial_fit()
C_difference = left_current_fit[2] - right_current_fit[2]
B_difference = left_current_fit[1] - right_current_fit[1]
A_difference = left_current_fit[0] - right_current_fit[0]
if not (
-655 < C_difference < -400 and
-0.29 < B_difference < 0.3 and
-2.33958539e-04 < A_difference < 1.59358225e-04
):
self._left_lane_line.discard_latest_fit()
self._right_lane_line.discard_latest_fit()
def draw_lanes_on_image(self, image, warped_image):
"""
Draws lane lines on an image.
:param np.ndarray image : Image to draw on
:param np.ndarray warped_image: Warped perspective image.
:rtype: np.ndarray
"""
y_points = np.linspace(0, warped_image.shape[0] - 1, warped_image.shape[0])
left_x_points = self._left_lane_line.generate_x_points(y_points)
right_x_points = self._right_lane_line.generate_x_points(y_points)
left_lane_points = np.array([np.transpose(np.vstack([left_x_points, y_points]))])
right_lane_points = np.array([np.flipud(np.transpose(np.vstack([right_x_points, y_points])))])
combined_points = np.hstack((left_lane_points, right_lane_points))
warp_zero = np.zeros_like(warped_image).astype(np.uint8)
warped_drawn_lane = np.dstack((warp_zero, warp_zero, warp_zero))
cv2.fillPoly(warped_drawn_lane, np.int_([combined_points]), (0, 255, 0))
original_perspective_drawn_lane = cv2.warpPerspective(
warped_drawn_lane, self._inverse_transformation_matrix, (image.shape[1], image.shape[0])
)
result = cv2.addWeighted(image.astype(np.uint8), 1, original_perspective_drawn_lane, 0.3, 0)
curvature_text = 'Radius of curvature: {:.2f} m'.format(self._left_lane_line.calculate_line_curvature())
distance_from_center_text = 'Distance from center: {:.2f} m'.format(
self._left_lane_line.calculate_distance_from_center(self._right_lane_line)
)
cv2.putText(result, curvature_text, (100, 100), cv2.FONT_HERSHEY_COMPLEX, 1.0, (255, 255, 255))
cv2.putText(result, distance_from_center_text, (100, 200), cv2.FONT_HERSHEY_COMPLEX, 1.0, (255, 255, 255))
return result
# -----------------------------------------------------------------------------
# sample points belong to lines
# -----------------------------------------------------------------------------
points = sample_lines(threshed)
left = []
right = []
for i in range(len(points)):
if points[i][0][0] < warped.shape[1] / 2:
left.append(i)
elif points[i][0][0] > warped.shape[1] / 2:
right.append(i)
y_fit = np.arange(warped.shape[0], 0, -10)
left_line = LaneLine(y_fit)
right_line = LaneLine(y_fit)
if len(left) > 0:
left_line.points = points[left[0]]
else:
left_line.points = None
if len(right) > 0:
right_line.points = points[right[-1]]
else:
right_line.points = None
# Draw yellow lane lines in a black ground
warp_zero = np.zeros_like(warped).astype(np.uint8)
for line in (left_line, right_line):
def find_lines_with_sliding_windows(binary_warped,
num_windows=9,
window_margin=100,
pixel_threshold=50):
"""
:param binary_warped: the input image
:param num_windows: the number of windows per line to use
:param window_margin: the width of the windows +/- margin
:param pixel_threshold: the minimum number of pixels found to recenter window
:return: left_lane: LaneLine, right_lane: LaneLine
"""
# Assuming you have created a warped binary image called "binary_warped"
# Take a histogram of the bottom half of the image
histogram = np.sum(binary_warped[binary_warped.shape[0] // 2:, :], axis=0)
# Find the peak of the left and right halves of the histogram
# These will be the starting point for the left and right lines
midpoint = np.int(histogram.shape[0] // 2)
leftx_base = np.argmax(histogram[:midpoint])
rightx_base = np.argmax(histogram[midpoint:]) + midpoint
# Set height of windows
window_height = np.int(binary_warped.shape[0] // num_windows)
# Identify the x and y positions of all nonzero pixels in the image
nonzero = binary_warped.nonzero()
nonzeroy = np.array(nonzero[0])
nonzerox = np.array(nonzero[1])
# Current positions to be updated for each window
leftx_current = leftx_base
rightx_current = rightx_base
# Create empty lists to receive left and right lane pixel indices
left_lane_inds = []
right_lane_inds = []
# Create lists to store the search windows
left_lane_windows = []
right_lane_windows = []
# Step through the windows one by one
for window in range(num_windows):
# Identify window boundaries in x and y (and right and left)
win_y_low = binary_warped.shape[0] - (window + 1) * window_height
win_y_high = binary_warped.shape[0] - window * window_height
win_xleft_low = leftx_current - window_margin
win_xleft_high = leftx_current + window_margin
win_xright_low = rightx_current - window_margin
win_xright_high = rightx_current + window_margin
# Collect the windows
left_lane_windows.append(
((win_xleft_low, win_y_low), (win_xleft_high, win_y_high)))
right_lane_windows.append(
((win_xright_low, win_y_low), (win_xright_high, win_y_high)))
# Identify the nonzero pixels in x and y within the window
good_left_inds = ((nonzeroy >= win_y_low) & (nonzeroy < win_y_high) &
(nonzerox >= win_xleft_low) &
(nonzerox < win_xleft_high)).nonzero()[0]
good_right_inds = ((nonzeroy >= win_y_low) & (nonzeroy < win_y_high) &
(nonzerox >= win_xright_low) &
(nonzerox < win_xright_high)).nonzero()[0]
# Append these indices to the lists
left_lane_inds.append(good_left_inds)
right_lane_inds.append(good_right_inds)
# If you found > minpix pixels, recenter next window on their mean position
if len(good_left_inds) > pixel_threshold:
leftx_current = np.int(np.mean(nonzerox[good_left_inds]))
if len(good_right_inds) > pixel_threshold:
rightx_current = np.int(np.mean(nonzerox[good_right_inds]))
# Concatenate the arrays of indices
left_lane_inds = np.concatenate(left_lane_inds)
right_lane_inds = np.concatenate(right_lane_inds)
# Extract left and right line pixel positions
leftx = nonzerox[left_lane_inds]
lefty = nonzeroy[left_lane_inds]
rightx = nonzerox[right_lane_inds]
righty = nonzeroy[right_lane_inds]
# Fit a second order polynomial to each
left_fit = np.polyfit(
lefty, leftx,
2) if len(lefty) > 0 and len(leftx) > 0 else [np.array([False])]
right_fit = np.polyfit(
righty, rightx,
2) if len(righty) > 0 and len(rightx) > 0 else [np.array([False])]
return LaneLine(left_fit, left_lane_inds, binary_warped.shape, left_lane_windows), \
LaneLine(right_fit, right_lane_inds, binary_warped.shape, right_lane_windows)
def process_image(self, image):
undistorted = cv2.undistort(image, self.calibration.camera_matrix, self.calibration.distortion_coefficients, None)
filtered = get_edge_mask(undistorted, input_image_color_space='rgb')
warped = warp_birds_eye(filtered, self.source_points, self.destination_points)
windows, left_window_points, right_window_points = sliding_window(warped)
left_line = LaneLine(warped, left_window_points)
right_line = LaneLine(warped, right_window_points)
left_line.fit()
right_line.fit()
moving_n = 8
self.left_lines.append(left_line)
self.right_lines.append(right_line)
original_birds_eye = warp_birds_eye(undistorted, self.source_points, self.destination_points)
filtered_birds_eye = warp_birds_eye(filtered, self.source_points, self.destination_points)
y = left_line.generate_y()
lane_drawing = np.zeros_like(original_birds_eye)
left_x = np.median(np.array([ l.evaluate() for l in self.left_lines[-moving_n:] ]), axis=0)
right_x = np.median(np.array([ l.evaluate() for l in self.right_lines[-moving_n:] ]), axis=0)
left_points = np.vstack([left_x, y]).T
right_points = np.vstack([right_x, y]).T
all_points = np.concatenate([left_points, right_points[::-1], left_points[:1]])
cv2.fillConvexPoly(lane_drawing, np.int32([all_points]), (0, 255, 0))
unwarped_lane_drawing = warp_birds_eye(lane_drawing, self.source_points, self.destination_points, reverse=True)
original_perspective_windows = warp_birds_eye(windows, self.source_points, self.destination_points, reverse=True)
frame = cv2.addWeighted(undistorted, 1.0, unwarped_lane_drawing, 0.2, 0)
l = np.average(np.array([line.camera_distance() for line in self.left_lines[-moving_n:]]))
r = np.average(np.array([line.camera_distance() for line in self.right_lines[-moving_n:]]))
if l - r > 0:
self.draw_text(frame, '{:.3} cm right of center'.format((l - r) * 100), 20, 115)
else:
self.draw_text(frame, '{:.3} cm left of center'.format((r - l) * 100), 20, 115)
self.curvatures.append(np.mean([left_line.curvature_radius(), right_line.curvature_radius()]))
curvature = np.average(self.curvatures[-moving_n:])
self.draw_text(frame, 'Radius of curvature: {:.3} km'.format(curvature / 1000), 20, 80)
if self.output_debug_image is True:
window_centroids = find_window_centroids(warped)
edges_with_windows = draw_windows(warped, window_centroids)
color_edge_mask = get_edge_mask(undistorted, input_image_color_space='rgb', return_all_channels=True)
color_edge_mask_birds_eye = warp_birds_eye(color_edge_mask, self.source_points, self.destination_points)
gray_edge_mask_birds_eye = warp_birds_eye(filtered, self.source_points, self.destination_points)
windows_perspective = warp_birds_eye(edges_with_windows, self.destination_points, self.source_points)
plt.imsave('../video_output/' + '{:03}'.format(self.frame_number) + '-0-undistorted.jpg', undistorted)
plt.imsave('../video_output/' + '{:03}'.format(self.frame_number) + '-1-edges.jpg', color_edge_mask)
plt.imsave('../video_output/' + '{:03}'.format(self.frame_number) + '-2-undistorted-birds-eye.jpg', original_birds_eye)
plt.imsave('../video_output/' + '{:03}'.format(self.frame_number) + '-3-edges-birds-eye.jpg', color_edge_mask_birds_eye)
plt.imsave('../video_output/' + '{:03}'.format(self.frame_number) + '-4-windows.jpg', edges_with_windows)
plt.imsave('../video_output/' + '{:03}'.format(self.frame_number) + '-5-windows-birds-eye.jpg', windows_perspective)
plt.imsave('../video_output/' + '{:03}'.format(self.frame_number) + '-6-synthetic-lane-birds-eye.jpg', lane_drawing)
plt.imsave('../video_output/' + '{:03}'.format(self.frame_number) + '-7-output.jpg', frame)
self.frame_number += 1
return frame