def visualize_perspective_transform(img):
pipeline = LaneDetection()
img = pipeline.distortion_correction(img)
thresh = pipeline.combined_threshold(img)
warped, M = pipeline.warp_perspective(thresh)
# convert from BGR to RGB
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
# add src lines to image
src_pts = np.array(pipeline.src, np.int32)
src_pts = src_pts.reshape((-1, 1, 2))
img = cv2.polylines(img, [src_pts], True, (255, 0, 0), 5)
# add dst lines to image & convert to rgb so the lines are red
thresh = np.array(np.dstack((thresh, thresh, thresh)) * 255, np.uint8)
dst_pts = np.array(pipeline.dst, np.int32)
dst_pts = dst_pts.reshape((-1, 1, 2))
thresh = cv2.polylines(thresh, [dst_pts], True, (255, 0, 0), 5)
warped = np.array(np.dstack((warped, warped, warped)) * 255, np.uint8)
dst_pts = np.array(pipeline.dst, np.int32)
dst_pts = dst_pts.reshape((-1, 1, 2))
warped = cv2.polylines(warped, [dst_pts], True, (255, 0, 0), 5)
# Visualize perspective transformation
side_by_side(
(img, thresh, warped),
('Original Image', 'Combined Threshold', 'Warped Perspective'),
cols=3)
def grab(self):
global window_search
window_search = True
# if TRUE, set mouse callback function
if self.mouse_callback:
cv2.namedWindow("screen_grab")
cv2.setMouseCallback("screen_grab", mouse_coords)
# create lane detection class instance
find_lane = LaneDetection()
with mss.mss() as sct:
while 'Screen capturing':
last_time = time.time()
# Get raw pixels from the screen, save it to a Numpy array
img = np.array(sct.grab(self.monitor))
try:
processed_img = find_lane.get_image(img)
cv2.imshow('screen_grab', processed_img)
except:
print("Finding Lanes...")
pass
if self.show_fps:
print('FPS: {0:.0f}'.format(1 / (time.time() - last_time)))
# Press "q" to quit
if cv2.waitKey(25) & 0xFF == ord('q'):
cv2.destroyAllWindows()
break
def calculate_score_for_leaderboard():
"""
Evaluate the performance of the network. This is the function to be used for
the final ranking on the course-wide leader-board, only with a different set
of seeds. Better not change it.
"""
# action variables
a = np.array([0.0, 0.0, 0.0])
# init environement
env = CarRacing()
env.render()
env.reset()
seeds = [
22597174, 68545857, 75568192, 91140053, 86018367, 49636746, 66759182,
91294619, 84274995, 31531469
]
total_reward = 0
for episode in range(10):
env.seed(seeds[episode])
observation = env.reset()
# init modules of the pipeline
LD_module = LaneDetection(gradient_threshold=25, spline_smoothness=20)
LatC_module = LateralController(gain_constant=1.8,
damping_constant=0.05)
LongC_module = LongitudinalController(KD=0.001)
reward_per_episode = 0
for t in range(600):
# perform step
s, r, done, speed, info = env.step(a)
# lane detection
lane1, lane2 = LD_module.lane_detection(s)
# waypoint and target_speed prediction
waypoints = waypoint_prediction(lane1, lane2)
target_speed = target_speed_prediction(waypoints,
max_speed=60,
exp_constant=6)
# control
a[0] = LatC_module.stanley(waypoints, speed)
a[1], a[2] = LongC_module.control(speed, target_speed)
# reward
reward_per_episode += r
env.render()
print('episode %d \t reward %f' % (episode, reward_per_episode))
total_reward += np.clip(reward_per_episode, 0, np.infty)
print('---------------------------')
print(' total score: %f' % (total_reward / 10))
print('---------------------------')
def evaluate():
"""
"""
# action variables
a = np.array([0.0, 0.0, 0.0])
# init environement
env = CarRacing()
env.render()
env.reset()
for episode in range(5):
observation = env.reset()
# init modules of the pipeline
LD_module = LaneDetection()
LatC_module = LateralController()
LongC_module = LongitudinalController()
reward_per_episode = 0
for t in range(500):
# perform step
s, r, done, speed, info = env.step(a)
# lane detection
lane1, lane2 = LD_module.lane_detection(s)
# waypoint and target_speed prediction
waypoints = waypoint_prediction(lane1, lane2)
target_speed = target_speed_prediction(waypoints,
max_speed=60,
exp_constant=4.5)
# control
a[0] = LatC_module.stanley(waypoints, speed)
a[1], a[2] = LongC_module.control(speed, target_speed)
# reward
reward_per_episode += r
env.render()
print('episode %d \t reward %f' % (episode, reward_per_episode))
def __main__():
# get video stream
video_cap = imageio.get_reader(config["project_video"])
# polynomial lane fit
lanes_fit = []
# history of heatmaps to reject false positives
history = deque(maxlen=config["history_limit"])
# classifier and scaler
classifier = Classifier.get_trained_classifier(use_pre_trained=True)
# load calibration parameters:
camera_matrix, dist_coef = PreProcessing.load_calibration_params()
for index, img in enumerate(video_cap):
if index % config["skip_frames"] == 0:
# get lanes
lanes_fit, img = LaneDetection.pipeline(img, lanes_fit,
camera_matrix, dist_coef)
# resize image to improve speed of vehicle detection using classifier
# jpg to png
if config["is_training_png"]:
img = Helper.scale_to_png(img)
# 3 channel without alpha
img = img[:, :, :config["channels"]]
bounding_boxes = []
# get bounding boxes for left side
x_start_stop_left, y_start_stop_left = config["xy_start_stop_left"]
bounding_boxes += WindowSearch.get_bounding_boxes(
img, classifier, x_start_stop_left, y_start_stop_left)
# # get bounding boxes for top side
x_start_stop_top, y_start_stop_top = config["xy_start_stop_top"]
bounding_boxes += WindowSearch.get_bounding_boxes(
img, classifier, x_start_stop_top, y_start_stop_top)
# get bounding boxes for right side
x_start_stop_right, y_start_stop_right = config[
"xy_start_stop_right"]
bounding_boxes += WindowSearch.get_bounding_boxes(
img, classifier, x_start_stop_right, y_start_stop_right)
# remove false positives and duplicates from detection
detected_cars = Helper.remove_false_positives(
img, bounding_boxes, history)
# visualization
plt.imshow(detected_cars)
plt.pause(0.0001)
def main():
NX = 9
NY = 6
h = 720
w = 1280
nwindows = 9
margin = 100
tol = 50
CALIBRATION_IMAGES = glob.glob('../camera_cal/*')
lane_detector = LaneDetection(h, w, CALIBRATION_IMAGES, NX, NY, nwindows,
margin, tol)
video_output = '../output_videos/project_video.mp4'
clip1 = VideoFileClip("../project_video.mp4")
white_clip = clip1.fl_image(lane_detector.process_frame)
white_clip.write_videofile(video_output, audio=False)
def main():
'''
Video streaming.
Press 'q' to exit and 'w' to pause video.
:return:
'''
if len(sys.argv) > 1:
cap = cv2.VideoCapture(sys.argv[1])
else:
cap = cv2.VideoCapture('videos/road01.mp4')
fps = cap.get(cv2.CAP_PROP_FPS)
while cap.isOpened():
time_start = time.time()
_, frame = cap.read()
frame = LaneDetection(frame).lane_detection()
cv2.imshow("Lane Detection", frame)
key = cv2.waitKey(1)
if key == ord('q'):
break
if key == ord('w'):
cv2.waitKey(-1)
sync_fps(time_start, fps)
cap.release()
cv2.destroyAllWindows()
if k==key.DOWN: a[2] = 0
# init environement
env = CarRacing()
env.render()
env.viewer.window.on_key_press = key_press
env.viewer.window.on_key_release = key_release
env.reset()
# define variables
total_reward = 0.0
steps = 0
restart = False
# init modules of the pipeline
LD_module = LaneDetection()
# init extra plot
fig = plt.figure()
plt.ion()
plt.show()
while True:
# perform step
s, r, done, speed, info = env.step(a)
# lane detection
lane1, lane2 = LD_module.lane_detection(s)
# waypoint and target_speed prediction
waypoints = waypoint_prediction(lane1, lane2)
#'GTKAgg', 'template', 'pgf', 'pdf', 'nbAgg', 'GTK3Cairo', 'GTKCairo', 'ps',
#'Qt5Agg', 'TkAgg', 'WX', 'gdk', 'GTK', 'GTK3Agg', 'svg', 'WebAgg', 'emf']
#matplotlib.use('agg')
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
#%matplotlib qt
#from IPython import get_ipython
#get_ipython().run_line_magic('matplotlib', 'qt')
from lane_detection import LaneDetection
'''
Compute the camera calibration matrix and distortion coefficients
given a set of chessboard images.
'''
## Instantiate ride
ride = LaneDetection()
# Get images path
images = glob.glob('../data/test_images/test*.jpg')
i = 0
for fname in images:
original_img = mpimg.imread(fname) #in RGB
undistorted_img = ride.camera.undistort(original_img)
'''
Get combined binary image
'''
combined_binary = ride.thresholding_pipeline(undistorted_img)
fig = plt.figure(i * 4)
# Plotting thresholded images
fig_handle = fig.add_subplot(1, 2, 1)
import numpy as np
import cv2
from lane_detection import LaneDetection
# Initiate The Lane Detection Pipeline
pipeline = LaneDetection()
name = 'project_'
# name = 'challenge_'
# name = 'harder_challenge_'
cap = cv2.VideoCapture(name + 'video.mp4')
# Check if camera opened successfully
if cap.isOpened() is False:
print("Error opening video stream or file")
# Default resolutions of the frame are obtained.The default resolutions are system dependent.
# We convert the resolutions from float to integer.
frame_width = int(cap.get(3))
frame_height = int(cap.get(4))
# Define the codec and create VideoWriter object.The output is stored in 'outpy.mp4' file.
out = cv2.VideoWriter(name + 'out.mp4', cv2.VideoWriter_fourcc(*'H264'), 24,
(frame_width, frame_height))
while cap.isOpened():
ret, frame = cap.read()
if ret == True:
def camera_callback(self, data):
# Dividing frame rate by 3 (10fps)
if self.counter % 3 != 0:
self.counter += 1
return
else:
self.counter = 1
try:
#### direct conversion to CV2 ####
np_arr = np.fromstring(data.data, np.uint8)
cv_image = cv2.imdecode(np_arr, cv2.IMREAD_COLOR)
except:
print("Error conversion to CV2")
# Undistorted image
# undist = cv2.undistort(cv_image, self.mtx, self.dist, None, self.mtx)
# Create lane detection object
lane_detection_object = LaneDetection()
# Lane lines detection and process cross track error
cte, angle, final_img = lane_detection_object.processImage(cv_image)
cmd_vel = Twist()
if cte is not None and angle is not None:
angular_z = self.Kp * cte + self.Kd * (cte - self.last_cte)
self.last_cte = cte
linear_x = self.max_desire_linear_vel
angular_z = max(angular_z, -2.0) if angular_z < 0 else min(
angular_z, 2.0)
else:
angular_z = self.Kp * self.last_cte * 1.9
linear_x = self.max_desire_linear_vel
angular_z = max(angular_z, -2.0) if angular_z < 0 else min(
angular_z, 2.0)
cmd_vel.linear.x = linear_x
cmd_vel.angular.z = angular_z
self.cmd_vel_pub.publish(cmd_vel)
if self.debug:
w_l = ((2 * linear_x) -
(angular_z * self.wheel_base)) / (2 * self.wheel_radius)
w_r = ((2 * linear_x) +
(angular_z * self.wheel_base)) / (2 * self.wheel_radius)
pwm_l = int((255 * w_l) / self.max_omega)
if abs(pwm_l) > 255:
pwm_l = 255
elif abs(pwm_l) < 0:
pwm_l = 0
else:
pwm_l = abs(pwm_l)
pwm_r = int((255 * w_r) / self.max_omega)
if abs(pwm_r) > 255:
pwm_r = 255
elif abs(pwm_r) < 0:
pwm_r = 0
else:
pwm_r = abs(pwm_r)
print("w_l : " + str(w_l))
print("pwm_l : " + str(pwm_l))
print("w_r : " + str(w_r))
print("pwm_r : " + str(pwm_r))
print("----------")
if lane_detection_object.draw_img:
cv2.imshow("Original Image", final_img)
cv2.waitKey(1)
import cv2
from lane_detection import LaneDetection
image_file = "D:/AutoDrive/0928/start.jpg"
video_file = "D:/AutoDrive/0928/raw.mp4"
output_file = "D:/AutoDrive/0928/out.mp4"
# 创建车道线检测对象
lane_det = LaneDetection(video_file, output_file)
img = cv2.imread(image_file)
# lane_det.gray_scale(image_file) # 输出灰度图
# lane_det.binarization(image_file) # 输出二值化图
# blur_gray = lane_det.gaussian_blur(image_file, show=False) # 输出高斯平滑图
# edges = lane_det.edge_detect(blur_gray, show=False) # 输出边界检测图，需要用到高斯平滑图结果
# roi_edges = lane_det.roi(edges, show=True) # 输出兴趣区域截取后的边界，需要用到边界检测图
# lane_det.hough_transform(roi_edges, show=True) # 输出霍夫变换图，需要用到兴趣区域边界
# cv2.imwrite("D:/AutoDrive/0928/start_edges.jpg", edges) # 将图片输出为文件
# res_img = lane_det.process_an_image(img, show=True) # 处理一张图片
lane_det.process_a_video() # 处理一个视频
# lane_det.generate_video() # 输出视频
# action variables
a = np.array( [0.0, 0.0, 0.0] )
# init environement
env = CarRacing()
env.render()
env.reset()
# define variables
total_reward = 0.0
steps = 0
restart = False
# init modules of the pipeline
LD_module = LaneDetection()
LatC_module = LateralController()
LongC_module = LongitudinalController()
# init extra plot
fig = plt.figure()
plt.ion()
plt.show()
while True:
# perform step
s, r, done, speed, info = env.step(a)
# lane detection
lane1, lane2 = LD_module.lane_detection(s)
# init environement
env = CarRacing()
env.render()
env.viewer.window.on_key_press = key_press
env.viewer.window.on_key_release = key_release
env.reset()
# define variables
total_reward = 0.0
steps = 0
restart = False
# init modules of the pipeline
LD_module = LaneDetection()
# init extra plot
fig = plt.figure()
plt.ion()
plt.show()
while True:
# perform step
s, r, done, speed, info = env.step(a)
# lane detection
splines = LD_module.lane_detection(s)
# reward
total_reward += r
# Import everything needed to edit/save/watch video clips
from moviepy.editor import VideoFileClip
#%matplotlib qt
from IPython import get_ipython
#get_ipython().run_line_magic('matplotlib', 'qt')
from lane_detection import LaneDetection
'''
Compute the camera calibration matrix and distortion coefficients
given a set of chessboard images.
'''
## Instantiate ride
ride = LaneDetection()
'''
Video rubric
'''
# Video file path
video_name = '../data/test_videos/project_video.mp4'
video_name_save = '../data/test_videos/project_video_48cfu.mp4'
#video = imageio.get_reader(video_name)
video = VideoFileClip(video_name)
new_clip = video.fl_image(ride.process_image)
#get_ipython().run_line_magic('matplotlib', 'time')
new_clip.write_videofile(video_name_save, audio=False)
input("Press Enter to continue...")
def main():
global original, image
tracker = {'apply mask': 0, 'canny high threshold': 0
,'canny low threshold' : 0, 'canny low threshold': 0
, 'hough threshold': 1, 'hough max line gap': 1
, 'hough min line length': 1}
img = image
img_g = image.copy()
font = cv2.FONT_HERSHEY_SIMPLEX
ld = LaneDetection()
while(True):
cv2.imshow('img', img)
k = cv2.waitKey(1) & 0xFF
if k == 27:
break
# Get current positions of the trackers
apply_mask = cv2.getTrackbarPos('apply mask', 'img')
canny_high_threshold = cv2.getTrackbarPos('canny high threshold', 'img')
canny_low_threshold = cv2.getTrackbarPos('canny low threshold', 'img')
hough_threshold = cv2.getTrackbarPos('hough threshold', 'img')
hough_max_line_gap = cv2.getTrackbarPos('hough max line gap', 'img')
hough_min_line_length = cv2.getTrackbarPos('hough min line length', 'img')
if canny_high_threshold != tracker['canny high threshold'] or canny_low_threshold != tracker['canny low threshold']:
img = ld.findCannyEdges(image.copy(), canny_low_threshold, canny_high_threshold)
img_g = img
tracker['canny high threshold'] = canny_high_threshold
tracker['canny low threshold'] = canny_low_threshold
'''
print 'Canny_low: ', canny_low_threshold
print 'Canny_high: ', canny_high_threshold
'''
if apply_mask == 0 and apply_mask != tracker['apply mask']:
img = image.copy()
elif apply_mask == 1:
img = ld.getRoi(img)
if hough_threshold != tracker['hough threshold'] or hough_max_line_gap != tracker['hough max line gap'] or hough_min_line_length != tracker['hough min line length']:
lines = ld.findHoughLines(img_g, hough_threshold, hough_min_line_length, hough_max_line_gap)
canvas = np.zeros(original.shape, np.uint8)
canvas = ld.drawLines(canvas, lines, (0, 200, 0), 2)
img_c = cv2.cvtColor(img_g.copy(), cv2.COLOR_GRAY2BGR)
img = cv2.bitwise_and(canvas, img_c)
tracker['hough threshold'] = hough_threshold
tracker['hough max line gap'] = hough_max_line_gap
tracker['hough min line length'] = hough_min_line_length
'''
print 'hough_threshold', hough_threshold
print 'hough_max_line_gap', hough_max_line_gap
print 'hough_min_line_length', hough_min_line_length
'''
# Put the text of tracker's parameters on the iamge
cv2.putText(img, 'Canny high thresh: '+ str(canny_high_threshold), (10, 440), font, 0.5, (255,255,255),1, cv2.LINE_AA)
cv2.putText(img, 'Canny low thresh: '+ str(canny_low_threshold), (10, 460), font, 0.5, (255,255,255),1, cv2.LINE_AA)
cv2.putText(img, 'hough thresh: '+ str(hough_threshold), (10, 480), font, 0.5, (255,255,255),1, cv2.LINE_AA)
cv2.putText(img, 'hough max line gap: '+ str(hough_max_line_gap), (10, 500), font, 0.5, (255,255,255),1, cv2.LINE_AA)
cv2.putText(img, 'hough min line length: '+ str(hough_min_line_length), (10, 520), font, 0.5, (255,255,255),1, cv2.LINE_AA)
cv2.destroyAllWindows()
leftLine = [0] * 4
count = 0
smooth = 5
mtx, dist = camera_calibration()
leftLines = []
rightLines = []
lastLeftLine = Line()
lastRightLine = Line()
cap = cv2.VideoCapture('./project_video.mp4')
# cap = cv2.VideoCapture('./challenge_video.mp4')
# cap = cv2.VideoCapture('./harder_challenge_video.mp4')
detection = LaneDetection(nwindows=20)
fourcc = cv2.VideoWriter_fourcc('X', '2', '6', '4')
out = cv2.VideoWriter('output_video.mp4', fourcc, cap.get(cv2.CAP_PROP_FPS),
(1280, 720))
while (True):
# Capture frame-by-frame
ret, frame = cap.read()
if ret == True:
detection.setImage(frame)
left_fit, right_fit, left_curvature, right_curvature, left_fitx, right_fitx, dif_meters, left_base, right_base = detection.process_detection(
mtx, dist)
