def frameProcessPipeline(frame, keepState=True):
global leftLine, rightLine, _processedFrameCount
# undistort the image using coefficients found in calibration
undistortedImage = undistort(frame, mtx, dist, verbose=False)
# binarize the frame s.t. lane lines are highlighted as much as possible
binaryImage = binarize(undistortedImage, verbose=False)
# compute perspective transform to obtain bird's eye view
warpedImage, M, Minv = warpPerspective(binaryImage, verbose=False)
# fit 2-degree polynomial curve onto lane lines found
if _processedFrameCount > 0 and keepState and leftLine.detected and rightLine.detected:
leftLine, rightLine, img_fit = usePreviousFits(warpedImage, leftLine, rightLine, verbose=False)
else:
leftLine, rightLine, img_fit = slidingWindowFits(warpedImage, leftLine, rightLine, numWindows=9, verbose=False)
# compute offset in meter from center of the lane
offFromCenter = calculateOffFromCenter(leftLine, rightLine, frame_width=frame.shape[1])
# draw the surface enclosed by lane lines back onto the original frame
image = drawOnRoad(undistortedImage, Minv, leftLine, rightLine, keepState)
# stitch on the top of final output images from different steps of the pipeline
blend_output = finalOutputDisplay(image, offFromCenter)
_processedFrameCount += 1
return blend_output
def process_pipeline(frame, keep_state=True):
"""
Apply whole lane detection pipeline to an input color frame.
:param frame: input color frame
:param keep_state: if True, lane-line state is conserved (this permits to average results)
:return: output blend with detected lane overlaid
"""
global line_lt, line_rt, processed_frames
# undistort the image using coefficients found in calibration
img_undistorted = frame
# binarize the frame s.t. lane lines are highlighted as much as possible
img_binary = binarize(img_undistorted, verbose=False)
# compute perspective transform to obtain bird's eye view
img_birdeye, M, Minv = birdeye(img_binary, verbose=False)
# fit 2-degree polynomial curve onto lane lines found
# if processed_frames > 0 and keep_state and line_lt.detected and line_rt.detected:
# line_lt, line_rt, img_fit = get_fits_by_previous_fits(img_birdeye, line_lt, line_rt, verbose=False)
# else:
line_lt, line_rt, img_fit = get_fits_by_sliding_windows(img_birdeye,
line_lt,
line_rt,
n_windows=9,
verbose=False)
# compute offset in meter from center of the lane
offset_meter = compute_offset_from_center(line_lt,
line_rt,
frame_width=frame.shape[1])
# draw the surface enclosed by lane lines back onto the original frame
blend_on_road, intensity_lt, intensity_rt, intensity_road = draw_back_onto_the_road(
img_undistorted, Minv, line_lt, line_rt, keep_state)
csv.write("{}, {:.3f}, {:.3f}, {:.3f}, {}\n".format(
processed_frames, intensity_lt, intensity_rt, intensity_road,
str(line_rt.detected)))
csv.flush()
# stitch on the top of final output images from different steps of the pipeline
blend_output = prepare_out_blend_frame(blend_on_road, img_binary,
img_birdeye, img_fit, line_lt,
line_rt, offset_meter)
processed_frames += 1
return blend_output
def collect(self):
print("Start video stream")
stream_bytes = b' '
# connection = self.client.makefile('rb')
data_name = 0
while True:
stream_bytes += self.client.recv(1024)
first = stream_bytes.find(b'\xff\xd8')
last = stream_bytes.find(b'\xff\xd9')
if first != -1 and last != -1:
try:
jpg = stream_bytes[first:last + 2]
stream_bytes = stream_bytes[last + 2:]
imgBGR = cv.imdecode(np.frombuffer(jpg, dtype=np.uint8),
cv.IMREAD_COLOR)
#imgGRAY = cv.imdecode(np.frombuffer(jpg, dtype=np.uint8), cv.IMREAD_GRAYSCALE)
imgBINARY = binarize(img=imgBGR, verbose=True)
data_name += 1
roiGRAY = imgBINARY[120:240, :]
cv.imwrite("../data/picamera/img.jpg", imgBGR)
#cv.imshow('Origin', imgBGR)
# cv.imshow('GRAY', imgGRAY)
#print("roi")
cv.imshow('roi', roiGRAY)
#print("roi show")
# reshape the roi image into a vector
image_array = np.reshape(roiGRAY, (-1, 120, 320, 1))
# neural network makes prediction
self.steer.Set_Line(self.model.predict(image_array))
# self.steer.Set_Stopline(self.stopline.GetStopLine(roiGRAY))
#print("detecting...")
self.objDetect.Detection()
#print("detection complete")
self.steer.Control()
except:
continue
if cv.waitKey(1) & 0xFF == ord('q'):
break
def process_pipeline(frame, keep_state=True):
"""
Apply whole lane detection pipeline to an input color frame.
:param frame: input color frame
:param keep_state: if True, lane-line state is conserved (this permits to average results)
:return: output blend with detected lane overlaid
"""
global line_lt, line_rt, processed_frames
# binarize the frame s.t. lane lines are highlighted as much as possible
img_binary = binarize(frame, verbose=False)
# compute perspective transform to obtain bird's eye view, forward and backward transformation matrices
img_birdeye, _, Minv = birdeye(img_binary, verbose=False)
# fit 2-degree polynomial curve onto lane lines found
line_lt, line_rt = get_fits_by_sliding_windows(img_birdeye,
line_lt,
line_rt,
n_windows=9)
# compute offset in meter from center of the lane
offset_meter = compute_offset_from_center(line_lt,
line_rt,
frame_width=frame.shape[1])
# draw the surface enclosed by lane lines back onto the original frame
blend_on_road = draw_back_onto_the_road(frame, Minv, line_lt, line_rt,
keep_state)
# stitch on the top of final output images from different steps of the pipeline
blend_output = prepare_out_blend_frame(blend_on_road, line_lt, line_rt,
offset_meter)
processed_frames += 1
return blend_output
def process_pipeline(frame, keep_state=True):
"""
Apply whole lane detection pipeline to an input color frame.
:param frame: input color frame
:param keep_state: if True, lane-line state is conserved (this permits to average results)
:return: output blend with detected lane overlaid
"""
global line_lt, line_rt, processed_frames
# undistort the image using coefficients found in calibration
img_undistorted = undistort(frame, mtx, dist, verbose=False)
# binarize the frame s.t. lane lines are highlighted as much as possible
img_binary = binarize(img_undistorted, verbose=False)
# compute perspective transform to obtain bird's eye view
img_birdeye, M, Minv = birdeye(img_binary, verbose=False)
# fit 2-degree polynomial curve onto lane lines found
if processed_frames > 0 and keep_state and line_lt.detected and line_rt.detected:
line_lt, line_rt, img_fit = get_fits_by_previous_fits(img_birdeye, line_lt, line_rt, verbose=False)
else:
line_lt, line_rt, img_fit = get_fits_by_sliding_windows(img_birdeye, line_lt, line_rt, n_windows=9, verbose=False)
# compute offset in meter from center of the lane
offset_meter = compute_offset_from_center(line_lt, line_rt, frame_width=frame.shape[1])
# draw the surface enclosed by lane lines back onto the original frame
blend_on_road = draw_back_onto_the_road(img_undistorted, Minv, line_lt, line_rt, keep_state)
# stitch on the top of final output images from different steps of the pipeline
blend_output = prepare_out_blend_frame(blend_on_road, img_binary, img_birdeye, img_fit, line_lt, line_rt, offset_meter)
processed_frames += 1
return blend_output
# axarray[0].imshow(img, cmap='gray')
# for point in src:
# axarray[0].plot(*point, '.')
# axarray[1].set_title('After perspective transform')
# axarray[1].imshow(warped, cmap='gray')
# for point in dst:
# axarray[1].plot(*point, '.')
# for axis in axarray:
# axis.set_axis_off()
# plt.show()
return warped, M, Minv
if __name__ == '__main__':
ret, mtx, dist, rvecs, tvecs = calibrate_camera(
calib_images_dir='camera_cal')
# show result on test images
for test_img in glob.glob('test_images/*.jpg'):
img = cv2.imread(test_img)
img_undistorted = undistort(img, mtx, dist, verbose=False)
img_binary = binarize(img_undistorted, verbose=False)
img_birdeye, M, Minv = birdeye(cv2.cvtColor(img_undistorted,
cv2.COLOR_BGR2RGB),
verbose=True)
axarray[0].imshow(img, cmap='gray')
for point in src:
axarray[0].plot(*point, '.')
axarray[1].set_title('After perspective transform')
axarray[1].imshow(warped, cmap='gray')
for point in dst:
axarray[1].plot(*point, '.')
for axis in axarray:
axis.set_axis_off()
plt.show()
return warped, M, Minv
if __name__ == '__main__':
ret, mtx, dist, rvecs, tvecs = calibrate_camera(calib_images_dir='camera_cal')
# show result on test images
for test_img in glob.glob('test_images/*.jpg'):
img = cv2.imread(test_img)
img_undistorted = undistort(img, mtx, dist, verbose=False)
img_binary = binarize(img_undistorted, verbose=False)
img_birdeye, M, Minv = birdeye(cv2.cvtColor(img_undistorted, cv2.COLOR_BGR2RGB), verbose=True)
def main():
logging.basicConfig(level=logging.DEBUG)
cap = cv2.VideoCapture('footage/7_edit.avi')
'''
cap = cv2.VideoCapture(1)
cap.set(cv2.CAP_PROP_AUTO_EXPOSURE, 0.25)
cap.set(cv2.CAP_PROP_EXPOSURE, 0.01)
cap.set(cv2.CAP_PROP_FRAME_WIDTH,1280)
cap.set(cv2.CAP_PROP_FRAME_HEIGHT,720)
'''
DIM = (640, 480)
K = np.array([[359.0717640266508, 0.0, 315.08914578097387],
[0.0, 358.06497428501837, 240.75242680088732],
[0.0, 0.0, 1.0]])
D = np.array([[-0.041705903204711826], [0.3677107787593379],
[-1.4047363783373128], [1.578157237454529]])
profile = (DIM, K, D)
CC = CameraCalibration(profile)
yellow_HSV_th_min = np.array([0, 70, 70])
yellow_HSV_th_max = np.array([50, 255, 255])
line_lt = Line(buffer_len=time_window) # line on the left of the lane
line_rt = Line(buffer_len=time_window) # line on the right of the lane
processed_frames = 0
### traffic light detector setup ###
'''
red_bound = ([0,0,0], [255,255,360])
green_bound = ([0,0,0], [255,255,360])
color_bounds = {'red':red_bound, 'green':green_bound}
reference_images = []
reference_paths = glob('./reference/*.jpg')
for path in reference_paths:
reference_images.append(cv2.imread(path))
TLD = TrafficLightDetector(reference_images, color_bounds)
'''
while cap.isOpened():
ret, frame = cap.read()
### traffic light detection
# TODO: replace placeholder values
'''
while True:
state = TLD.get_state(frame)
logging.debug('traffic light state:', str(state))
if state == 'green':
comm.write_serial_message('s30')
break
'''
frame = CC.undistort(frame)
### calibration ###
### crop to ROI ###
### perspective transform ###
img, warped = debug_roi(frame, None, None)
### binarize frame ###
### get steering angle v1 ###
'''
_, lines = pathfinder.get_line_segments(warped)
grey = cv2.cvtColor(warped, cv2.COLOR_BGR2GRAY)
turn_angle = pathfinder.compute_turn_angle(grey)
print('turn angle:', turn_angle)
cv2.imshow('lines', lines)
'''
### get steering angle v2 ###
#mask, res = filter_hsv_colour(img, yellow_HSV_th_max, yellow_HSV_th_min)
img_binary = binarization_utils.binarize(warped, verbose=False)
cv2.imshow('img binary', img_binary)
### contours ###
_, contours, hierarchy = cv2.findContours(img_binary,
cv2.RETR_EXTERNAL,
cv2.CHAIN_APPROX_NONE)
img_binary_colour = cv2.cvtColor(img_binary, cv2.COLOR_GRAY2BGR)
if len(contours) > 0:
for cnt in contours:
if cnt.size >= 5:
epsilon = 0.1 * cv2.arcLength(cnt, True)
approx = cv2.approxPolyDP(cnt, epsilon, True)
ellipse = cv2.fitEllipse(cnt)
cv2.ellipse(img_binary_colour, ellipse, (255, 0, 0), 5)
rect = cv2.minAreaRect(cnt)
box = cv2.boxPoints(rect)
box = np.int0(box)
cv2.drawContours(img_binary_colour, [box], 0, (0, 0, 255),
5)
### basic pathfinder ###
#img_binary_colour = img_binary_colour[:][100:]
_, lines = pathfinder.get_line_segments(img_binary_colour)
turn_angle = pathfinder.compute_turn_angle(img_binary_colour)
print('turn angle:', turn_angle)
turn_angle += 90
turn_angle_message = str('a%d' % int(turn_angle))
comm.write_serial_message(turn_angle_message)
#comm.write_serial_message('s50')
cv2.putText(lines, str(int(turn_angle)), (200, 450),
cv2.FONT_HERSHEY_SIMPLEX, 4, (0, 255, 0), 4, cv2.LINE_AA)
cv2.imshow('lines', lines)
'''
keep_state = False
if processed_frames > 0 and keep_state and line_lt.detected and line_rt.detected:
line_lt, line_rt, img_fit = line_utils.get_fits_by_previous_fits(img_binary, line_lt, line_rt, verbose=False)
else:
line_lt, line_rt, img_fit = line_utils.get_fits_by_sliding_windows(img_binary, line_lt, line_rt, n_windows=9, verbose=False)
offset_meter = compute_offset_from_center(line_lt, line_rt, frame_width=frame.shape[1])
'''
#print(offset_meter)
#Minv = np.zeros(shape=(3, 3))
# draw the surface enclosed by lane lines back onto the original frame
#blend_on_road = line_utils.draw_back_onto_the_road(img, Minv, line_lt, line_rt, keep_state)
#cv2.imshow('asfdfd', blend_on_road)
# stitch on the top of final output images from different steps of the pipeline
#blend_output = prepare_out_blend_frame(blend_on_road, img_binary, warped, img_fit, line_lt, line_rt, offset_meter)
#processed_frames += 1
#cv2.imshow('mask', mask)
#cv2.imshow('res', res)
cv2.imshow('frame', frame)
cv2.imshow('warped', warped)
if cv2.waitKey(1) & 0xFF == ord('q'):
break
cap.release()
cv2.destroyAllWindows()
