def run_video():
print("Run Video")
from moviepy.editor import VideoFileClip
file = "videos/challenge_video"
clip = VideoFileClip("./" + file + ".mp4")
output_video = "./" + file + "_processed.mp4"
data_dir = './data'
num_classes = 2
global g_session
global g_logits
global g_keep_prob
global g_input_image
with tf.Session() as g_session:
vgg_path = os.path.join(data_dir, 'vgg')
correct_label = tf.placeholder(tf.int32, [None, None, None, num_classes], name='correct_label')
learning_rate = tf.placeholder(tf.float32, name='learning_rate')
g_input_image, g_keep_prob, layer3_out, layer4_out, layer7_out = load_vgg(g_session, vgg_path)
layer_output = layers(layer3_out, layer4_out, layer7_out, num_classes)
g_logits, train_op, cross_entropy_loss = optimize(layer_output, correct_label, learning_rate, num_classes)
print("Restoring model...")
saver = tf.train.Saver()
saver.restore(g_session, "./model/semantic_segmentation_model.ckpt")
print("Model restored.")
output_clip = clip.fl_image(process_video_image)
# output_clip = clip.subclip(0, 1).fl_image(process_video_image)
output_clip.write_videofile(output_video, audio=False)
# -*- coding: utf-8 -*-
# use pipeline to precess project_video.mp4
from lane.pipeline import pipeline
from moviepy.editor import VideoFileClip
white_output = "../output_video/project_video.mp4"
clip1 = VideoFileClip("../project_video.mp4")
white_clip = clip1.fl_image(pipeline)
white_clip.write_videofile(white_output, audio=False)
def process_video(input_path, output_path):
#clip = VideoFileClip(input_path).subclip(23,28)
clip = VideoFileClip(input_path)
result = clip.fl_image(process_image)
result.write_videofile(output_path)
carslist = Sync(captured,carslist)
# Draw bounding boxes on a copy of the image
draw_img = draw_labeled_bboxes(np.copy(image), carslist)
return draw_img
def draw_labeled_bboxes(img, carlist):
# Iterate through all detected cars
for car_number in range(len(carlist)):
if carlist[car_number].detected:
bbox = carlist[car_number].Box()
# Draw the box on the image
cv2.rectangle(img, bbox[0], bbox[1], (38, 133, 197), 6)
# Return the image
return img
def seriesHeatmap(refimg,nframe):
heatmap = np.zeros_like(refimg[:,:,0])
nframe = min(5,len(heatmaps))
for i in np.arange(1,nframe):
heatmap += heatmaps[-i]
return heatmap
video_output = 'tracked2.mp4'
Input_video = 'project_video.mp4'
clip1 = VideoFileClip(Input_video)
video_clip = clip1.fl_image(process_image) #NOTE: this function expects color images!!
video_clip.write_videofile(video_output, audio=False)
from process_image import ProcessImage
import lane_line_finding
from moviepy.editor import VideoFileClip
if __name__ == "__main__":
cal = lane_line_finding.Calibration.create_calibration()
pt = lane_line_finding.PerspectiveTransform.create_perspective_transform(
0, 0, 0)
video_name = "project_video.mp4"
# video_name = "00_crop_project_video.mp4"
pi = ProcessImage(pt, cal, smooth_window=5)
project_output = '170527_0.5.1-project-output_{}'.format(video_name)
clip = VideoFileClip("../" + video_name)
project_clip = clip.fl_image(pi.run)
project_clip.write_videofile(project_output, audio=False)
print("Output video: {}".format(project_output))
print('')
print('Retaining with best hyper-parameters')
scaler = StandardScaler().fit(X_features)
X_features = scaler.transform(X_features)
svc = LinearSVC(C=best_c, penalty=best_penalty, loss=best_loss).fit(X_features, y_features)
vehicle_detector = vehicle.VehicleDetector(color_space=color_space,
orient=orient,
pix_per_cell=pix_per_cell,
cell_per_block=cell_per_block,
hog_channel=hog_channel,
spatial_size=spatial_size,
hist_bins=hist_bins,
spatial_feat=spatial_feat,
hist_feat=hist_feat,
hog_feat=hog_feat,
y_start_stop=y_start_stop,
x_start_stop=x_start_stop,
xy_window=xy_window,
xy_overlap=xy_overlap,
heat_threshold = 15,
scaler=scaler,
classifier=svc)
output_file = './processed_project_video.mp4'
input_file = './project_video.mp4'
clip = VideoFileClip(input_file)
out_clip = clip.fl_image(vehicle_detector.detect)
out_clip.write_videofile(output_file, audio=False)
# adapt to movie based on PD_2016.py
import cv2
import numpy as np
from moviepy.editor import VideoFileClip
def draw_detections(img, rects, thickness = 1):
for x, y, w, h in rects:
pad_w, pad_h = int(0.15*w), int(0.05*h)
cv2.rectangle(img, (x+pad_w, y+pad_h), (x+w-pad_w, y+h-pad_h), (0, 255, 0), thickness)
def process_image(img):
# initialize pedestrian detector
hog = cv2.HOGDescriptor() #derive HOG features
hog.setSVMDetector(cv2.HOGDescriptor_getDefaultPeopleDetector()) #setSVMDetector
# start pedestrian detection
found, w = hog.detectMultiScale(img, winStride = (8,8), padding = (8, 8), scale = 1.15, finalThreshold = 1)
draw_detections(img, found) # draw rectangles
result = img
return result
test_output = 'test_output.mp4'
clip1 = VideoFileClip('../Dataset/YouTube/【ドラレコ】横浜市営バスの恐怖2.mp4')
test_clip = clip1.fl_image(process_image)
test_clip.write_videofile(test_output, audio=False)
def start(file_name):
white_output = 'test_videos_output/' + file_name
clip1 = VideoFileClip('test_videos/' + file_name)
white_clip = clip1.fl_image(process_image)
white_clip.write_videofile(white_output, audio=False)
right_marked_image = weighted_img(right_color_fit_line, left_marked_image)
# plt.imshow(right_marked_image)
# plt.show()
return right_marked_image
def process_image_file(img_file):
# reading in an image
image = mpimg.imread(img_file)
return process_image(image)
import os
# image_files = os.listdir("test_images/")
# for file in image_files:
# ret_image = process_image_file("test_images/" + file)
# cv2.imwrite("marked_" + file, ret_image)
# Import everything needed to edit/save/watch video clips
from moviepy.editor import VideoFileClip
from IPython.display import HTML
# white_output = 'white.mp4'
# clip1 = VideoFileClip("solidWhiteRight.mp4").subclip(t_start=0)
# #clip1 = VideoFileClip("solidWhiteRight.mp4").subclip(t_start=8.32)
# white_clip = clip1.fl_image(process_image) #NOTE: this function expects color images!!
# white_clip.write_videofile(white_output, audio=False)
yellow_output = 'yellow.mp4'
clip2 = VideoFileClip('solidYellowLeft.mp4')
yellow_clip = clip2.fl_image(process_image)
yellow_clip.write_videofile(yellow_output, audio=False)
(0.95 * imshape[1], imshape[0])
]], # bottomright
dtype=np.int32) # bottomright
masked_edges = region_of_interest(edges, vertices)
# Get lines
line_image = hough_lines(masked_edges,
rho=2,
theta=1 * np.pi / 180,
threshold=30,
min_line_len=15,
max_line_gap=5)
# Combine lines with original image
result = cv2.addWeighted(image, 0.9, line_image, 1, 0)
return result
name = 'solidWhiteRight.mp4'
# name = 'solidYellowLeft.mp4'
# name = 'challenge.mp4'
output = 'test_videos_output/' + name
# clip1 = VideoFileClip("test_videos/solidWhiteRight.mp4").subclip(0, 2)
clip = VideoFileClip('test_videos/' + name)
white_clip = clip.fl_image(
process_image) #NOTE: this function expects color images!!
white_clip.write_videofile(output, audio=False)
print('Finished')
# <codecell>
from moviepy.editor import VideoFileClip
from IPython.display import HTML
#output = 'output_movies/solidWhiteRight_out.mp4'
#clip1 = VideoFileClip("test_movies/solidWhiteRight.mp4")
#output = 'output_movies/solidYellowLeft_out.mp4'
#clip1 = VideoFileClip("test_movies/solidYellowLeft.mp4")
output = 'output_movies/challenge_out.mp4'
clip1 = VideoFileClip("../datas/test_videos/challenge.mp4")
reset_params_for_averaging()
clip_out = clip1.fl_image(
draw_lanes) #NOTE: this function expects color images!!
#from IPython import get_ipython
#get_ipython().magic('time clip_out.write_videofile(output, audio=False)')
####
#white_output = 'test_videos_output/solidWhiteRight.mp4'
## To speed up the testing process you may want to try your pipeline on a shorter subclip of the video
## To do so add .subclip(start_second,end_second) to the end of the line below
## Where start_second and end_second are integer values representing the start and end of the subclip
## You may also uncomment the following line for a subclip of the first 5 seconds
##clip1 = VideoFileClip("test_videos/solidWhiteRight.mp4").subclip(0,5)
#clip1 = VideoFileClip("test_videos/solidWhiteRight.mp4")
#white_clip = clip1.fl_image(process_image) #NOTE: this function expects color images!!
clip_out.write_videofile(output, audio=False)
####
HTML("""
<video width="960" height="540" controls>
#plt.savefig('./output_images/detected_lane.png')
# # 3. Pipeline for video
#
# The processed project video can be found here.
#
# The result on project_video.mp4 is shown below. The algorithm works did not work on the two challenge videos. I didn't go further to modify the code to work on these challenges. As mentioned above, I am not really convinced by the material in this project, so even it succeeds on the challenge videos, I have no confidence at all that it will work on new scenarios.
# In[39]:
output_dir= './output_images/'
clip_input_file = 'project_video.mp4'
clip_output_file = output_dir +'sample_' + clip_input_file
clip = VideoFileClip(clip_input_file).subclip(30, 40)
clip_output = clip.fl_image(process_image)
get_ipython().magic('time clip_output.write_videofile(clip_output_file, audio=False)')
# In[40]:
output_dir= './output_images/'
clip_input_file = 'project_video.mp4'
clip_output_file = output_dir +'processed_' + clip_input_file
clip = VideoFileClip(clip_input_file)
clip_output = clip.fl_image(process_image)
get_ipython().magic('time clip_output.write_videofile(clip_output_file, audio=False)')
# In[2]:
'theta':1,
'threshold':40,
'min_line_length':3,
'max_line_gap':1}
# Process images in the "test_images" directory.
for path in glob.glob('test_images/solid*.jpg'):
fname = path.split("/")[1]
image = mpimg.imread(path)
processed_image = process_image1(image)
mpimg.imsave("test_images/processed_%s" % fname, processed_image)
# Process first test video.
white_output = 'white.mp4'
clip1 = VideoFileClip("solidWhiteRight.mp4")
white_clip = clip1.fl_image(process_image2)
white_clip.write_videofile(white_output, audio=False)
# Process second test video.
yellow_output = 'yellow.mp4'
clip1 = VideoFileClip("solidYellowLeft.mp4")
yellow_clip = clip1.fl_image(process_image2)
yellow_clip.write_videofile(yellow_output, audio=False)
# Parameters for part 2 (challenge)
theta = {'horizon':0.61,
'hood':0.07,
'trapezoid_top_factor':0.10,
'trapezoid_bottom_factor':0.90,
'angle_cutoff':0.75,
##---------------------------------------------------------------------------------------
from moviepy.editor import VideoFileClip
def process_video(img):
out_img = detect_cars_with_heat(img, svc, X_scaler, orient, pix_per_cell,
cell_per_block, (spatial,spatial), histbin, heatmap_threshold)
return out_img
white_output = '/home/haoyang/CarND-Vehicle-Detection/output_project_video.mp4'
clip1 = VideoFileClip('/home/haoyang/CarND-Vehicle-Detection/project_video.mp4')
white_clip = clip1.fl_image(process_video)
white_clip.write_videofile(white_output, audio=False)
def annotate_video(input_file, output_file):
""" Given input_file video, save annotated video to output_file """
video = VideoFileClip(input_file)
annotated_video = video.fl_image(annotate_image_array)
annotated_video.write_videofile(output_file, audio=False)
# Set up sliding window parameters for lane line detection win_width = 25 win_height = 80 margin = 50 smooth_factor = 15 # Make a finder instance from centroid_finder class finder = centroid_finder(win_width, win_height, margin, smooth_factor=smooth_factor) # Set up input and output video input_video = "project_video.mp4" output_video = "project_video_output.mp4" # Make the lane-on video clip = VideoFileClip(input_video) video_clip = clip.fl_image(find_lane) video_clip.write_videofile(output_video, audio=False)
# Add heat to each box in box list
heat = add_heat(heat,hot_windows)
# Apply threshold to help remove false positives
heat = apply_threshold(heat,2)
# Visualize the heatmap when displaying
heatmap = np.clip(heat, 0, 255)
# Find final boxes from heatmap using label function
labels = label(heatmap)
draw_img = draw_labeled_bboxes(np.copy(draw_image), labels)
return draw_img
from moviepy.editor import VideoFileClip
from IPython.display import HTML
challenge_output = 'project_video_mod.mp4'
clip1 = VideoFileClip("project_video.mp4")
white_clip = clip1.fl_image(process_image)
white_clip.write_videofile(challenge_output, audio=False)
def main(argv=None): # pylint: disable=unused-argument
# JKM For now set seed
print(
"\n NOTE: Setting seed to get predictable results during development phase"
)
np.random.seed(SEED)
# Run everything inside of main
###
### This section for exploring & training a classifier.
## Fitted classifier & parms used for that get pickled.
###
if EXPLORE:
print(
"Starting - Step 1 - Explore & Find Best Feature Extraction Parms")
# Want to override so as to reduce parms later
if OVERRIDE == True:
best_parms = BEST_PARMS
print(" \n Overridden - using these parms: ", best_parms)
else:
# This explores best parms
best_parms = explore_feature_extraction_parms(cars,
notcars,
seed=SEED)
print("\n Best parms found: ", best_parms)
car_dict = run_classifier_and_pickle(cars,
notcars,
best_parms,
seed=SEED)
# Run the classifier again with the selected parms & pickle the results
else:
print("Starting - Step 1 - Get Pickled Data")
car_dict = get_pickle_data()
# Get the stored parms & classifier
svc = car_dict['svc']
X_scaler = car_dict['scaler']
colorspace = car_dict['colorspace']
hog_channel = car_dict['hog_channel']
spatial_size = car_dict['spatial_size']
hist_bins = car_dict['hist_bins']
orient = car_dict['orient']
pix_per_cell = car_dict['pix_per_cell']
cell_per_block = car_dict['cell_per_block']
spatial_feat = True
hist_feat = True
hog_feat = True
print("\n Step_1__Program paused. Press Ctrl-D to continue.\n")
code.interact(local=dict(globals(), **locals()))
print(" ... continuing\n ")
###
### This section for finding cars in an image using sliding window.
###
print(
"\n Step 2 : Explore create and search windows for cars using sliding window algorithm. \n "
)
# For now use a sample image
# IMAGE 1
# -------
image = mpimg.imread('../examples/bbox-example-image.jpg')
draw_image = np.copy(image)
# jkm - move this up to other imports
from sliding_window import get_windows, search_windows, slide_window
# w_list = get_windows(image)
# flat_list_of_windows = [item for sublist in w_list for item in sublist] - already flattened
# jkm - just work with one set of windows at this time
#Y_START_P = 0.6 # Eliminate 60% of the search space in the y direction
#y_start = int( Y_START_P * image.shape[0])
#y_start_stop = [y_start, None]
#w96_list = slide_window(image, x_start_stop=[None, None], y_start_stop=y_start_stop,
# xy_window=(96, 96), xy_overlap=(0.5, 0.5))
#w192_list = slide_window(image, x_start_stop=[None, None], y_start_stop=y_start_stop,
# xy_window=(192, 192), xy_overlap=(0.5, 0.5))
# Debugging code - Large & medium windows - HARD CODE
w_large = slide_window(image,
x_start_stop=[None, None],
y_start_stop=[500, None],
xy_window=(250, 200),
xy_overlap=(0.5, 0.5))
w_med = slide_window(image,
x_start_stop=[100, 1200],
y_start_stop=[490, 600],
xy_window=(110, 80),
xy_overlap=(0.5, 0.5))
# combine lists
w_all = w_large + w_med
# Find the set of windows that match
hot_windows = search_windows(image,
w_all,
svc,
X_scaler,
color_space=colorspace,
spatial_size=spatial_size,
hist_bins=hist_bins,
orient=orient,
pix_per_cell=pix_per_cell,
cell_per_block=cell_per_block,
hog_channel=hog_channel,
spatial_feat=spatial_feat,
hist_feat=hist_feat,
hog_feat=hog_feat)
print("\n Windows Found: ", len(hot_windows))
# temp - for testing - draw the hot windows
window_img = draw_boxes(draw_image,
hot_windows,
color=(0, 0, 255),
thick=6)
plt.imshow(window_img)
# IMAGE 2
# --------
# try with another image. the one they use for hog sub-sampling
# Need a whooe new set of windows to search
image2 = mpimg.imread('../test_images/test4.jpg')
plt.imshow(image2) # look at it first
#
draw_image2 = np.copy(image2)
# Get new set of windows
w_large2 = slide_window(image2,
x_start_stop=[None, None],
y_start_stop=[350, None],
xy_window=(250, 200),
xy_overlap=(0.5, 0.5))
w_med2 = slide_window(image2,
x_start_stop=[200, None],
y_start_stop=[350, 550],
xy_window=(110, 80),
xy_overlap=(0.5, 0.5))
w_all2 = w_large2 + w_med2
#
image2_bb = draw_boxes(draw_image2, w_all2, color=(0, 0, 255), thick=6)
plt.imshow(image2_bb)
# Find the set of windows that match
# - this doesn't find any
hot_windows2 = search_windows(image2,
w_all2,
svc,
X_scaler,
color_space=colorspace,
spatial_size=spatial_size,
hist_bins=hist_bins,
orient=orient,
pix_per_cell=pix_per_cell,
cell_per_block=cell_per_block,
hog_channel=hog_channel,
spatial_feat=spatial_feat,
hist_feat=hist_feat,
hog_feat=hog_feat)
print("\n Windows Found: ", len(hot_windows2))
# temp - for testing - draw the hot windows
window_img2 = draw_boxes(draw_image2,
hot_windows2,
color=(0, 0, 255),
thick=6)
plt.imshow(window_img2)
print("\n Step_2__Program paused. Press Ctrl-D to continue.\n")
code.interact(local=dict(globals(), **locals()))
print(" ... continuing\n ")
###
### This section for finding cars in an image using hog subsampling.
###
print("\n Step 3 : Exploring Use of hog_subsample to locate cars.")
# using their image for hog-subsampling
image2 = mpimg.imread('../test_images/test4.jpg')
ystart = 400
ystop = 656
scale = 1.5
from hog_subsample import find_cars
# use the same image as in the lesson
# from feature_explore import convert_color, bin_spatial, color_hist, get_hog_features
out_img2, match_l, conf_l, search_l = find_cars(
image2, ystart, ystop, scale, svc, X_scaler, orient, pix_per_cell,
cell_per_block, spatial_size, hist_bins, colorspace)
plt.imshow(out_img2)
# Draw the same thing again but with the box list
match_img2 = draw_boxes(image2, match_l)
plt.imshow(match_img2)
# Draw all the search areas
search_img2 = draw_boxes(image2, search_l)
plt.imshow(search_img2)
# Need different start stops for this one - these ones don't work that well
out_img1, match_l1, conf_l1, search_l1 = find_cars(
image, ystart, ystop, scale, svc, X_scaler, orient, pix_per_cell,
cell_per_block, spatial_size, hist_bins, colorspace)
plt.imshow(out_img1)
match_img1 = draw_boxes(image, match_l1)
plt.imshow(match_img1)
# Draw all the search areas
search_img1 = draw_boxes(image, search_l1)
plt.imshow(search_img1)
# try a loop to vary the search area - JKM - Is this too many?
# TODO: later try to see which of these yields the most hits
find_cars_search_area = [
[380, 660, 2.5], # 0
[380, 656, 2.0], # 1
[380, 656, 1.5], # 2
[380, 550, 1.75], # 3
[380, 530, 1.25]
] # 4
# Image2
out_images2_list = []
match_list, conf_list, search_list = [], [], []
for ystart, ystop, scale in find_cars_search_area:
if JKM_DEBUG: print(" \n Search for cars: ", ystart, ystop, scale)
out2, match_l, conf_l, search_l = find_cars(
image2, ystart, ystop, scale, svc, X_scaler, orient, pix_per_cell,
cell_per_block, spatial_size, hist_bins, colorspace)
out_images2_list.append(out2)
match_list = match_list + match_l
conf_list = conf_list + conf_l
search_list = search_list + search_l
# Draw the same thing again but with the box list
match_list_img2 = draw_boxes(image2, match_list)
plt.imshow(match_list_img2)
# Draw all the search areas
search_list_img2 = draw_boxes(image2, search_list)
plt.imshow(search_list_img2)
# use this to print out the images
#plt.imshow(out_images2[0])
# Image1 aka Image
# Image
out_images1_list = []
match_list_1, conf_list_1, search_list_1 = [], [], []
out_images1 = []
for ystart, ystop, scale in find_cars_search_area:
print(" \n Search for cars: ", ystart, ystop, scale)
out1, match_l, conf_l, search_l = find_cars(
image, ystart, ystop, scale, svc, X_scaler, orient, pix_per_cell,
cell_per_block, spatial_size, hist_bins, colorspace)
out_images1.append(out1)
#
match_list_1 = match_list_1 + match_l
conf_list_1 = conf_list_1 + conf_l
search_list_1 = search_list_1 + search_l
#plt.imshow(out_images1[0])
# Draw the same thing again but with the box list
match_list_img1 = draw_boxes(image, match_list_1)
plt.imshow(match_list_img1)
# Draw all the search areas
search_list_img1 = draw_boxes(image, search_list_1)
plt.imshow(search_list_img1)
print("\n Step_3__Program paused. Press Ctrl-D to continue.\n")
code.interact(local=dict(globals(), **locals()))
print(" ... continuing\n ")
###
### This section for exploring heat maps.
###
print(
"\n Step 4 : Exploring Use of heat maps to remove duplicates & false positives."
)
# Working with the same image - image2 from before
#
box_list = match_list
heat = np.zeros_like(image2[:, :, 0]).astype(np.float)
# Add heat to each box in box list
heat = add_heat(heat, box_list)
# Apply threshold to help remove false positives
FP_THRESH = 2 # 1
heat = apply_threshold(heat, FP_THRESH)
# Visualize the heatmap when displaying
heatmap = np.clip(heat, 0, 255)
# Find final boxes from heatmap using label function
labels = label(heatmap)
draw_img = draw_labeled_bboxes(np.copy(image2), labels)
fig = plt.figure()
plt.subplot(121)
plt.imshow(draw_img)
plt.title('Car Positions')
plt.subplot(122)
plt.imshow(heatmap, cmap='hot')
plt.title('Heat Map')
fig.tight_layout()
# 2nd image aka bbox image
# Try the other image aka image
# NOTE - issue here is that these are clumped together
box_list = match_list_1
heat = np.zeros_like(image[:, :, 0]).astype(np.float)
# Add heat to each box in box list
heat = add_heat(heat, box_list)
# Apply threshold to help remove false positives
FP_THRESH = 2 # 1
heat = apply_threshold(heat, FP_THRESH)
# Visualize the heatmap when displaying
heatmap = np.clip(heat, 0, 255)
# Find final boxes from heatmap using label function
labels = label(heatmap)
# print(labels[1], 'cars found')
draw_img = draw_labeled_bboxes(np.copy(image), labels)
fig = plt.figure()
plt.subplot(121)
plt.imshow(draw_img)
plt.title('Car Positions')
plt.subplot(122)
plt.imshow(heatmap, cmap='hot')
plt.title('Heat Map')
fig.tight_layout()
print("\n Step_4__Program paused. Press Ctrl-D to continue.\n")
code.interact(local=dict(globals(), **locals()))
print(" ... continuing\n ")
###
### This section for exploring pipelines.
###
print("\n Step 5 : Exploring Pipeline")
# To test single invokation of pipeline
fn = '../test_images/test4.jpg'
results_test_output_dir = '../output_images/output_test_images/'
out_fn = results_test_output_dir + 'result_' + os.path.basename(fn)
init_carDetectPipeline()
result_image = carDetectPipeline(image,
out_fn=out_fn,
interim=True,
debug=True,
video=False)
plt.imshow(result_image)
print("\n Step_5__Program paused. Press Ctrl-D to continue.\n")
code.interact(local=dict(globals(), **locals()))
print(" ... continuing\n ")
print("\n Step 6 : Exploring Video")
# To test pipeline on video
# uncomment this to test with a short video
# output_path = '../output_videos/test_video.mp4'
# input_path = '../test_video.mp4'
# Use this to do the assignment video
output_path = '../output_videos/project_video.mp4'
input_path = '../project_video.mp4'
# Start here
init_carDetectPipeline()
input_clip = VideoFileClip(input_path) #
#input_clip = VideoFileClip(input_path).subclip(40,43) # problematic part
output_clip = input_clip.fl_image(carDetectPipeline)
output_clip.write_videofile(output_path, audio=False)
print("\n Step_6__Program paused. Press Ctrl-D to continue.\n")
code.interact(local=dict(globals(), **locals()))
print(" ... continuing\n ")
print("\n Step 7 : Exploring Combined Video")
# This is to try to merge the two results
# Note: This doesn't work as well as I would like
from advancedLaneLines import pipeline
def combinedPipeline(image):
image = pipeline(image) # advanced Lane Finding pipeline
image = carDetectPipeline(image) # car detection Pipeline
return image
# uncomment this to test with a short video
#output_path = '../output_videos/test_video.mp4'
#input_path = '../test_video.mp4'
# Use this to do the assignment video
output_path = '../output_videos/combined_project_video.mp4'
input_path = '../project_video.mp4'
# Start here
init_carDetectPipeline()
input_clip = VideoFileClip(input_path) #
#input_clip = VideoFileClip(input_path).subclip(40,43) # problematic part
output_clip = input_clip.fl_image(combinedPipeline)
output_clip.write_videofile(output_path, audio=False)
print("\n Step_7__Program paused. Press Ctrl-D to continue.\n")
code.interact(local=dict(globals(), **locals()))
print(" ... continuing\n ")
test_images = os.listdir("test_images/")
for test_image in test_images:
# load in image
initial_img = mpimg.imread('test_images/' + test_image)
img = process_image(initial_img)
# save to output
mpimg.imsave("output/" + test_image, img)
# Import everything needed to edit/save/watch video clips
from moviepy.editor import VideoFileClip
from IPython.display import HTML
making_videos = True
if making_videos:
white_output = 'white.mp4'
clip1 = VideoFileClip("solidWhiteRight.mp4")
white_clip = clip1.fl_image(process_image) #NOTE: this function expects color images!!
white_clip.write_videofile(white_output, audio=False)
yellow_output = 'yellow.mp4'
clip2 = VideoFileClip('solidYellowLeft.mp4')
yellow_clip = clip2.fl_image(process_image)
yellow_clip.write_videofile(yellow_output, audio=False)
challenge_output = 'extra.mp4'
clip2 = VideoFileClip('challenge.mp4')
challenge_clip = clip2.fl_image(process_image)
challenge_clip.write_videofile(challenge_output, audio=False)
img_BGR0 = cv2.imread(fname)
img_RGB0 = cv2.cvtColor(img_BGR0, cv2.COLOR_BGR2RGB)
img_out0 = lane_detection_pipeline(img_BGR0)
if (ind_plot == 1): #(idx == 0):
fig, axes = plt.subplots(1, 2) # , figsize=(20, 10)
axes[0].imshow(img_RGB0)
axes[0].set_title('Original Image : {}'.format(ii+1)); axes[0].set_axis_off()
axes[1].imshow(img_out0, cmap='gray'); axes[1].set_axis_off()
plt.show()
# ------------------------------------------------------------------------------------------------------------
# Create Video Output ----------------------------------------------------------------------------------------
# ------------------------------------------------------------------------------------------------------------
from moviepy.editor import VideoFileClip
white_output = 'project_video_output.mp4'
## To speed up the testing process you may want to try your pipeline on a shorter subclip of the video
## To do so add .subclip(start_second,end_second) to the end of the line below
## Where start_second and end_second are integer values representing the start and end of the subclip
## You may also uncomment the following line for a subclip of the first 5 seconds
##clip1 = VideoFileClip("test_videos/solidWhiteRight.mp4").subclip(0,5)
clip1 = VideoFileClip("project_video.mp4")
white_clip = clip1.fl_image(lane_detection_pipeline) #NOTE: this function expects color images!!
%time white_clip.write_videofile(white_output, audio=False)
tf.import_graph_def(od_graph_def, name='')
return graph
detection_graph = load_graph(SSD_V4)
image_tensor = detection_graph.get_tensor_by_name('image_tensor:0')
detection_boxes = detection_graph.get_tensor_by_name('detection_boxes:0')
detection_scores = detection_graph.get_tensor_by_name('detection_scores:0')
detection_classes = detection_graph.get_tensor_by_name('detection_classes:0')
clip = VideoFileClip('raw_video.avi')
def pipeline(img):
draw_img = Image.fromarray(img)
boxes, scores, classes = sess.run([detection_boxes, detection_scores, detection_classes], feed_dict={image_tensor: np.expand_dims(img, 0)})
boxes = np.squeeze(boxes)
scores = np.squeeze(scores)
classes = np.squeeze(classes)
confidence_cutoff = 0.8
boxes, scores, classes = filter_boxes(confidence_cutoff, boxes, scores, classes)
width, height = draw_img.size
box_coords = to_image_coords(boxes, height, width)
draw_boxes(draw_img, box_coords, classes)
return np.array(draw_img)
with tf.Session(graph=detection_graph) as sess:
image_tensor = sess.graph.get_tensor_by_name('image_tensor:0')
detection_boxes = sess.graph.get_tensor_by_name('detection_boxes:0')
detection_scores = sess.graph.get_tensor_by_name('detection_scores:0')
detection_classes = sess.graph.get_tensor_by_name('detection_classes:0')
new_clip = clip.fl_image(pipeline)
new_clip.write_videofile('SSD_V4_result.mp4')
with open("./svc_pickle/svc_pickle.p", 'rb') as f:
svc_dict = pickle.load(f)
svc = svc_dict['svc']
X_scaler = svc_dict['scaler']
roi = ((None, None), (None, None))
scale = (1, 1.25)
accumulate = 10
thresh = 30
detector = car_detector(svc=svc,
X_scaler=X_scaler,
orient=orient,
pix_per_cell=pix_per_cell,
cell_per_block=cell_per_block,
spatial_size=spatial_size,
hist_bins=hist_bins,
roi=roi,
scale=scale,
accumulate=accumulate,
thresh=thresh)
test_input = './test_videos/project_video.mp4'
test_output = './test_videos/project_video_result.mp4'
t = time.time()
clip = VideoFileClip(test_input) #.subclip(4,10)
test_clip = clip.fl_image(detector.process_image)
test_clip.write_videofile(test_output, audio=False)
print('Finish detection. Using time: ', round(time.time() - t, 2))
image = mpimg.imread(filename)
#(1) Yolo pipeline
yolo_result = pipeline_yolo(image)
plt.figure()
plt.imshow(yolo_result)
plt.title('yolo pipeline', fontsize=30)
#(2) SVM pipeline
draw_img = pipeline_svm(image)
fig = plt.figure()
plt.imshow(draw_img)
plt.title('svm pipeline', fontsize=30)
plt.show()
elif demo == 2:
# YOLO Pipeline
video_output = 'examples/project_YOLO.mp4'
clip1 = VideoFileClip("examples/project_video.mp4").subclip(30,32)
clip = clip1.fl_image(pipeline_yolo)
clip.write_videofile(video_output, audio=False)
else:
# SVM pipeline
video_output = 'examples/project_svm.mp4'
clip1 = VideoFileClip("examples/project_video.mp4").subclip(30,32)
clip = clip1.fl_image(pipeline_svm)
clip.write_videofile(video_output, audio=False)
import cv2
from PIL import Image
from image_thresholding import *
from lane_detection import *
import copy
from Pipeline import *
import sys
from moviepy.editor import VideoFileClip
if __name__ == '__main__':
arg_count = len(sys.argv)
#print(arg_count)
if arg_count != 3:
print(
'Run this script as python lane_finder.py your_input_file.mp4 your_output_file.mp4'
)
else:
try:
pipeline = Pipeline()
inputfile = str(sys.argv[1])
outputfile = str(sys.argv[2])
clip1 = VideoFileClip(inputfile)
white_clip = clip1.fl_image(
pipeline.process_image
) #NOTE: this function expects color images!!
white_clip.write_videofile(outputfile, audio=False)
except Exception as e:
print(str(e))
result = cv2.addWeighted(img, 1.0, road, 0.3, 0)
# Calculate curvature radius of both lines and average them
left_rad = lf.left_lines.current_curvature_radius()
right_rad = lf.right_lines.current_curvature_radius()
mean_rad = (left_rad + right_rad) / 2
# Calculate center of the road to the center of the image
left_x = lf.left_lines.best_x() # left line bottom pixel x position
right_x = lf.right_lines.best_x() # right line bottom pixel x position
offset_x = (1280 / 2) - (left_x + right_x) / 2
offset_direction = "right" if offset_x > 0 else "left"
offset_x_meter = offset_x * 3.7 / 700
# write radius and offset onto image
result = cv2.putText(result, 'Radius of Curvature = %.0f' % (mean_rad),
(50, 70), cv2.FONT_HERSHEY_SIMPLEX, 1.5,
(255, 255, 255), 3)
result = cv2.putText(
result, 'Vehicle is %.2f m %s of center' %
(abs(offset_x_meter), offset_direction), (50, 140),
cv2.FONT_HERSHEY_SIMPLEX, 1.5, (255, 255, 255), 3)
return result
if __name__ == "__main__":
camera_calibrator.calc_dist_matrix()
clip1 = VideoFileClip(INPUT_FILE_NAME).subclip(0, 5)
white_clip = clip1.fl_image(lane_detection)
white_clip.write_videofile(OUTPUT_FILE_NAME, audio=False)
def find_vehicles_in_video(video):
output = "tracked2_" + video
input_clip = VideoFileClip(video)
clip = input_clip.fl_image(find_vehicles_in_frame)
#clip = input_clip.fl_image(save_image)
clip.write_videofile(output, audio=False)
def video_pipeline(mtx, dist, input_video_filename, output_video_filename):
# write the video out to a file
clip1 = VideoFileClip(input_video_filename)
video_clip = clip1.fl_image(ImagePipeline(mtx, dist))
video_clip.write_videofile(output_video_filename, audio=False)
labels = label( thresholded_total_heatmap )
# Draw the bounding box of each labelled island region
# on the final output image
tracking_image = draw_labeled_bboxes( draw_image, labels )
# fig = plt.figure()
# plt.subplot(221)
# plt.imshow(heatmap, cmap='hot')
# plt.subplot(222)
# plt.imshow( heat_running.get_running_heatmap(), cmap='hot' )
# plt.subplot(223)
# plt.imshow(labels[0], cmap='gray')
# plt.subplot(224)
# plt.imshow(tracking_image)
# fig.tight_layout()
# plt.show()
return tracking_image
# clip.iter_frames is a Python generator that loops through the frames.
# for image in clip.iter_frames():
# single_image_pipeline( image )
# Open the input video
clip = VideoFileClip('project_video.mp4')
# Process the input video to create the output clip
output_clip = clip.fl_image( pipeline )
# Write the output clip
output_clip.write_videofile( 'project_output.mp4', audio=False)
def process_video(input_file="./project_video.mp4",
output_file="./project_video_output.mp4"):
clip = VideoFileClip(input_file)
video_clip = clip.fl_image(pipeline)
video_clip.write_videofile(output_file, audio=False)
last_rightx = 0
# n_samples is the number of samples used when averaging the fit coefficients
n_samples = 10
coefficient_avgs = np.zeros((3,2,n_samples))
# Process the test images
test_image_files = glob.glob('test_images/*.jpg')
images = []
for test_image_file in test_image_files:
images.append(mpimg.imread(test_image_file))
# for image in images:
# process_image(image)
# process_image(images[0])
project_video_annotated = 'annotated_videos/project_video_annotated.mp4'
challenge_video_annotated = 'annotated_videos/challenge_video_annotated.mp4'
project_video = VideoFileClip('project_video.mp4')
clip = project_video.fl_image(process_image)
clip.write_videofile(project_video_annotated, audio=False)
# challenge_video = VideoFileClip('challenge_video.mp4')
# clip = challenge_video.fl_image(process_image)
# clip.write_videofile(challenge_video_annotated, audio=False)
return lines_edges
def read_image_for_marking(img_filepath):
# read in the image
image = mpimg.imread(img_filepath)
print('Reading image :', img_filepath, '\nDimensions:', image.shape)
marked_lanes = mark_lanes(image)
# show the image to plotter and then save it to a file
plt.imshow(marked_lanes)
plt.savefig(img_filepath[:-4] + '_output.png')
if __name__ == "__main__":
# set up parser
parser = argparse.ArgumentParser()
parser.add_argument("-f", "--file", help="filepath for image to mark", default='test_images/solidWhiteRight.jpg')
parser.add_argument("-v", "--video", help="filepath for video to mark")
args = parser.parse_args()
if args.video:
clip = VideoFileClip(args.video)
clip = clip.fl_image(mark_lanes)
clip.write_videofile('output_' + args.video, audio=False)
else:
# if nothing passed running algorithm on image
read_image_for_marking(args.file)
f, (ax1, ax2) = plt.subplots(1, 2, figsize=(20, 7))
f.tight_layout()
ax1.imshow(heatmap, cmap="hot")
ax1.set_title('Heatmap', fontsize=30)
ax2.imshow(window_img)
ax2.set_title('Thresholded heatmap', fontsize=30)
extent = f.get_window_extent().transformed(f.dpi_scale_trans.inverted())
plt.savefig('output_images/heat.jpg',
bbox_inches=extent.expanded(1, 1.2),
dpi=50)
## Show heatmaps on video example ##
from moviepy.editor import VideoFileClip
processObj = ProcessClass(20,
xstart,
xstop,
ystart,
ystop,
scales,
colorspace,
orient,
pix_per_cell,
cell_per_block,
hog_channel,
feat_scaler,
svc,
debug=True)
clip1 = VideoFileClip("project_video.mp4").subclip(16, 19) #.subclip(39,41)
white_clip = clip1.fl_image(processObj.process_frame)
white_clip.write_videofile('test.mp4', audio=False)
def run():
num_classes = 2
image_shape = (160, 576)
data_dir = './data'
runs_dir = './runs'
tests.test_for_kitti_dataset(data_dir)
# Download pretrained vgg model
helper.maybe_download_pretrained_vgg(data_dir)
export_dir = './inference'
has_pretrained_graph = False
if os.path.exists(export_dir):
has_pretrained_graph = True
else:
builder = tf.saved_model.builder.SavedModelBuilder(export_dir)
# OPTIONAL: Train and Inference on the cityscapes dataset instead of the Kitti dataset.
# You'll need a GPU with at least 10 teraFLOPS to train on.
# https://www.cityscapes-dataset.com/
clip = VideoFileClip('driving.mp4')
def pipeline(img):
img=scipy.misc.imresize(img, image_shape)
im_softmax = sess.run([output_mask],
feed_dict={image_input: [img],keep_prob:1.0})
im_softmax = im_softmax[0][:, 1].reshape(image_shape[0], image_shape[1])
segmentation = (im_softmax > 0.5).reshape(image_shape[0], image_shape[1], 1)
mask = np.dot(segmentation, np.array([[0, 255, 0, 127]]))
mask = scipy.misc.toimage(mask, mode="RGBA")
street_im = scipy.misc.toimage(img)
street_im.paste(mask, box=None, mask=mask)
return np.array(street_im)
with tf.Session() as sess:
if has_pretrained_graph:
tf.saved_model.loader.load(sess, ['inference_fcn'], export_dir)
vgg_input_tensor_name = 'image_input:0'
vgg_keep_prob_tensor_name = 'keep_prob:0'
output_mask_tensor_name = 'output_mask:0'
graph = tf.get_default_graph()
image_input = graph.get_tensor_by_name(vgg_input_tensor_name)
keep_prob = graph.get_tensor_by_name(vgg_keep_prob_tensor_name)
output_mask = graph.get_tensor_by_name(output_mask_tensor_name)
new_clip = clip.fl_image(pipeline)
# write to file
new_clip.write_videofile('result.mp4')
else:
# Path to vgg model
vgg_path = os.path.join(data_dir, 'vgg')
# Create function to get batches
get_batches_fn = helper.gen_batch_function(os.path.join(data_dir, 'data_road/training'), image_shape)
# OPTIONAL: Augment Images for better results
# https://datascience.stackexchange.com/questions/5224/how-to-prepare-augment-images-for-neural-network
# TODO: Build NN using load_vgg, layers, and optimize function
image_input, keep_prob, layer3_out, layer4_out, layer7_out = load_vgg(sess,vgg_path)
output = layers(vgg_layer3_out=layer3_out,vgg_layer4_out=layer4_out,vgg_layer7_out=layer7_out,num_classes=num_classes)
correct_labels = tf.placeholder(tf.float32, shape=[None,None,None, num_classes])
learning_rate = tf.placeholder(tf.float32)
logits, train_op, cross_entropy_loss = optimize(output,correct_labels,learning_rate,num_classes)
train_nn(sess,20,6,get_batches_fn,train_op,cross_entropy_loss,image_input,correct_labels,keep_prob,learning_rate)
output_mask = tf.nn.softmax(logits,name='output_mask')
builder.add_meta_graph_and_variables(sess,['inference_fcn'])
# TODO: Train NN using the train_nn function
# TODO: Save inference data using helper.save_inference_samples
input_image = image_input
helper.save_inference_samples(runs_dir, data_dir, sess, image_shape, logits, keep_prob, input_image)
#print trained image IoU
helper.iou_output(sess,logits,keep_prob,input_image,data_dir,image_shape,num_classes)
# OPTIONAL: Apply the trained model to a video
if not has_pretrained_graph:
builder.save()
def main(mode=None, source=None, out=None, log=False, mine=False):
Logger.logging = log
Logger.mode = mode
model = Model(
pixels_per_cell=8,
cells_per_block=2,
orientations=9,
use_hog=True,
use_spatial=True,
use_hist=False,
hog_color_space=cv2.COLOR_RGB2YCrCb,
spatial_color_space=cv2.COLOR_RGB2HSV,
hist_color_space=None,
)
if mode == 'train':
print("Reading images.csv -- contains image paths")
images = pd.read_csv('images.csv', names=['filepath', 'class'])
X_all = images['filepath'].values
y_all = images['class'].values.astype('uint8')
print('Cars:', len(np.where(y_all == 1)[0]))
print('Not-Cars:', len(np.where(y_all == 0)[0]))
print("Load Images...")
X = []
y = []
start = time.time()
for idx, x in enumerate(X_all):
if os.path.isfile(x):
X.append(image.imread(x))
y.append(y_all[idx])
end = time.time()
print('time (load images):', end-start)
print(len(y))
model.fit(X, y)
elif mode == 'video':
Logger.source = source
pipeline = Pipeline(model=model, mode='video', mine=mine)
source_video = VideoFileClip(source)
output_video = source_video.fl_image(pipeline.process)
output_video.write_videofile(out, audio=False)
elif mode == 'test_images':
pipeline = Pipeline(model=model, mode='test_images', mine=mine)
images = glob.glob('test_images/*.jpg')
for idx, fname in enumerate(images):
pipeline.reset()
Logger.source = fname
img = image.imread(fname)
img = pipeline.process(img)
elif mode == 'visualise':
print("Reading images.csv -- contains image paths")
images = pd.read_csv('images.csv', names=['filepath', 'class'])
X_all = images['filepath'].values
y_all = images['class'].values.astype('uint8')
print('Cars:', len(np.where(y_all == 1)[0]))
print('Not-Cars:', len(np.where(y_all == 0)[0]))
print("Load Images...")
X = []
y = []
X_all, y_all = shuffle(X_all, y_all)
X_all, y_all = X_all[:5], y_all[:5]
start = time.time()
for idx, x in enumerate(X_all):
if os.path.isfile(x):
X.append(image.imread(x))
y.append(y_all[idx])
end = time.time()
print('time (load images):', end-start)
model.visualise(X, y)
heatmap_img = add_heat(heatmap_img, squares)
heatmap_img = apply_threshold(heatmap_img, 1)
labels = label(heatmap_img)
draw_img, rects = draw_labeled_bboxes(np.copy(img), labels)
return draw_img
#################PRACTICAL ##############################
test_images = glob.glob('./test_images/test*.jpg')
fig, axs = plt.subplots(3, 2, figsize=(10,10))
axs = axs.ravel()
for i, im in enumerate(test_images):
axs[i].imshow(pipeline(mpimg.imread(im)))
axs[i].axis('off')
plt.show()
############################################### 9. VIDEO APPLICATION ############################
from moviepy.editor import VideoFileClip
test_video = VideoFileClip('project_video.mp4')
piped_video = test_video.fl_image(pipeline)
piped_video.write_videofile('result_project_video.mp4')
test_video_2 = VideoFileClip('test_video.mp4')
piped_video = test_video_2.fl_image(pipeline)
piped_video.write_videofile('result_test_video.mp4')
def detector(video_input, video_output):
# parameters for loading data and images
detection_model_path = '../trained_models/detection_models/haarcascade_frontalface_default.xml'
emotion_model_path = '../trained_models/emotion_models/fer2013_mini_XCEPTION.102-0.66.hdf5'
emotion_labels = get_labels('fer2013')
gender_model_path = '../trained_models/gender_models/simple_CNN.81-0.96.hdf5'
gender_labels = get_labels('imdb')
font = cv2.FONT_HERSHEY_SIMPLEX
gender_to_cnt = {}
emotion_to_cnt = {}
age_to_cnt = {}
# hyper-parameters for bounding boxes shape
frame_window = 10
emotion_offsets = (20, 40)
gender_offsets = (10, 10)
# loading models
face_detection = load_detection_model(detection_model_path)
emotion_classifier = load_model(emotion_model_path, compile=False)
# getting input model shapes for inference
emotion_target_size = emotion_classifier.input_shape[1:3]
gender_classifier = load_model(gender_model_path, compile=False)
gender_target_size = gender_classifier.input_shape[1:3]
depth = 16
k = 8
weight_file = "weights.18-4.06.hdf5"
# load model and weights
gender_age_prediction_img_size = 64
model = WideResNet(gender_age_prediction_img_size, depth=depth, k=k)()
model.load_weights(weight_file)
cnn_face_detector = dlib.cnn_face_detection_model_v1(
'mmod_human_face_detector.dat')
def pipeline(bgr_image):
bgr_image = cv2.resize(bgr_image, (640, 360))
faces = cnn_face_detector(bgr_image, 1)
global total_faces
total_faces = total_faces + len(faces)
gray_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2GRAY)
rgb_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2RGB)
for face in faces:
x1, y1, x2, y2, w, h = face.rect.left(), face.rect.top(
), face.rect.right() + 1, face.rect.bottom() + 1, face.rect.width(
), face.rect.height()
xw1 = max(int(x1 - 0.4 * w), 0)
yw1 = max(int(y1 - 0.4 * h), 0)
xw2 = min(int(x2 + 0.4 * w), bgr_image.shape[1] - 1)
yw2 = min(int(y2 + 0.4 * h), bgr_image.shape[0] - 1)
gray_face = gray_image[yw1:yw2 + 1, xw1:xw2 + 1]
try:
gray_face = cv2.resize(gray_face, (emotion_target_size))
except:
continue
gray_face = preprocess_input(gray_face, False)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
emotion_label_arg = np.argmax(
emotion_classifier.predict(gray_face))
emotion_text = emotion_labels[emotion_label_arg]
if emotion_text not in emotion_to_cnt:
emotion_to_cnt[emotion_text] = 0
emotion_to_cnt[emotion_text] = emotion_to_cnt[emotion_text] + 1
color = (255, 255, 255)
cv2.putText(rgb_image, emotion_text,
(face.rect.left(), face.rect.top() - 5),
cv2.FONT_HERSHEY_SIMPLEX, 1, color, 1, cv2.LINE_AA)
face_list = np.empty((len(faces), gender_age_prediction_img_size,
gender_age_prediction_img_size, 3))
for i in range(0, len(faces)):
face = faces[i]
x1, y1, x2, y2, w, h = face.rect.left(), face.rect.top(
), face.rect.right() + 1, face.rect.bottom() + 1, face.rect.width(
), face.rect.height()
xw1 = max(int(x1 - 0.4 * w), 0)
yw1 = max(int(y1 - 0.4 * h), 0)
xw2 = min(int(x2 + 0.4 * w), bgr_image.shape[1] - 1)
yw2 = min(int(y2 + 0.4 * h), bgr_image.shape[0] - 1)
rgb_face = rgb_image[yw1:yw2 + 1, xw1:xw2 + 1, :]
try:
face_list[i, :, :, :] = cv2.resize(
rgb_face, (gender_age_prediction_img_size,
gender_age_prediction_img_size))
except:
continue
gender_age_prediction = model.predict(face_list)
for i in range(0, len(faces)):
face = faces[i]
predicted_genders = gender_age_prediction[0]
gender_text = "FEMALE" if predicted_genders[i][0] > 0.5 else "MALE"
ages = np.arange(0, 101).reshape(101, 1)
predicted_ages = gender_age_prediction[1].dot(ages).flatten()
age_text = str(predicted_ages[i])
if gender_text not in gender_to_cnt:
gender_to_cnt[gender_text] = 0
gender_to_cnt[gender_text] = gender_to_cnt[gender_text] + 1
if age_text not in age_to_cnt:
age_to_cnt[age_text] = 0
age_to_cnt[age_text] = age_to_cnt[age_text] + 1
gender_color = (255, 0, 0) if gender_text == "MALE" else (0, 0,
255)
cv2.rectangle(rgb_image, (face.rect.left(), face.rect.top()),
(face.rect.right(), face.rect.bottom()),
gender_color, 1)
color = (255, 255, 255)
cv2.putText(rgb_image, gender_text,
(face.rect.left(), face.rect.top() - 20),
cv2.FONT_HERSHEY_SIMPLEX, 1, gender_color, 1,
cv2.LINE_AA)
cv2.putText(rgb_image, age_text,
(face.rect.left(), face.rect.top() - 35),
cv2.FONT_HERSHEY_SIMPLEX, 1, color, 1, cv2.LINE_AA)
bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
return bgr_image
clip2 = VideoFileClip(video_input)
white_clip = clip2.fl_image(
pipeline) # NOTE: this function expects color images!!
white_clip.write_videofile(video_output, audio=False)
def annotate_video(input_file, output_file): """ Given input_file video, save annotated video to output_file """ video = VideoFileClip(input_file) annotated_video = video.fl_image(annotate_image) annotated_video.write_videofile(output_file, audio=False)
def run():
num_classes = 2
image_shape = (160, 576)
data_dir = './data'
runs_dir = './runs'
tests.test_for_kitti_dataset(data_dir)
epochs = args.epochs
batch_size = 16
correct_label = tf.placeholder(tf.float32, (None, None, None, num_classes))
learning_rate = tf.placeholder(tf.float32)
# Download pretrained vgg model
helper.maybe_download_pretrained_vgg(data_dir)
# OPTIONAL: Train and Inference on the cityscapes dataset instead of the Kitti dataset.
# You'll need a GPU with at least 10 teraFLOPS to train on.
# https://www.cityscapes-dataset.com/
with tf.Session() as sess:
# Path to vgg model
vgg_path = os.path.join(data_dir, 'vgg')
# Create function to get batches
get_batches_fn = helper.gen_batch_function(
os.path.join(data_dir, 'data_road/training'), image_shape)
# OPTIONAL: Augment Images for better results
# https://datascience.stackexchange.com/questions/5224/how-to-prepare-augment-images-for-neural-network
# Note: see helper.augment_image
# Build NN using load_vgg, layers, and optimize function
image_input, keep_prob, layer3_out, layer4_out, layer7_out = load_vgg(
sess, vgg_path)
final_layer = layers(layer3_out, layer4_out, layer7_out, num_classes)
logits, train_op, cross_entropy_loss = optimize(
final_layer, correct_label, learning_rate, num_classes)
# Train NN using the train_nn function
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
saver = tf.train.Saver()
if args.action == 'train' or args.action == 'all':
train_nn(sess, epochs, batch_size, get_batches_fn, train_op,
cross_entropy_loss, image_input, correct_label, keep_prob,
learning_rate)
save_path = saver.save(
sess,
os.path.join('saved_models',
'model_' + str(time.time()) + '.ckpt'))
print("Saving model: {}".format(save_path))
else:
saver.restore(sess, tf.train.latest_checkpoint('./saved_models'))
# Save inference data using helper.save_inference_samples
if args.action == 'images' or args.action == 'all':
helper.save_inference_samples(runs_dir, data_dir, sess,
image_shape, logits, keep_prob,
image_input)
# OPTIONAL: Apply the trained model to a video
if args.action == 'video' or args.action == 'all':
video_out_name = 'inference_video.mp4'
print('Generating {}'.format(video_out_name))
def save_inference_video(img):
image = scipy.misc.imresize(img, image_shape)
im_softmax = sess.run([tf.nn.softmax(logits)], {
keep_prob: 1.0,
image_input: [image]
})
im_softmax = im_softmax[0][:,
1].reshape(image_shape[0],
image_shape[1])
segmentation = (im_softmax > 0.5).reshape(
image_shape[0], image_shape[1], 1)
mask = np.dot(segmentation, np.array([[0, 255, 0, 127]]))
mask = scipy.misc.toimage(mask, mode="RGBA")
street_im = scipy.misc.toimage(image)
street_im.paste(mask, box=None, mask=mask)
return np.array(street_im)
clip = VideoFileClip('harder_challenge_video.mp4')
new_clip = clip.fl_image(save_inference_video)
print('Saving sample video to: {}'.format(video_out_name))
new_clip.write_videofile(video_out_name, audio=False)
sess.close()
exit(0)
combined_grad_mag_dir_hls[(combined_grad_mag_dir == 1) | (combined_sl == 1)] = 1
# display(img, combined_grad_mag_dir_hls)
warped, M = warp(combined_grad_mag_dir_hls)
# display2(combined_grad_mag_dir_hls, warped)
ploty, left_fitx, right_fitx, avg_left_fitx, avg_right_fitx = lanelines(warped)
left_curverad, right_curverad = radius_of_curvature(ploty, left_fitx, right_fitx)
avg_curverad = (left_curverad + right_curverad)/2
offset = find_offset_from_center(img, left_fitx, right_fitx)
result = project(warped, img, ploty, left_fitx, right_fitx, M, avg_curverad, offset, avg_left_fitx, avg_right_fitx)
return result
test_images_path = 'test_images/'
images = glob.glob(test_images_path+"*.jpg")
for image in images:
orig = mpimg.imread(image)
left_fit_current = None
right_fit_current = None
pipeline(orig)
white_output = 'output_images/project_video_mapped.mp4'
clip1 = VideoFileClip("project_video.mp4")
white_clip = clip1.fl_image(pipeline)
white_clip.write_videofile(white_output, audio=False)
pts = np.hstack((pts_left, pts_right))
# Draw the lane onto the warped blank image
cv2.fillPoly(color_warp, np.int_([pts]), (0, 255, 0))
# Warp the blank back to original image space using inverse perspective matrix (Minv)
newwarp = cv2.warpPerspective(
color_warp, Minv, (binary_warped.shape[1], binary_warped.shape[0]))
# Combine the result with the original image
result = cv2.addWeighted(img, 1, newwarp, 0.3, 0)
addCurvature(result, ploty, left_fit_cr, right_fit_cr)
addPosition(result, ploty, left_fit, right_fit)
if False:
addAnalysis(result, meta_data, gradient_image, binary_warped)
return result
if __name__ == "__main__":
if True:
from moviepy.editor import VideoFileClip
clip1 = VideoFileClip("project_video.mp4")
#clip1 = VideoFileClip("challenge_video.mp4")
output_video = clip1.fl_image(process_image)
output_video.write_videofile("project_video_output.mp4", audio=False)
#output_video.write_videofile("challenge_video_output.mp4", audio=False)
else:
img = cv2.imread('last_image.jpg')
pimg = process_image(img)
plt.imshow(pimg)
plt.show()
road_warped = cv2.warpPerspective(road, Minv, img_size, flags=cv2.INTER_LINEAR)
road_warped_bkg = cv2.warpPerspective(road_bkg, Minv, img_size, flags=cv2.INTER_LINEAR)
base = cv2.addWeighted(img, 1.0, road_warped_bkg, -1.0, 0.0)
result = cv2.addWeighted(base, 1.0, road_warped, .7, 0.0)
ym_per_pix = curve_centers.ym_per_pix
xm_per_pix= curve_centers.xm_per_pix
curve_fit_cr = np.polyfit(np.array(res_yvals,np.float32)*ym_per_pix, np.array(leftx, np.float32)*xm_per_pix, 2)
curverad = ((1 + (2*curve_fit_cr[0]*yvals[-1]*ym_per_pix + curve_fit_cr[1])**2)**1.5) / np.absolute(2*curve_fit_cr[0])
camera_center = (left_fitx[-1] + right_fitx[-1])/2
center_diff = (camera_center-warped.shape[1]/2)*xm_per_pix
side_pos = 'left'
if center_diff <=0:
side_pos = 'right'
cv2.putText(result, 'Radius of Curvature = '+str(round(curverad, 3))+'(m)',(50,50),cv2.FONT_HERSHEY_SIMPLEX,1,(255,255,255),2)
cv2.putText(result, 'Vehicle is '+str(abs(round(center_diff, 3)))+'m '+side_pos+' of center',(50,100),cv2.FONT_HERSHEY_SIMPLEX,1,(255,255,255),2)
return result
if __name__ == '__main__':
output_video = 'output_tracked1.mp4'
input_video = 'project_video.mp4'
clip1 = VideoFileClip(input_video)
video_clip = clip1.fl_image(process_img)
video_clip.write_videofile(output_video, audio=False)
try:
left_fit, right_fit, left_fit_cr, right_fit_cr = curv_calc_from_previous(
warped, prev_left_fit, prev_right_fit)
except:
pass
if left_fit[0] == None:
# if the current fit did not find a line, we try to recalculate it with sliding window
left_fit, right_fit, left_fit_cr, right_fit_cr = curv_calc_sliding_window(
warped)
# step5 - process the lines by comparing them to the line history
left_fit, left_fit_cr = left_line.process_line(left_fit, left_fit_cr)
right_fit, right_fit_cr = right_line.process_line(right_fit, right_fit_cr)
if left_fit[0] == None or right_fit[0] == None:
return img # if after everything, we still failed to find a line, we return the original image...
# step6 - unwarp the lines and draw them back on the original image
lines_img = add_lines_to_img(img, left_fit, right_fit, Minv)
# step7 - caculate the offset and curvature in real world units ([m])
out = measure_curvature_and_offset(lines_img, left_fit_cr, right_fit_cr)
return out
#%%
video_output = 'output_images/projectVideo.mp4'
clip1 = VideoFileClip("project_video.mp4")
project_clip = clip1.fl_image(video_frame_pipeline)
return draw_img
def process_image(img):
output = pipeline(img)
return output
# In[40]:
frame_ctx = frame()
video_output1 = 'project_video_output.mp4'
video_input1 = VideoFileClip('project_video.mp4')#.subclip(4, 7)#subclip(7, 12)#subclip(40, 42)
processed_video = video_input1.fl_image(process_image)
get_ipython().run_line_magic('time', 'processed_video.write_videofile(video_output1, audio=False)')
#
# #### Discussion
#
# ####1. Briefly discuss any problems / issues you faced in your implementation of this project. Where will your pipeline likely fail? What could you do to make it more robust?
#
# There are some false positive (which I do not consider to be false) as those cars are comming from the other direction (on the left) and I could have used the xstart to regulate the starting of the sliding windows in X direction to filterout those detection or detect the lane line as in previous project and ignore any detection outside of the lane line but that will not be practicle as I think this is important to detect anything along the x axis even if they are outside of the lane.
#
# I could have use more sliding windows to make the bounded box more stable but just wanted to make it simple for now.
#
# I could have calculate the centroid of each detected boxes per frame, measure the distance of these centroids among the boxes near the same location (with some += mergin) over few frames and can estimate the projected distance of the same box(car) on the next upcomming frame and draw it and could make the pipeline faster as was suggested in the course video. But that should also take into consideration the fact that car passing my car can slow down and in next frame it may appear closer as oppose to farther.
#
# I coud have shown the number of car detected (labels[1]) based on heatmap & before and after applying threshold in realtime to make the video more predictable about the pipeline, also could have implemented a dynamic sliding window where more scanning will be done when there is a change in the number of detection and apply even a harder threshold in this transient time to ignore the false positive etc with a firm detection.
# then save them to the test_images directory.
#imagenames = os.listdir("test_images/")
#for i in range(len(imagenames)):
# img = (mpimg.imread("test_images/"+imagenames[i])*255).astype('uint8')
# lineimg = process_image(img)
# name = imagenames[i][0:len(img)-4]
# ext = imagenames[i][len(img)-4:len(img)]
# cv2.imwrite("test_images/"+name+'-lines'+".jpg", lineimg)
# plt.imshow(lineimg)
# print("test_images/"+name+'-lines'+".jpg")
################### TEST ON VIDEO ############################
# 1. White video
white_output = 'white.mp4'
clip1 = VideoFileClip("solidWhiteRight.mp4")
white_clip = clip1.fl_image(
process_image) #NOTE: this function expects color images!!
white_clip.write_videofile(white_output, audio=False)
#Yellow video
yellow_output = 'yellow.mp4'
clip2 = VideoFileClip('solidYellowLeft.mp4')
yellow_clip = clip2.fl_image(process_image)
yellow_clip.write_videofile(yellow_output, audio=False)
challenge_output = 'extra.mp4'
clip2 = VideoFileClip('challenge.mp4')
challenge_clip = clip2.fl_image(process_image)
challenge_clip.write_videofile(challenge_output, audio=False)
result = cv2.cvtColor(result, cv2.COLOR_RGB2BGR)
return result
#%%
#import glob
#img_list = glob.glob('test_images/*.jpg')
#for i in range(len(img_list)):
# process(cv2.imread(img_list[i]))
#%%
#import glob
#img_list = glob.glob('test_images/*.jpg')
#i = 0
#
#for img_path in img_list:
# img = cv2.imread(img_path)
# output_img = process(img)
# print(output_img.shape)
# i += 1
# cv2.imwrite('test_images_output/output_image_'+str(i)+'.jpg', output_img)
#%%
video = VideoFileClip('project_video.mp4')
input = video.fl_image(process)
input.write_videofile('project_video_output_test.mp4', audio = False)
image_name = test_files[image_index]
## Read in and grayscale the image
image = mpimg.imread(image_name)
process(image)
#%% Run White mp4
from moviepy.editor import VideoFileClip
from IPython.display import HTML
test_dir = "../project1/"
white_output = os.path.join(test_dir, 'white.mp4')
clip1 = VideoFileClip( os.path.join(test_dir, "solidWhiteRight.mp4"))
white_clip = clip1.fl_image(process) #NOTE: this function expects color images!!
white_clip.write_videofile(white_output, audio=False)
HTML("""
<video width="960" height="540" controls>
<source src="{0}">
</video>
""".format(white_output))
#%% Run Yellow mp4
from moviepy.editor import VideoFileClip
from IPython.display import HTML
test_dir = "../project1/"
