def load_dataset(start_idx=1, end_idx=10, need_ori=False):
all_features = []
all_labels = []
print('Start to load datasets...')
if not os.path.exists(params.pickle_dir):
os.makedirs(params.pickle_dir)
for epoch_id in range(start_idx, end_idx):
print('Processing epoch{:0>2}>>>'.format(epoch_id))
features_file = utils.join_dir(params.pickle_dir,
'features{:0>2}.p'.format(epoch_id))
labels_file = utils.join_dir(params.pickle_dir,
'labels{:0>2}.p'.format(epoch_id))
features = load_features(features_file, epoch_id, need_ori)
labels = load_labels(labels_file, epoch_id, need_ori)
all_features += features
all_labels += labels
print("Done")
print("Length of features : " + str(len(all_features)) +
", length of labels : " + str(len(all_labels)))
return all_features, all_labels
def load_train(color_mode='RGB', flip=True):
imgs = []
wheels = []
epochs = [1, 2, 3, 4, 5, 6, 7, 8, 9]
#epochs = [1, 2, 3, 4]
for epoch in epochs:
vid_path = utils.join_dir(params.data_dir,
'epoch{:0>2}_front.mp4'.format(epoch))
assert os.path.isfile(vid_path)
frame_count = frame_count_func(vid_path)
cap = cv2.VideoCapture(vid_path)
for frame_id in range(frame_count):
while True:
ret, img = cap.read()
if not ret:
break
img = a_image_convert.img_preprocess(img, color_mode, flip)
imgs.append(img)
csv_path = os.path.join(data_dir,
'epoch{:0>2}_steering.csv'.format(epoch))
rows = pd.read_csv(csv_path)
yy = rows['wheel'].values
if flip:
yy = yy * (-1.0)
wheels.extend(yy)
cap.release()
imgs = np.array(imgs)
wheels = np.array(wheels)
wheels = np.reshape(wheels, (len(wheels), 1))
return imgs, wheels
def get_combine_img(start_idx, end_idx, need_ori=False):
imgs = []
for i in range(start_idx, end_idx + 1):
cap = cv2.VideoCapture(
utils.join_dir(params.data_dir, 'epoch{:0>2}_front.mkv'.format(i)))
while True:
ret, frame = cap.read()
if not ret:
break
#pre-process
frame = img_pre_process(frame, need_ori)
imgs.append(frame)
cap.release()
return imgs
## Tools
import utils
## Parameters
import params ## you can modify the content of params.py
#import preprocess
import a_video_capture
## Test epoch
epoch_ids = [10]
## Load model
model = utils.get_model()
## Process video
for epoch_id in epoch_ids:
print(
'---------- processing video for epoch {} ----------'.format(epoch_id))
vid_path = utils.join_dir(params.data_dir,
'epoch{:0>2}_front.mp4'.format(epoch_id))
assert os.path.isfile(vid_path)
cap = cv2.VideoCapture(vid_path)
frame_count = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
imgs_test, wheels_test = a_video_capture.load_test(color_mode='RGB')
machine_steering = []
print('performing inference...')
time_start = time.time()
machine_steering = model.predict(imgs_test, batch_size=128, verbose=0)
img = img[int(shape[0] / 3):shape[0] - 150, 0:shape[1]]
## Resize the image
img = cv2.resize(img, (params.FLAGS.img_w, params.FLAGS.img_h),
interpolation=cv2.INTER_AREA)
## Return the image sized as a 4D array
return np.resize(
img, (params.FLAGS.img_w, params.FLAGS.img_h, params.FLAGS.img_c))
## Process video
for epoch_id in epoch_ids:
print(
'---------- processing video for epoch {} ----------'.format(epoch_id))
#vid_path = utils.join_dir(params.data_dir, 'epoch{:0>2}_front.mkv'.format(epoch_id))
vid_path = utils.join_dir(
'C:/Users/ccc/Downloads/udacity-capstone-deeptesla-master/epochs/',
'epoch{:0>2}_front.mkv'.format(epoch_id))
print(vid_path)
assert os.path.isfile(vid_path)
frame_count = utils.frame_count(vid_path)
cap = cv2.VideoCapture(vid_path)
machine_steering = []
print('performing inference...')
time_start = time.time()
for frame_id in range(frame_count):
ret, img = cap.read()
assert ret
## you can modify here based on your model
img = img_pre_process(img)
## Training data
epoch_dir = params.data_dir
t_epoch_ids = [1, 2, 3, 4, 5, 6, 7, 8]
## validation data
v_epoch_ids = [9]
## test data
test_epoch_ids = [10]
##--------count all frames for training set and validation set-----------###
n_frames_t = np.zeros((len(t_epoch_ids), ), dtype=np.uint16)
k = 0
for i in t_epoch_ids:
video_path = utils.join_dir(epoch_dir, 'epoch{:0>2}_front.mkv'.format(i))
n_frames_t[k] = utils.ffmpeg_frame_count(video_path)
k += 1
sum_frames_t = n_frames_t.sum(axis=0)
print("total frames number----training set:", sum_frames_t)
n_frames_v = np.zeros((len(v_epoch_ids), ), dtype=np.uint16)
k = 0
for j in v_epoch_ids:
video_path = utils.join_dir(epoch_dir, 'epoch{:0>2}_front.mkv'.format(j))
n_frames_v[k] = utils.ffmpeg_frame_count(video_path)
k += 1
sum_frames_v = n_frames_v.sum(axis=0)
print("total frames number----validation set:", sum_frames_v)
# Chop off 1/2 from the top and cut bottom 150px(which contains the head of car)
ratio = img_height / img_width
h1, h2 = int(img.shape[0]/2),img.shape[0]-150
w = (h2-h1) / ratio
padding = int(round((img.shape[1] - w) / 2))
img = img[h1:h2, padding:-padding]
## Resize the image
img = cv2.resize(img, (img_width, img_height), interpolation=cv2.INTER_AREA)
## Image Normalization
#img = img / 255.
return img
## Process video
for epoch_id in epoch_ids:
print('---------- processing video for epoch {} ----------'.format(epoch_id))
vid_path = utils.join_dir(params.data_dir, 'epoch{:0>2}_front.mkv'.format(epoch_id))
assert os.path.isfile(vid_path)
cap = cv2.VideoCapture(vid_path)
# frame_count = utils.frame_count(vid_path)
# this line goes wrong, instead with below code
############ New codes added by student ############
frame_count = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
cap.release()
# import test data: epoch 10
imgs_test, wheels_test = preprocess.load_data('test')
imgs_test = np.array(imgs_test)
imgs_test = imgs_test.astype('float32')
imgs_test /= 255
####################################################
def save_model(model):
json_string = model.to_json()
open(utils.join_dir(params.model_dir, "model.json"),
'w').write(json_string)
model.save_weights(utils.join_dir(params.model_dir, "model.h5"))
print("Model and weights already saved")
def get_steering_file(epoch_id):
return utils.join_dir(params.data_dir,
'epoch{:0>2}_steering.csv'.format(epoch_id))
def get_front_file(epoch_id):
return utils.join_dir(params.data_dir,
'epoch{:0>2}_front.mkv'.format(epoch_id))
def process_epoch_data(epoch_id, YUV=True, flip=True, shadow=True):
print('---------- processing video for epoch {} ----------'.format(epoch_id))
vid_mkv_path = utils.join_dir(params.data_dir, 'epoch{:0>2}_front.mkv'.format(epoch_id))
# convert mkv to mp4
vid_mp4_path = utils.join_dir(params.data_dir, 'epoch{:0>2}_front.mp4'.format(epoch_id))
if not os.path.exists(vid_mp4_path):
mkv_to_mp4(vid_mkv_path, remove_mkv=False)
# slice mp4 to images
vid = imageio.get_reader(vid_mp4_path,'ffmpeg')
images = []
for num in range(len(vid)):
# for num in range(200):
images.append(vid.get_data(num))
print('epoch{:0>2}_front.mkv has {} images'.format(epoch_id,len(images)))
## Crop and Resize before data agumentation
images = [crop_resize_image(img) for img in images]
images = np.stack(images, axis=0)
# get steerings from csv file
steerings = process_epoch_csv(epoch_id)
print("images: ",len(images))
print("steerings: ",steerings.shape)
print("\t")
# do flip agumentation
if flip:
print('--------------------start flip images--------')
t0 = time.time()
# steering not equal 0, flip
flip_idx = np.where(steerings!=0)[0]
images_flip = images[flip_idx, :, ::-1, :]
steering_flip = -steerings[flip_idx]
print("images_flip num: ", images_flip.shape)
print("steering_flip num: ", steering_flip.shape)
# combine images_flip and images
images = np.concatenate((images, images_flip), axis=0)
steerings = np.concatenate((steerings,steering_flip), axis=0)
print("combine images num: ", images.shape)
print("combine steerings num: ", steerings.shape)
print("---------------------flip images using time: ",time.time()-t0)
# shadow agumentation
if shadow:
print("-------------------start adding shadow-------")
t0 = time.time()
# abs(steering) > 2, copy images, add shadowon new iamges
idx_new = np.where(np.abs(steerings)>2)[0]
# add shadow if num >3
if len(idx_new)>3:
images, steerings = add_shadow_images(idx_new, images, steerings, add_new=True)
# abs(steering) <= 2, add shadow on original images
idx_old = np.where(np.abs(steerings)<=2)[0]
if len(idx_old)>3:
images, steerings = add_shadow_images(idx_old, images, steerings, add_new=False)
print("images num after shadow : ", images.shape)
print("steerings num after shadow: ", steerings.shape)
print("--------------------adding shadow using time: ",time.time()-t0)
#yuv, normalization
images = [YUV_normal_image(img, YUV) for img in images]
images = np.stack(images, axis=0)
print("epoch {} has {} images after process: \n \n ".format(epoch_id,images.shape[0]))
assert np.max(images) <=1
assert np.min(images) >=0
return images,steerings
import cv2
import numpy as np
import utils
import variables
import preprocess
img_height = variables.img_height
img_width = variables.img_width
img_channels = variables.img_channels
epoch_id = 10
model = utils.get_model()
print('processing data for epoch {} ...'.format(epoch_id))
vid_path = utils.join_dir(variables.data_dir,
'epoch{:0>2}_front.mkv'.format(epoch_id))
assert os.path.isfile(vid_path)
cap = cv2.VideoCapture(vid_path)
frame_count = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
cap.release()
imgs_test, wheels_test = preprocess.load_data('test')
imgs_test = np.array(imgs_test)
machine_steering = []
print('performing inference...')
time_start = time.time()
machine_steering = model.predict(imgs_test, batch_size=128, verbose=0)
