def load_dataset(start_idx=1, end_idx=10, need_ori=False): all_features = [] all_labels = [] print('Start to load datasets...') if not os.path.exists(params.pickle_dir): os.makedirs(params.pickle_dir) for epoch_id in range(start_idx, end_idx): print('Processing epoch{:0>2}>>>'.format(epoch_id)) features_file = utils.join_dir(params.pickle_dir, 'features{:0>2}.p'.format(epoch_id)) labels_file = utils.join_dir(params.pickle_dir, 'labels{:0>2}.p'.format(epoch_id)) features = load_features(features_file, epoch_id, need_ori) labels = load_labels(labels_file, epoch_id, need_ori) all_features += features all_labels += labels print("Done") print("Length of features : " + str(len(all_features)) + ", length of labels : " + str(len(all_labels))) return all_features, all_labels
def load_train(color_mode='RGB', flip=True): imgs = [] wheels = [] epochs = [1, 2, 3, 4, 5, 6, 7, 8, 9] #epochs = [1, 2, 3, 4] for epoch in epochs: vid_path = utils.join_dir(params.data_dir, 'epoch{:0>2}_front.mp4'.format(epoch)) assert os.path.isfile(vid_path) frame_count = frame_count_func(vid_path) cap = cv2.VideoCapture(vid_path) for frame_id in range(frame_count): while True: ret, img = cap.read() if not ret: break img = a_image_convert.img_preprocess(img, color_mode, flip) imgs.append(img) csv_path = os.path.join(data_dir, 'epoch{:0>2}_steering.csv'.format(epoch)) rows = pd.read_csv(csv_path) yy = rows['wheel'].values if flip: yy = yy * (-1.0) wheels.extend(yy) cap.release() imgs = np.array(imgs) wheels = np.array(wheels) wheels = np.reshape(wheels, (len(wheels), 1)) return imgs, wheels
def get_combine_img(start_idx, end_idx, need_ori=False): imgs = [] for i in range(start_idx, end_idx + 1): cap = cv2.VideoCapture( utils.join_dir(params.data_dir, 'epoch{:0>2}_front.mkv'.format(i))) while True: ret, frame = cap.read() if not ret: break #pre-process frame = img_pre_process(frame, need_ori) imgs.append(frame) cap.release() return imgs
## Tools import utils ## Parameters import params ## you can modify the content of params.py #import preprocess import a_video_capture ## Test epoch epoch_ids = [10] ## Load model model = utils.get_model() ## Process video for epoch_id in epoch_ids: print( '---------- processing video for epoch {} ----------'.format(epoch_id)) vid_path = utils.join_dir(params.data_dir, 'epoch{:0>2}_front.mp4'.format(epoch_id)) assert os.path.isfile(vid_path) cap = cv2.VideoCapture(vid_path) frame_count = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)) imgs_test, wheels_test = a_video_capture.load_test(color_mode='RGB') machine_steering = [] print('performing inference...') time_start = time.time() machine_steering = model.predict(imgs_test, batch_size=128, verbose=0)
img = img[int(shape[0] / 3):shape[0] - 150, 0:shape[1]] ## Resize the image img = cv2.resize(img, (params.FLAGS.img_w, params.FLAGS.img_h), interpolation=cv2.INTER_AREA) ## Return the image sized as a 4D array return np.resize( img, (params.FLAGS.img_w, params.FLAGS.img_h, params.FLAGS.img_c)) ## Process video for epoch_id in epoch_ids: print( '---------- processing video for epoch {} ----------'.format(epoch_id)) #vid_path = utils.join_dir(params.data_dir, 'epoch{:0>2}_front.mkv'.format(epoch_id)) vid_path = utils.join_dir( 'C:/Users/ccc/Downloads/udacity-capstone-deeptesla-master/epochs/', 'epoch{:0>2}_front.mkv'.format(epoch_id)) print(vid_path) assert os.path.isfile(vid_path) frame_count = utils.frame_count(vid_path) cap = cv2.VideoCapture(vid_path) machine_steering = [] print('performing inference...') time_start = time.time() for frame_id in range(frame_count): ret, img = cap.read() assert ret ## you can modify here based on your model img = img_pre_process(img)
## Training data epoch_dir = params.data_dir t_epoch_ids = [1, 2, 3, 4, 5, 6, 7, 8] ## validation data v_epoch_ids = [9] ## test data test_epoch_ids = [10] ##--------count all frames for training set and validation set-----------### n_frames_t = np.zeros((len(t_epoch_ids), ), dtype=np.uint16) k = 0 for i in t_epoch_ids: video_path = utils.join_dir(epoch_dir, 'epoch{:0>2}_front.mkv'.format(i)) n_frames_t[k] = utils.ffmpeg_frame_count(video_path) k += 1 sum_frames_t = n_frames_t.sum(axis=0) print("total frames number----training set:", sum_frames_t) n_frames_v = np.zeros((len(v_epoch_ids), ), dtype=np.uint16) k = 0 for j in v_epoch_ids: video_path = utils.join_dir(epoch_dir, 'epoch{:0>2}_front.mkv'.format(j)) n_frames_v[k] = utils.ffmpeg_frame_count(video_path) k += 1 sum_frames_v = n_frames_v.sum(axis=0) print("total frames number----validation set:", sum_frames_v)
# Chop off 1/2 from the top and cut bottom 150px(which contains the head of car) ratio = img_height / img_width h1, h2 = int(img.shape[0]/2),img.shape[0]-150 w = (h2-h1) / ratio padding = int(round((img.shape[1] - w) / 2)) img = img[h1:h2, padding:-padding] ## Resize the image img = cv2.resize(img, (img_width, img_height), interpolation=cv2.INTER_AREA) ## Image Normalization #img = img / 255. return img ## Process video for epoch_id in epoch_ids: print('---------- processing video for epoch {} ----------'.format(epoch_id)) vid_path = utils.join_dir(params.data_dir, 'epoch{:0>2}_front.mkv'.format(epoch_id)) assert os.path.isfile(vid_path) cap = cv2.VideoCapture(vid_path) # frame_count = utils.frame_count(vid_path) # this line goes wrong, instead with below code ############ New codes added by student ############ frame_count = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)) cap.release() # import test data: epoch 10 imgs_test, wheels_test = preprocess.load_data('test') imgs_test = np.array(imgs_test) imgs_test = imgs_test.astype('float32') imgs_test /= 255 ####################################################
def save_model(model): json_string = model.to_json() open(utils.join_dir(params.model_dir, "model.json"), 'w').write(json_string) model.save_weights(utils.join_dir(params.model_dir, "model.h5")) print("Model and weights already saved")
def get_steering_file(epoch_id): return utils.join_dir(params.data_dir, 'epoch{:0>2}_steering.csv'.format(epoch_id))
def get_front_file(epoch_id): return utils.join_dir(params.data_dir, 'epoch{:0>2}_front.mkv'.format(epoch_id))
def process_epoch_data(epoch_id, YUV=True, flip=True, shadow=True): print('---------- processing video for epoch {} ----------'.format(epoch_id)) vid_mkv_path = utils.join_dir(params.data_dir, 'epoch{:0>2}_front.mkv'.format(epoch_id)) # convert mkv to mp4 vid_mp4_path = utils.join_dir(params.data_dir, 'epoch{:0>2}_front.mp4'.format(epoch_id)) if not os.path.exists(vid_mp4_path): mkv_to_mp4(vid_mkv_path, remove_mkv=False) # slice mp4 to images vid = imageio.get_reader(vid_mp4_path,'ffmpeg') images = [] for num in range(len(vid)): # for num in range(200): images.append(vid.get_data(num)) print('epoch{:0>2}_front.mkv has {} images'.format(epoch_id,len(images))) ## Crop and Resize before data agumentation images = [crop_resize_image(img) for img in images] images = np.stack(images, axis=0) # get steerings from csv file steerings = process_epoch_csv(epoch_id) print("images: ",len(images)) print("steerings: ",steerings.shape) print("\t") # do flip agumentation if flip: print('--------------------start flip images--------') t0 = time.time() # steering not equal 0, flip flip_idx = np.where(steerings!=0)[0] images_flip = images[flip_idx, :, ::-1, :] steering_flip = -steerings[flip_idx] print("images_flip num: ", images_flip.shape) print("steering_flip num: ", steering_flip.shape) # combine images_flip and images images = np.concatenate((images, images_flip), axis=0) steerings = np.concatenate((steerings,steering_flip), axis=0) print("combine images num: ", images.shape) print("combine steerings num: ", steerings.shape) print("---------------------flip images using time: ",time.time()-t0) # shadow agumentation if shadow: print("-------------------start adding shadow-------") t0 = time.time() # abs(steering) > 2, copy images, add shadowon new iamges idx_new = np.where(np.abs(steerings)>2)[0] # add shadow if num >3 if len(idx_new)>3: images, steerings = add_shadow_images(idx_new, images, steerings, add_new=True) # abs(steering) <= 2, add shadow on original images idx_old = np.where(np.abs(steerings)<=2)[0] if len(idx_old)>3: images, steerings = add_shadow_images(idx_old, images, steerings, add_new=False) print("images num after shadow : ", images.shape) print("steerings num after shadow: ", steerings.shape) print("--------------------adding shadow using time: ",time.time()-t0) #yuv, normalization images = [YUV_normal_image(img, YUV) for img in images] images = np.stack(images, axis=0) print("epoch {} has {} images after process: \n \n ".format(epoch_id,images.shape[0])) assert np.max(images) <=1 assert np.min(images) >=0 return images,steerings
import cv2 import numpy as np import utils import variables import preprocess img_height = variables.img_height img_width = variables.img_width img_channels = variables.img_channels epoch_id = 10 model = utils.get_model() print('processing data for epoch {} ...'.format(epoch_id)) vid_path = utils.join_dir(variables.data_dir, 'epoch{:0>2}_front.mkv'.format(epoch_id)) assert os.path.isfile(vid_path) cap = cv2.VideoCapture(vid_path) frame_count = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)) cap.release() imgs_test, wheels_test = preprocess.load_data('test') imgs_test = np.array(imgs_test) machine_steering = [] print('performing inference...') time_start = time.time() machine_steering = model.predict(imgs_test, batch_size=128, verbose=0)