def generate_thunderhill_batches(gen, batch_size): batch_x = [] batch_y = [] while True: for img, steering_angle in gen: for img1, steering_angle1 in RandomFlip(img, steering_angle): img1, steering_angle1 = RandomBrightness(img1, steering_angle1) img1, steering_angle1 = RandomShift(img1, steering_angle1) img1 = Preproc(img1) if steering_angle1 > 1: steering_angle1 = 1 if steering_angle1 < -1: steering_angle1 = -1 font = cv2.FONT_HERSHEY_SIMPLEX if show_images: imgp = Preproc(img.copy()) imgshow = (np.concatenate((imgp, img1), axis=1) + 0.5) cv2.putText(imgshow, '%.3f' % (steering_angle), (10, 50), font, 1, (0, 0, 255), 1, cv2.LINE_AA) cv2.putText(imgshow, '%.3f' % (steering_angle1), (330, 50), font, 1, (0, 0, 255), 1, cv2.LINE_AA) cv2.imshow('img', imgshow) cv2.waitKey(0) batch_x.append(np.reshape(img1, (1, HEIGHT, WIDTH, DEPTH))) batch_y.append([steering_angle1]) if len(batch_x) == batch_size: batch_x, batch_y = shuffle(batch_x, batch_y) yield np.vstack(batch_x), np.vstack(batch_y) batch_x = [] batch_y = []
def generate_thunderhill_batches(df, args): while True: batch_x = [] batch_y = [] for idx, row in df.iterrows(): steering_angle = row['steering'] speed = row['speed'] brake = row['brake'] throttle = row['throttle'] img = ReadImg(row['center']) img, steering_angle = RandomShift(img, steering_angle, args.adjustement) img, steering_angle = RandomFlip(img, steering_angle) img, steering_angle = RandomBrightness(img, steering_angle) img, steering_angle = RandomRotation(img, steering_angle) img, steering_angle = RandomBlur(img, steering_angle) # Preproc is after .... img = Preproc(img) batch_x.append(np.reshape(img, (1, HEIGHT, WIDTH, DEPTH))) batch_y.append([steering_angle, throttle, brake]) if len(batch_x) == args.batch: batch_x, batch_y = shuffle(batch_x, batch_y) yield np.vstack(batch_x), np.vstack(batch_y) batch_x = [] batch_y = []
def generate_batches(df, batch_size, basename='data'): while True: batch_x = [] batch_y = [] for idx, row in df.iterrows(): camera = np.random.choice(['left', 'center', 'right']) img = ReadImg('{}/{}'.format(basename, row[camera].strip())) if camera == 'left': steering_angle = min(row['steering'] + .20, 1) elif camera == 'center': steering_angle = row['steering'] elif camera == 'right': steering_angle = max(row['steering'] - .20, -1) img, steering_angle = RandomShift(img, steering_angle) img, steering_angle = RandomFlip(img, steering_angle) img, steering_angle = RandomBrightness(img, steering_angle) img = Preproc(img) batch_x.append(np.reshape(img, (1, 66, 200, 3))) batch_y.append([steering_angle]) if len(batch_x) == batch_size: batch_x, batch_y = shuffle(batch_x, batch_y) yield np.vstack(batch_x), np.vstack(batch_y) batch_x = [] batch_y = []
def generate_thunderhill_batches(gen, batch_size): batch_x = [] batch_x2 = [] batch_y = [] tasks=[] while True: for img, steering_angle, throttle, brake, speed in gen: for img1, steering_angle1 in RandomFlip(img, steering_angle): img1 = Preproc(img1) tasks.append((img1, steering_angle1, throttle, brake, speed)) if len(tasks)==BATCH_SIZE: results = dispatch_tasks(worker, tasks, multiprocessing.cpu_count()) tasks=[] for i,result in enumerate(results): res_list = result.get() batch_x.append(res_list[0][0]) batch_x2.append(res_list[0][1]) batch_y.append(res_list[1]) print(len(batch_x)) if len(batch_x) == batch_size: batch_x, batch_x2, batch_y = shuffle(batch_x, batch_x2, batch_y) yield [np.vstack(batch_x), np.vstack(batch_x2)], np.vstack(batch_y) batch_x = [] batch_x2 = [] batch_y = []
def telemetry(sid, data): global controller, count # The current steering angle of the car steering_angle = data["steering_angle"] # The current throttle of the car throttle = data["throttle"] # The current speed of the car speed = data["speed"] # The current image from the center camera of the car imgString = data["image"] image = Image.open(BytesIO(base64.b64decode(imgString))) image_array = cv2.cvtColor(np.asarray(image), code=cv2.COLOR_RGB2BGR) image_array = Preproc(image_array[50:-30, :, :]) font = cv2.FONT_HERSHEY_SIMPLEX # print(image_array.shape) transformed_image_array = image_array.reshape(1, 80, 160, 3) # This model currently assumes that the features of the model are just the images. Feel free to change this. steering_angle = float(model.predict(transformed_image_array, batch_size=1)) # The driving model currently just outputs a constant throttle. Feel free to edit this. throttle = controller.update(float(speed)) # Limit the vehicle speed if float(speed) > set_speed: throttle = 0 # Tap the brakes if steering is large (big turns) if steering_angle > 0.3 or steering_angle < -0.3: throttle = 0 if steering_angle > 0.4 or steering_angle < -0.4: count += 1 if count < 3: throttle = -1 count = 0 img = image_array + 0.5 cv2.putText(img, '%.1f %.1f' % (steering_angle, throttle), (10, 150), font, 1, (0, 0, 255), 1, cv2.LINE_AA) # cv2.imshow('img',img) # cv2.waitKey(1) # print(steering_angle, throttle, float(speed)) controller = SimplePIController(0.1, 0.01) controller.set_desired(set_speed) send_control(steering_angle, throttle)
def telemetry(sid, data): if data: # The current steering angle of the car steering_angle = data["steering_angle"] # The current throttle of the car throttle = data["throttle"] # The current speed of the car speed = data["speed"] # The current image from the center camera of the car imgString = data["image"] # The current image position positionX, positionY, positionZ = data['position'].split(":") # the current image rotation rotationX, rotationY, rotationZ = data['rotation'].split(":") image = Image.open(BytesIO(base64.b64decode(imgString))) image_array = np.asarray(image) image_array = image_array[20:140, :, :] image_array = Preproc(image_array) ############################# Steering angles ########################################################## transformed_image_array = image_array[None, :, :, :] predictions = model.predict(transformed_image_array, batch_size=1) # The driving model currently just outputs a constant throttle. Feel free to edit this. ############################# THROTTLE ########################################################## throttle = args.throttle throttle = float(steering_angle[1]) ############################## BRAKE ############################################################ if float(speed) > 70: if np.abs(steering_angle) > 0.01: throttle = 0 elif float(speed) > 70: if steering_angle > 0.2: throttle = throttle / 2. if steering_angle > 0.5: throttle = -throttle / 4. ############################## CONTROL ########################################################### print(steering_angle, throttle) send_control(steering_angle, throttle) # save frame if args.image_folder != '': timestamp = datetime.utcnow().strftime('%Y_%m_%d_%H_%M_%S_%f')[:-3] image_filename = os.path.join(args.image_folder, timestamp) image.save('{}.jpg'.format(image_filename)) else: # NOTE: DON'T EDIT THIS. sio.emit('manual', data={}, skip_sid=True)
def generate_validation(df, basename='data'): batch_x, batch_y = [], [] for idx, row in df.iterrows(): basename = '{}/{}'.format(basename, row['center'].strip()) steering_angle = row['steering'] img = ReadImg(basename) img = Preproc(img) batch_x.append(np.reshape(img, (1, 66, 200, 3))) batch_y.append([steering_angle]) return batch_x, batch_y
def telemetry(sid, data): # The current steering angle of the car steering_angle = data["steering_angle"] # The current throttle of the car throttle = data["throttle"] # The current speed of the car speed = data["speed"] # The current image from the center camera of the car imgString = data["image"] image = Image.open(BytesIO(base64.b64decode(imgString))) image_array = cv2.cvtColor(np.asarray(image), code=cv2.COLOR_RGB2BGR) image_array = Preproc(image_array) transformed_image_array = image_array[None, :, :, :] # This model currently assumes that the features of the model are just the images. Feel free to change this. steering_angle = float(model.predict(transformed_image_array, batch_size=1)) # The driving model currently just outputs a constant throttle. Feel free to edit this. throttle = 1 print(steering_angle, throttle) send_control(steering_angle, throttle)
def generate_thunderhill_batches(df, args): while True: batch_x = [] batch_y = [] sample_weights = [] for idx, row in df.iterrows(): steering_angle = row['steering'] speed = row['speed'] brake = row['brake'] throttle = row['throttle'] longitude = row['longitude'] latitude = row['latitude'] img = ReadImg(row['center']) img, steering_angle = RandomShift(img, steering_angle, args.adjustement) img, steering_angle = RandomFlip(img, steering_angle) img, steering_angle = RandomBrightness(img, steering_angle) img, steering_angle = RandomRotation(img, steering_angle) img, steering_angle = RandomBlur(img, steering_angle) # Preproc is after .... img = Preproc(img) batch_x.append(np.reshape(img, (1, HEIGHT, WIDTH, DEPTH))) # Lap Distance UTMp = LatLontoUTM(RadToDeg(longitude), RadToDeg(latitude)) _, _, LapDistance = NearestWayPointCTEandDistance(UTMp) LapDistance = 2 * (LapDistance / total_lap_distance) - 1 batch_y.append([steering_angle, LapDistance]) sample_weights.append(row['norm']) if len(batch_x) == args.batch: yield (np.vstack(batch_x), np.vstack(batch_y), np.array(sample_weights)) batch_x = [] batch_y = [] sample_weights = []
def telemetry(sid, data): if data: # The current steering angle of the car steering_angle = data["steering_angle"] # The current throttle of the car throttle = data["throttle"] # The current speed of the car speed = data["speed"] # The current image from the center camera of the car imgString = data["image"] # image = Image.open(BytesIO(base64.b64decode(imgString))) # image_array = np.asarray(image) # steering_angle = float(model.predict(image_array[None, :, :, :], batch_size=1)) # # throttle = controller.update(float(speed)) # # print(steering_angle, throttle) # send_control(steering_angle, throttle) image = Image.open(BytesIO(base64.b64decode(imgString))) image_array = cv2.cvtColor(np.asarray(image), code=cv2.COLOR_RGB2BGR) image_array = Preproc(image_array) transformed_image_array = image_array[None, :, :, :] # This model currently assumes that the features of the model are just the images. Feel free to change this. steering_angle = float(model.predict(transformed_image_array, batch_size=1)) # The driving model currently just outputs a constant throttle. Feel free to edit this. throttle = 0.3 print(steering_angle, throttle) send_control(steering_angle, throttle) # save frame if args.image_folder != '': timestamp = datetime.utcnow().strftime('%Y_%m_%d_%H_%M_%S_%f')[:-3] image_filename = os.path.join(args.image_folder, timestamp) image.save('{}.jpg'.format(image_filename)) else: # NOTE: DON'T EDIT THIS. sio.emit('manual', data={}, skip_sid=True)
def generate_thunderhill_batches_time(gen, batch_size, time=TIME): batch_x = [] batch_y = [] time_x = [] time_y = [] while True: for img1, steering_angle1 in gen: img1 = Preproc(img1) if steering_angle1 > 1: steering_angle1 = 1 if steering_angle1 < -1: steering_angle1 = -1 font = cv2.FONT_HERSHEY_SIMPLEX time_x.append(np.reshape(img1, (HEIGHT, WIDTH, DEPTH))) if len(time_x) == time: if show_images: print(steering_angle1) for t in range(time): imgshow = (time_x[t] + 0.5) font = cv2.FONT_HERSHEY_SIMPLEX cv2.putText(imgshow, '%d' % (t), (10, 50), font, 1, (0, 0, 255), 1, cv2.LINE_AA) cv2.imshow('img', imgshow) cv2.waitKey(100) batch_x.append(time_x) batch_y.append(steering_angle1) time_x = [] time_y = [] if len(batch_x) == batch_size: batch_x, batch_y = shuffle(batch_x, batch_y) batch_x = np.array(batch_x) batch_y = np.array(batch_y) yield batch_x, batch_y batch_x = [] batch_y = []
def generate_thunderhill_batches(df, batch_size): while True: batch_x = [] batch_y = [] for idx, row in df.iterrows(): steering_angle = row['steering'] # img = ReadImg('{}/{}'.format(basename, row['center'].strip())) img = ReadImg(row['center']) img, steering_angle = RandomShift(img, steering_angle) img, steering_angle = RandomFlip(img, steering_angle) img, steering_angle = RandomBrightness(img, steering_angle) img = Preproc(img) batch_x.append(np.reshape(img, (1, 66, 200, 3))) batch_y.append([steering_angle]) if len(batch_x) == batch_size: batch_x, batch_y = shuffle(batch_x, batch_y) yield np.vstack(batch_x), np.vstack(batch_y) batch_x = [] batch_y = []
def generate_thunderhill_batches(gen, batch_size): batch_x = [] batch_x2 = [] batch_y = [] while True: for img, steering_angle, throttle, brake, speed, longitude, latitude in gen: for img1, steering_angle1 in RandomFlip(img, steering_angle): img1, steering_angle1 = RandomBrightness(img1, steering_angle1) img1 = Preproc(img1) img1, steering_angle1, throttle1 = RandomShift(img1, steering_angle1,throttle) brake1=brake if steering_angle1>0.5: brake1=1 throttle1=0 if steering_angle1<-0.5: brake1=1 throttle1=0 if np.random.uniform() < 0.5: speed1 = speed * np.random.uniform() else: speed1 = speed # speed_cl=np.array(speedToClass(speed)) # speed_cl1=np.array(speedToClass(speed1)) if speed1 < 20*0.44704: throttle1=1 if speed1 > 50*0.44704: throttle1=0 if speed1 > 70*0.44704: throttle1=0 brake=1 if brake>0.7: throttle=0 # if speed_cl[-2]==1 and steering_angle1!=0: # throttle1=0 # if speed_cl[-1]==1 and steering_angle1!=0: # brake1=1 # throttle1=0 batch_x.append(np.reshape(img1, (1, HEIGHT, WIDTH, DEPTH))) spd1 = speed1/40 - 0.5 batch_x2.append(spd1) font = cv2.FONT_HERSHEY_SIMPLEX if show_images: imgp = Preproc(img.copy()[::-1,:,:]) imgshow=(np.concatenate((imgp,img1[::-1,:,:]),axis=0)+0.5)*0.5 imgshow = cv2.resize(imgshow,(0,0),fx=2,fy=2) # cv2.putText(imgshow,'%s'%(str(list(speed_cl))),(10,30), font, 0.8,(0,0,255),2,cv2.LINE_AA) # cv2.putText(imgshow,'%s'%(str(list(speed_cl1))),(10,190), font, 0.8,(0,0,255),2,cv2.LINE_AA) cv2.putText(imgshow,'%.3f %.1f %.1f %.1f'%(steering_angle,throttle, brake,speed),(10,140), font, 0.8,(0,0,255),2,cv2.LINE_AA) cv2.putText(imgshow,'%.3f %.1f %.1f %.1f'%(steering_angle1,throttle1, brake1,speed1),(10,300), font, 0.8,(0,0,255),2,cv2.LINE_AA) cv2.putText(imgshow,'%.5f %.5f'%(longitude, latitude),(10,110), font, 0.8,(0,0,255),2,cv2.LINE_AA) cv2.imshow('img',imgshow) cv2.waitKey(0) batch_y.append([steering_angle1,throttle1, brake1]) if len(batch_x) == batch_size: batch_x, batch_x2, batch_y = shuffle(batch_x, batch_x2, batch_y) yield [np.vstack(batch_x), np.vstack(batch_x2)], np.vstack(batch_y) batch_x = [] batch_x2 = [] batch_y = []
def generate_thunderhill_batches(gen, batch_size): batch_x = [] batch_x2 = [] batch_y = [] while True: for img, steering_angle, throttle, brake, speed in gen: for img1, steering_angle1 in RandomFlip(img, steering_angle): img1, steering_angle1 = RandomBrightness(img1, steering_angle1) img1, steering_angle1, throttle1 = RandomShift( img1, steering_angle1, throttle) img1 = Preproc(img1) if steering_angle1 > 1: steering_angle1 = 1 if steering_angle1 < -1: steering_angle1 = -1 brake1 = brake if steering_angle1 > 0.5: brake1 = 1 throttle1 = 0 if steering_angle1 < -0.5: brake1 = 1 throttle1 = 0 speed_cl = np.array(speedToClass(speed)) if speed_cl[-2] == 1 and steering_angle1 != 0: throttle1 = 0 if speed_cl[-1] == 1 and steering_angle1 != 0: brake1 = 1 throttle1 = 0 font = cv2.FONT_HERSHEY_SIMPLEX if show_images: imgp = Preproc(img.copy()) imgshow = (np.concatenate( (imgp, img1), axis=0) + 0.5) * 0.5 cv2.putText(imgshow, '%s' % (str(list(speed_cl))), (10, 30), font, 0.4, (0, 0, 255), 1, cv2.LINE_AA) cv2.putText( imgshow, '%.3f %.1f %.1f %.1f' % (steering_angle, throttle, brake, speed), (10, 70), font, 0.4, (0, 0, 255), 1, cv2.LINE_AA) cv2.putText( imgshow, '%.3f %.1f %.1f' % (steering_angle1, throttle1, brake1), (10, 150), font, 0.4, (0, 0, 255), 1, cv2.LINE_AA) cv2.imshow('img', imgshow) cv2.waitKey(0) batch_x.append(np.reshape(img1, (1, HEIGHT, WIDTH, DEPTH))) batch_x2.append(speed_cl) batch_y.append([steering_angle1, throttle1, brake1]) if len(batch_x) == batch_size: batch_x, batch_x2, batch_y = shuffle( batch_x, batch_x2, batch_y) yield [np.vstack(batch_x), np.vstack(batch_x2)], np.vstack(batch_y) batch_x = [] batch_x2 = [] batch_y = []
def generate_thunderhill_batches(gen, batch_size): batch_x = [] batch_x2 = [] batch_x3 = [] batch_y = [] while True: for img, steering_angle, throttle, brake, speed, longitude, latitude in gen: for img1, steering_angle1 in RandomFlip(img, steering_angle): img1, steering_angle1 = RandomBrightness(img1, steering_angle1) img1 = Preproc(img1) img1, steering_angle1, throttle1 = RandomShift( img1, steering_angle1, throttle) brake1 = brake if steering_angle1 > 0.5: brake1 = 1 throttle1 = 0 if steering_angle1 < -0.5: brake1 = 1 throttle1 = 0 if np.random.uniform() < 0.5: speed1 = speed * np.random.uniform() else: speed1 = speed speed_cl = np.array(speedToClass(speed)) speed_cl1 = np.array(speedToClass(speed1)) if speed1 < 20 * 0.44704: throttle1 = 1 if speed1 > 50 * 0.44704: throttle1 = 0 if speed1 > 70 * 0.44704: throttle1 = 0 brake = 1 if brake > 0.7: throttle = 0 # if speed_cl[-2]==1 and steering_angle1!=0: # throttle1=0 # if speed_cl[-1]==1 and steering_angle1!=0: # brake1=1 # throttle1=0 batch_x.append(np.reshape(img1, (1, HEIGHT, WIDTH, DEPTH))) batch_x2.append(speed_cl1) UTMp = LatLontoUTM(RadToDeg(longitude), RadToDeg(latitude)) CTE, Distance2NextWaypoint, LapDistance = NearestWayPointCTEandDistance( UTMp) CTE /= 5.0 LapDistance = 2 * (LapDistance / total_lap_distance) - 1 font = cv2.FONT_HERSHEY_SIMPLEX if show_images and (CTE > 100 or LapDistance > 2 or LapDistance < -2): imgp = Preproc(img.copy()[::-1, :, :]) imgshow = (np.concatenate( (imgp, img1[::-1, :, :]), axis=0) + 0.5) * 0.5 imgshow = cv2.resize(imgshow, (0, 0), fx=2, fy=2) cv2.putText(imgshow, '%s' % (str(list(speed_cl))), (10, 30), font, 0.8, (0, 0, 255), 2, cv2.LINE_AA) cv2.putText(imgshow, '%s' % (str(list(speed_cl1))), (10, 190), font, 0.8, (0, 0, 255), 2, cv2.LINE_AA) cv2.putText( imgshow, '%.3f %.1f %.1f %.1f' % (steering_angle, throttle, brake, speed), (10, 140), font, 0.8, (0, 0, 255), 2, cv2.LINE_AA) cv2.putText( imgshow, '%.3f %.1f %.1f %.1f' % (steering_angle1, throttle1, brake1, speed1), (10, 300), font, 0.8, (0, 0, 255), 2, cv2.LINE_AA) cv2.putText(imgshow, '%.5f %.5f' % (longitude, latitude), (10, 110), font, 0.8, (0, 0, 255), 2, cv2.LINE_AA) cv2.putText(imgshow, '%.3f %.3f' % (CTE, LapDistance), (10, 250), font, 0.8, (0, 0, 255), 2, cv2.LINE_AA) cv2.imshow('img', imgshow) cv2.waitKey(0) batch_x3.append((CTE, LapDistance)) batch_y.append([steering_angle1, throttle1, brake1]) if len(batch_x) == batch_size: batch_x, batch_x2, batch_x3, batch_y = shuffle( batch_x, batch_x2, batch_x3, batch_y) yield [ np.vstack(batch_x), np.vstack(batch_x2), np.vstack(batch_x3) ], np.vstack(batch_y) batch_x = [] batch_x2 = [] batch_x3 = [] batch_y = []
def telemetry(sid, data): if data: # The current steering angle of the car steering_angle = data["steering_angle"] # The current throttle of the car throttle = data["throttle"] # The current speed of the car speed = data["speed"] # The current image from the center camera of the car imgString = data["image"] # The current image position positionX, positionY, positionZ = data['position'].split(":") # the current image rotation rotationX, rotationY, rotationZ = data['rotation'].split(":") # image = Image.open(BytesIO(base64.b64decode(imgString))) # image_array = np.asarray(image) # steering_angle = float(model.predict(image_array[None, :, :, :], batch_size=1)) # # throttle = controller.update(float(speed)) # print(steering_angle, throttle) # send_control(steering_angle, throttle) image = Image.open(BytesIO(base64.b64decode(imgString))) # image_array = cv2.cvtColor(np.asarray(image), code=cv2.COLOR_RGB2BGR) image_array = np.asarray(image) image_array = Preproc(image_array) ################### LSTM transformed_image_array = feature_extractor.predict(np.reshape(image_array, (1, HEIGHT, WIDTH, DEPTH)))[0] # Adding other features speed and other stuff... # X = np.array([positionX, positionY, positionZ, rotationX, rotationY, rotationZ, speed], dtype=float) # X = X_scaler.transform(X) # transformed_image_array = np.hstack((transformed_image_array, X)) prev_image_array.pop(0) prev_image_array.append(transformed_image_array) steering_angle = float(model.predict(np.array(prev_image_array)[None, :])) ############################# Steering angles ########################################################## # transformed_image_array = image_array[None, :, :, :] # steering_angle = float(model.predict(transformed_image_array, batch_size=1)) # The driving model currently just outputs a constant throttle. Feel free to edit this. throttle = args.throttle ############################# THROTTLE ########################################################## # X = np.array([positionX, positionY, positionZ, rotationX, rotationY, rotationZ, 0, speed, steering_angle], dtype=float) # X_test = X_scaler.transform(X.reshape(1, -1)) # throttle = np.min([1, svr.predict(X_test)]) ############################## BRAKE ############################################################ # if np.abs(steering_angle) > 0.2 and int(speed) > 50: # throttle = -throttle/4 # elif np.abs(steering_angle) > 0.5: # throttle = -throttle print(steering_angle, throttle) send_control(steering_angle, throttle) # save frame if args.image_folder != '': timestamp = datetime.utcnow().strftime('%Y_%m_%d_%H_%M_%S_%f')[:-3] image_filename = os.path.join(args.image_folder, timestamp) image.save('{}.jpg'.format(image_filename)) else: # NOTE: DON'T EDIT THIS. sio.emit('manual', data={}, skip_sid=True)