def _load_settings(self): settings = Settings() # Screen data image_front_border_left = settings.get_value(settings.IMAGE_FRONT_BORDER_LEFT) image_front_border_right = settings.get_value(settings.IMAGE_FRONT_BORDER_RIGHT) image_front_border_top = settings.get_value(settings.IMAGE_FRONT_BORDER_TOP) image_front_border_bottom = settings.get_value(settings.IMAGE_FRONT_BORDER_BOTTOM) # Controller binding vjoy = settings.get_value(settings.VJOY_DEVICE) autopilot = settings.get_value(settings.AUTOPILOT) left_indicator = settings.get_value(settings.LEFT_INDICATOR) right_indicator = settings.get_value(settings.RIGHT_INDICATOR) steering_axis = settings.get_value(settings.STEERING_AXIS) throttle_axis = settings.get_value(settings.THROTTLE_AXIS) self.fill_screen_list() self.fill_device_list() self.select_screen() # Set scrollbar values if image_front_border_right is not None: self.ui.slider_right.setValue(int(image_front_border_right)) else: self.ui.slider_right.setValue(self.ui.slider_right.maximum()) if image_front_border_left is not None: self.ui.slider_left.setValue(int(image_front_border_left)) else: self.ui.slider_left.setValue(self.ui.slider_left.minimum()) if image_front_border_bottom is not None: self.ui.slider_bottom.setValue(int(image_front_border_bottom)) else: self.ui.slider_bottom.setValue(self.ui.slider_bottom.maximum()) if image_front_border_top is not None: self.ui.slider_top.setValue(int(image_front_border_top)) else: self.ui.slider_top.setValue(self.ui.slider_top.minimum()) # Display key bindings if vjoy is not None: self.ui.e_vjoy.setText(str(vjoy)) if autopilot is not None: self.ui.e_autopilot.setText(str(autopilot)) if left_indicator is not None: self.ui.e_leftIndicator.setText(str(left_indicator)) if right_indicator is not None: self.ui.e_rightIndicator.setText(str(right_indicator)) if steering_axis is not None: self.ui.e_steering.setText(str(steering_axis)) if throttle_axis is not None: self.ui.e_throttle.setText(str(throttle_axis)) self.fill_screen_cap()
def run(self): # Settings instance s = Settings() # State of autopilot self.autopilot = False img_wheel = cv2.imread('steering_wheel_image.jpg', 0) img_wheel = cv2.cvtColor(img_wheel, cv2.COLOR_BGR2RGB) functions.set_image(img_wheel.copy(), self.steering_wheel_ui) rows, cols, _ = img_wheel.shape keyboard.add_hotkey('shift+w', self.hotkey_callback) start_time = functions.current_milli_time() while AutopilotThread.running: self.controller_thread.set_autopilot(self.autopilot) # Get frame of game frame_raw = ImageGrab.grab(bbox=functions.get_screen_bbox()) frame = np.uint8(frame_raw) frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) # Relevant image region for steering angle prediction main = frame[s.get_value(Settings.IMAGE_FRONT_BORDER_TOP):s. get_value(Settings.IMAGE_FRONT_BORDER_BOTTOM), s.get_value(Settings.IMAGE_FRONT_BORDER_LEFT):s. get_value(Settings.IMAGE_FRONT_BORDER_RIGHT)] # Resize the image to the size of the neural network input layer image = scipy.misc.imresize(main, [66, 200]) / 255.0 # Let the neural network predict the new steering angle y_eval = model.y.eval(session=self.sess, feed_dict={ model.x: [image], model.keep_prob: 1.0 })[0][0] degrees = y_eval * 180 / scipy.pi steering = int(round((degrees + 180) / 180 * 32768 / 2)) # Value for vjoy controller # Set the value of the vjoy joystick to the predicted steering angle if self.autopilot: self.controller_thread.set_angle(steering) self.statusbar.showMessage("Autopilot active, steering: " + str(steering)) else: self.statusbar.showMessage("Autopilot inactive") M = cv2.getRotationMatrix2D((cols / 2, rows / 2), -degrees, 1) dst = cv2.warpAffine(img_wheel, M, (cols, rows)) functions.set_image(dst.copy(), self.steering_wheel_ui) functions.set_image(main.copy(), self.image_front_ui) wait_time = functions.current_milli_time() - start_time - 40 if wait_time < 0: time.sleep(-wait_time / 1000) start_time = functions.current_milli_time() keyboard.clear_hotkey('shift+w') self.controller_thread.set_autopilot(False)
def run(self): # Settings instance s = Settings() # State of autopilot autopilot = False # Previous state of the autopilot button autopilot_button_prev = 0 # Previous value of steering (gamepad) manual_steering_prev = 0 img_wheel = cv2.imread('steering_wheel_image.jpg', 0) rows, cols = img_wheel.shape while AutopilotThread.running: pygame.event.pump() # Button to activate/deactivate autopilot autopilot_button_act = self.joystick.get_button(self.b_autopilot) # Button was pressed if autopilot_button_act != autopilot_button_prev and autopilot_button_act == 1: autopilot = not autopilot #if autopilot and settings.AUTOPILOT_SOUND_ACTIVATE: # autopilot_engage.play() autopilot_button_prev = autopilot_button_act # Read the steering value of joystick axis = round( (self.joystick.get_axis(self.steering_axis) + 1) * 32768 / 2) # Interrupt autopilot if manual steering was detected if abs(manual_steering_prev - axis) > 500 and autopilot: img_id = Data().get_next_fileid() sequence_id = Data().add_sequence(country=Settings().get_value( Settings.COUNTRY_DEFAULT), note="correction") self.controller_thread.set_autopilot(False) self.statusbar.showMessage("Autopilot inactive") # TODO: Deactivate this feature in settings # TODO: Amount of images to save in settings # Save the next 3 images for i in range(3): # Get frame of game frame_raw = ImageGrab.grab( bbox=functions.get_screen_bbox()) frame = np.uint8(frame_raw) frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) # Relevant image region for steering angle prediction main = frame[ s.get_value(Settings.IMAGE_FRONT_BORDER_TOP):s. get_value(Settings.IMAGE_FRONT_BORDER_BOTTOM), s.get_value(Settings.IMAGE_FRONT_BORDER_LEFT):s. get_value(Settings.IMAGE_FRONT_BORDER_RIGHT)] # Resize image to save some space (height = 100px) ratio = main.shape[1] / main.shape[0] resized = cv2.resize(main, (round(ratio * 100), 100)) axis = self.joystick.get_axis( s.get_value(Settings.STEERING_AXIS)) * 180 cv2.imwrite("captured/%d.png" % img_id, resized) Data().add_image("%d.png" % img_id, axis, 0, 0, 0, sequence_id) img_id += 1 time.sleep(0.150) autopilot = False manual_steering_prev = axis self.controller_thread.set_autopilot(autopilot) # Get frame of game frame_raw = ImageGrab.grab(bbox=functions.get_screen_bbox()) frame = np.uint8(frame_raw) frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) # Relevant image region for steering angle prediction main = frame[s.get_value(Settings.IMAGE_FRONT_BORDER_TOP):s. get_value(Settings.IMAGE_FRONT_BORDER_BOTTOM), s.get_value(Settings.IMAGE_FRONT_BORDER_LEFT):s. get_value(Settings.IMAGE_FRONT_BORDER_RIGHT)] # Resize the image to the size of the neural network input layer image = scipy.misc.imresize(main, [66, 200]) / 255.0 # Let the neural network predict the new steering angle y_eval = model.y.eval(session=self.sess, feed_dict={ model.x: [image], model.keep_prob: 1.0 })[0][0] degrees = y_eval * 180 / scipy.pi steering = int(round((degrees + 180) / 180 * 32768 / 2)) # Value for vjoy controller # Set the value of the vjoy joystick to the predicted steering angle if autopilot: self.controller_thread.set_angle(steering) self.statusbar.showMessage("Autopilot active") else: self.statusbar.showMessage("Autopilot inactive") # TODO: Show steering wheel in GUI M = cv2.getRotationMatrix2D((cols / 2, rows / 2), -degrees, 1) dst = cv2.warpAffine(img_wheel, M, (cols, rows)) # functions.set_image(dst.copy(), self.steering_wheel) functions.set_image(main.copy(), self.image_front) self.controller_thread.set_autopilot(False)
class DrivingData(object): def __init__(self): self.settings = Settings() self.data = Data() country_string = self.settings.get_value(Settings.COUNTRIES_MODEL) countries = country_string.split(",") self.xs = [] self.ys = [] # points to the end of the last batch self.train_batch_pointer = 0 self.val_batch_pointer = 0 # Get all images self.image_list = [] for country in countries: self.image_list += self.data.get_image_list_filter(country=country, maneuver=0) for image in self.image_list: self.steering_deg = float(image[2]) * scipy.pi / 180 # higher steering angles are rare, so add four times # if abs(steering_deg) > 40: # for i in range(int(steering_deg/10-2)*4): # xs.append("../captured/" + line.split()[0]) # ys.append(steering_deg) self.xs.append(os.path.join("captured/", image[1])) # the paper by Nvidia uses the inverse of the turning radius, # but steering wheel angle is proportional to the inverse of turning radius # so the steering wheel angle in radians is used as the output self.ys.append(self.steering_deg) # get number of images self.num_images = len(self.xs) # shuffle list of images self.c = list(zip(self.xs, self.ys)) random.shuffle(self.c) self.xs, self.ys = zip(*self.c) # Training data self.train_xs = self.xs[:int(len(self.xs) * 0.8)] self.train_ys = self.ys[:int(len(self.xs) * 0.8)] # Validation data self.val_xs = self.xs[-int(len(self.xs) * 0.2):] self.val_ys = self.ys[-int(len(self.xs) * 0.2):] self.num_train_images = len(self.train_xs) self.num_val_images = len(self.val_xs) print("Total data:", len(self.xs), self.num_images) print("Training data:", len(self.train_xs)) print("Validation data:", len(self.val_xs)) def LoadTrainBatch(self, batch_size): x_out = [] y_out = [] for i in range(0, batch_size): x_out.append( scipy.misc.imresize( scipy.misc.imread(self.train_xs[ (self.train_batch_pointer + i) % self.num_train_images])[-150:], [66, 200]) / 255.0) y_out.append([ self.train_ys[(self.train_batch_pointer + i) % self.num_train_images] ]) self.train_batch_pointer += batch_size return x_out, y_out def LoadValBatch(self, batch_size): x_out = [] y_out = [] for i in range(0, batch_size): x_out.append( scipy.misc.imresize( scipy.misc.imread( self.val_xs[(self.val_batch_pointer + i) % self.num_val_images])[-150:], [66, 200]) / 255.0) y_out.append([ self.val_ys[(self.val_batch_pointer + i) % self.num_val_images] ]) self.val_batch_pointer += batch_size return x_out, y_out
import sys import os from PyQt5.QtWidgets import QApplication, QMainWindow print("Loading...") from UI.main import MainUI from database import Settings dbs = Settings() # MIGRATE if not dbs.get_value("migrated"): print("Migrating data. This may take a while...") dbs.set_value(dbs.COUNTRY_DEFAULT, "DE") dbs.set_value(dbs.COUNTRIES_MODEL, "DE") if os.path.exists("captured/data.txt") and os.path.exists( "captured/sequence.txt"): import migrate migrate.migrate() if os.path.exists("settings.py"): import settings as s dbs.set_value(dbs.CONTROLLER, s.JOYSTICK) dbs.set_value(dbs.VJOY_DEVICE, s.VJOY_DEVICE) dbs.set_value(dbs.AUTOPILOT_SOUND_ACTIVATE, s.AUTOPILOT_SOUND_ACTIVATE) dbs.set_value(dbs.ADAPTIVE_STEERING, s.ADAPTIVE_STEERING) dbs.set_value(dbs.AUTOPILOT, s.AUTOPILOT_BUTTON) dbs.set_value(dbs.STEERING_AXIS, s.STEERING_AXIS) dbs.set_value(dbs.THROTTLE_AXIS, s.THROTTLE_AXIS) dbs.set_value(dbs.LEFT_INDICATOR, s.INDICATOR_LEFT_BUTTON)
def run(self): s = Settings() d = Data(batch=True) img_id = d.get_next_fileid() recording = False recording_button_prev = 0 maneuver = 0 # 0 - normal, 1 - indicator left, 2 - indicator right indicator_left = False indicator_left_prev = 0 indicator_right = False indicator_right_prev = 0 last_record = 0 while RecordingThread.running: pygame.event.pump() recording_button_act = self.joystick.get_button(s.get_value(Settings.AUTOPILOT)) if recording_button_act != recording_button_prev and recording_button_act == 1: recording = not recording if recording: # started recording sequence_id = d.add_sequence() else: # stopped recording self.fill_sequence_list() recording_button_prev = recording_button_act indicator_left_act = self.joystick.get_button(s.get_value(Settings.LEFT_INDICATOR)) if indicator_left_act != indicator_left_prev and indicator_left_act == 1: indicator_left = not indicator_left # Switch indicator if indicator_left and indicator_right: indicator_right = False indicator_left_prev = indicator_left_act indicator_right_act = self.joystick.get_button(s.get_value(Settings.RIGHT_INDICATOR)) if indicator_right_act != indicator_right_prev and indicator_right_act == 1: indicator_right = not indicator_right # Switch indicator if indicator_right and indicator_left: indicator_left = False indicator_right_prev = indicator_right_act if indicator_left: maneuver = 1 elif indicator_right: maneuver = 2 else: maneuver = 0 if recording: self.statusbar.showMessage("Recording: active | Indicator: %s" % functions.get_indicator(maneuver)) else: self.statusbar.showMessage("Recording: inactive | Indicator: %s" % functions.get_indicator(maneuver)) # Capture the whole game frame_raw = ImageGrab.grab(bbox=functions.get_screen_bbox()) frame = np.uint8(frame_raw) frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) main = frame[s.get_value(Settings.IMAGE_FRONT_BORDER_TOP):s.get_value(Settings.IMAGE_FRONT_BORDER_BOTTOM), s.get_value(Settings.IMAGE_FRONT_BORDER_LEFT): s.get_value(Settings.IMAGE_FRONT_BORDER_RIGHT)] # gray = cv2.cvtColor(main, cv2.COLOR_BGR2GRAY) # blur_gray = cv2.GaussianBlur(gray, (3, 3), 0) # edges = cv2.Canny(blur_gray, 50, 150) # dilated = cv2.dilate(edges, (3,3), iterations=2) # Resize image to save some space (height = 100px) ratio = main.shape[1] / main.shape[0] resized = cv2.resize(main, (round(ratio * 100), 100)) # cv2.imshow('cap', dilated) # cv2.imshow('resized', resized) functions.set_image(main.copy(), self.image_front) axis = self.joystick.get_axis(s.get_value(Settings.STEERING_AXIS)) * 180 # -180 to 180 "degrees" throttle = self.joystick.get_axis(s.get_value(Settings.THROTTLE_AXIS)) * 100 # -100=full throttle, 100=full brake speed = speed_detection.get_speed(frame) # Save frame every 150ms if recording and (functions.current_milli_time() - last_record) >= 150: last_record = functions.current_milli_time() cv2.imwrite("captured/%d.png" % img_id, resized) d.add_image("%d.png" % img_id, axis, speed, throttle, maneuver, sequence_id) img_id += 1 d.append()
def run(self): # Settings instance s = Settings() # State of autopilot autopilot = False # Previous state of the autopilot button autopilot_button_prev = 0 # Previous value of steering (gamepad) manual_steering_prev = 0 while AutopilotThread.running: pygame.event.pump() # Button to activate/deactivate autopilot autopilot_button_act = self.joystick.get_button(self.b_autopilot) # Button was pressed if autopilot_button_act != autopilot_button_prev and autopilot_button_act == 1: autopilot = not autopilot # if autopilot and settings.AUTOPILOT_SOUND_ACTIVATE: # autopilot_engage.play() autopilot_button_prev = autopilot_button_act # Read the steering value of joystick axis = round((self.joystick.get_axis(self.steering_axis) + 1) * 32768 / 2) # Interrupt autopilot if manual steering was detected if abs(manual_steering_prev - axis) > 1000 and autopilot: self.controller_thread.set_autopilot(False) autopilot = False manual_steering_prev = axis self.controller_thread.set_autopilot(autopilot) # Get frame of game frame_raw = ImageGrab.grab(bbox=functions.get_screen_bbox()) frame = np.uint8(frame_raw) frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) # Relevant image region for steering angle prediction main = frame[s.get_value(Settings.IMAGE_FRONT_BORDER_TOP):s.get_value(Settings.IMAGE_FRONT_BORDER_BOTTOM), s.get_value(Settings.IMAGE_FRONT_BORDER_LEFT):s.get_value(Settings.IMAGE_FRONT_BORDER_RIGHT)] # Resize the image to the size of the neural network input layer image = scipy.misc.imresize(main, [66, 200]) / 255.0 ### Detect lane and steer ### # Do a perspective transformation of the lane. M, Minv = get_perspective_transform_matrix(main, 1, 0.2, 0.4, 0, [300, 300]) image_warped = cv2.warpPerspective(main.copy(), M, (300, 300), flags=cv2.INTER_LINEAR) # Filter lane markings. lower_white = np.array([180, 180, 180]) upper_white = np.array([255, 255, 255]) mask = cv2.inRange(image_warped, lower_white, upper_white) image_warped_filtered = cv2.bitwise_and(image_warped, image_warped, mask=mask) _, image_warped_filtered_binary = cv2.threshold(image_warped_filtered, 1, 255, cv2.THRESH_BINARY) # # Find position of left and right markings. # histogram = generate_column_histogram(image_warped_filtered_binary) # left_markings = histogram.index(max(histogram[:150])) # right_markings = histogram.index(max(histogram[150:])) # log.debug((left_markings, right_markings)) window_width = 75 window_height = 50 # First half (left markings) column_count = int(image_warped_filtered_binary.shape[1]/window_width) left_prediction = [] right_prediction = [] for column in range(0, int(column_count/2)): left_predicted_center = int(window_width/2 + column*window_width) left_prediction.append(get_centered_sliding_window(image_warped_filtered_binary, left_predicted_center, window_width, window_height)) right_predicted_center = int(window_width / 2 + (column + column_count/2) * window_width) right_prediction.append(get_centered_sliding_window(image_warped_filtered_binary, right_predicted_center, window_width, window_height)) # Select the sector with the highest maximum. left_markings_histogram_max = [x[2] for x in left_prediction] left_markings_center = [x[1] for x in left_prediction] left_markings = left_markings_center[left_markings_histogram_max.index(max(left_markings_histogram_max))] right_markings_histogram_max = [x[2] for x in right_prediction] right_markings_center = [x[1] for x in right_prediction] right_markings = right_markings_center[right_markings_histogram_max.index(max(right_markings_histogram_max))] log.debug(('LEFT', left_markings_center, left_markings_histogram_max)) log.debug(('RIGHT', right_markings_center, right_markings_histogram_max)) log.debug(('CHOSE', left_markings, right_markings)) left_centers = [None] right_centers = [None] if left_markings > 0 and right_markings > 0: # Apply sliding window technique. window_count = int(image_warped_filtered_binary.shape[0]/window_height) left_centers = [left_markings] right_centers = [right_markings] # Go through all rows (from bottom to top of the image). for row in range(1, window_count): if row > 0: last_value = row-1 else: last_value = 0 # Take the center (global position) of the last row and use it as entry point. # Then look window_width/2 to the left and to the right and determine the more precise # center in that area. _, corrected_center_left, _ = get_centered_sliding_window(image_warped_filtered_binary, left_centers[last_value], window_width, window_height, row) _, corrected_center_right, _ = get_centered_sliding_window(image_warped_filtered_binary, right_centers[last_value], window_width, window_height, row) if row == 0: left_centers = [] right_centers = [] left_centers.append(corrected_center_left) right_centers.append(corrected_center_right) log.debug(('LEFT_CENTERS', left_centers)) log.debug(('RIGHT_CENTERS', right_centers)) lane_final_image = image_warped_filtered_binary.copy() if left_centers[0]: lane_final_image = cv2.line(lane_final_image, (left_centers[0], 0), (left_centers[0], 300), (255, 0, 0), 5) if right_centers[0]: lane_final_image = cv2.line(lane_final_image, (right_centers[0], 0), (right_centers[0], 300), (0, 255, 0), 5) # TODO: Determine center of lane and calculate degrees to reach this center. y_eval = 0 degrees = y_eval * 180 / scipy.pi steering = int(round((degrees + 180) / 180 * 32768 / 2)) # Value for vjoy controller # Set the value of the vjoy joystick to the predicted steering angle if autopilot: self.controller_thread.set_angle(steering) self.statusbar.showMessage("Autopilot active") else: self.statusbar.showMessage("Autopilot inactive") # functions.set_image(main.copy(), self.image_front) functions.set_image(lane_final_image.copy(), self.image_front)
def run(self): s = Settings() d = Data(batch=True) img_id = d.get_next_fileid() self.recording = False maneuver = 0 # 0 - normal, 1 - indicator left, 2 - indicator right last_record = functions.current_milli_time() keyboard.add_hotkey('shift+w', self.record_callback, args=[d]) while RecordingThread.running: # Capture the whole game frame_raw = ImageGrab.grab(bbox=functions.get_screen_bbox()) frame = np.uint8(frame_raw) frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) main = frame[s.get_value(Settings.IMAGE_FRONT_BORDER_TOP):s. get_value(Settings.IMAGE_FRONT_BORDER_BOTTOM), s.get_value(Settings.IMAGE_FRONT_BORDER_LEFT):s. get_value(Settings.IMAGE_FRONT_BORDER_RIGHT)] # gray = cv2.cvtColor(main, cv2.COLOR_BGR2GRAY) # blur_gray = cv2.GaussianBlur(gray, (3, 3), 0) # edges = cv2.Canny(blur_gray, 50, 150) # dilated = cv2.dilate(edges, (3,3), iterations=2) # Resize image to save some space (height = 100px) ratio = main.shape[1] / main.shape[0] resized = cv2.resize(main, (round(ratio * 100), 100)) # cv2.imshow('cap', dilated) # cv2.imshow('resized', resized) functions.set_image(main.copy(), self.image_front) axis, throttle, speed = get_steering_throttle_speed() if axis == 0: maneuver = 0 elif axis > 0: maneuver = 1 else: maneuver = 2 if self.recording: self.statusbar.showMessage( "Recording: active | Indicator: %s" % functions.get_indicator(maneuver)) else: self.statusbar.showMessage( "Recording: inactive | Indicator: %s" % functions.get_indicator(maneuver)) # Save frame every 150ms if self.recording: cv2.imwrite("captured/%d.png" % img_id, resized) d.add_image("%d.png" % img_id, axis, speed, throttle, maneuver, self.sequence_id) img_id += 1 # at least wait 150ms wait_milli_time = functions.current_milli_time( ) - last_record - 150 if wait_milli_time < 0: time.sleep(-wait_milli_time / 1000) last_record = functions.current_milli_time() else: time.sleep(0.15) keyboard.clear_hotkey('shift+w') d.append()