def _load_settings(self):
settings = Settings()
# Screen data
image_front_border_left = settings.get_value(settings.IMAGE_FRONT_BORDER_LEFT)
image_front_border_right = settings.get_value(settings.IMAGE_FRONT_BORDER_RIGHT)
image_front_border_top = settings.get_value(settings.IMAGE_FRONT_BORDER_TOP)
image_front_border_bottom = settings.get_value(settings.IMAGE_FRONT_BORDER_BOTTOM)
# Controller binding
vjoy = settings.get_value(settings.VJOY_DEVICE)
autopilot = settings.get_value(settings.AUTOPILOT)
left_indicator = settings.get_value(settings.LEFT_INDICATOR)
right_indicator = settings.get_value(settings.RIGHT_INDICATOR)
steering_axis = settings.get_value(settings.STEERING_AXIS)
throttle_axis = settings.get_value(settings.THROTTLE_AXIS)
self.fill_screen_list()
self.fill_device_list()
self.select_screen()
# Set scrollbar values
if image_front_border_right is not None:
self.ui.slider_right.setValue(int(image_front_border_right))
else:
self.ui.slider_right.setValue(self.ui.slider_right.maximum())
if image_front_border_left is not None:
self.ui.slider_left.setValue(int(image_front_border_left))
else:
self.ui.slider_left.setValue(self.ui.slider_left.minimum())
if image_front_border_bottom is not None:
self.ui.slider_bottom.setValue(int(image_front_border_bottom))
else:
self.ui.slider_bottom.setValue(self.ui.slider_bottom.maximum())
if image_front_border_top is not None:
self.ui.slider_top.setValue(int(image_front_border_top))
else:
self.ui.slider_top.setValue(self.ui.slider_top.minimum())
# Display key bindings
if vjoy is not None:
self.ui.e_vjoy.setText(str(vjoy))
if autopilot is not None:
self.ui.e_autopilot.setText(str(autopilot))
if left_indicator is not None:
self.ui.e_leftIndicator.setText(str(left_indicator))
if right_indicator is not None:
self.ui.e_rightIndicator.setText(str(right_indicator))
if steering_axis is not None:
self.ui.e_steering.setText(str(steering_axis))
if throttle_axis is not None:
self.ui.e_throttle.setText(str(throttle_axis))
self.fill_screen_cap()
def run(self):
# Settings instance
s = Settings()
# State of autopilot
self.autopilot = False
img_wheel = cv2.imread('steering_wheel_image.jpg', 0)
img_wheel = cv2.cvtColor(img_wheel, cv2.COLOR_BGR2RGB)
functions.set_image(img_wheel.copy(), self.steering_wheel_ui)
rows, cols, _ = img_wheel.shape
keyboard.add_hotkey('shift+w', self.hotkey_callback)
start_time = functions.current_milli_time()
while AutopilotThread.running:
self.controller_thread.set_autopilot(self.autopilot)
# Get frame of game
frame_raw = ImageGrab.grab(bbox=functions.get_screen_bbox())
frame = np.uint8(frame_raw)
frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
# Relevant image region for steering angle prediction
main = frame[s.get_value(Settings.IMAGE_FRONT_BORDER_TOP):s.
get_value(Settings.IMAGE_FRONT_BORDER_BOTTOM),
s.get_value(Settings.IMAGE_FRONT_BORDER_LEFT):s.
get_value(Settings.IMAGE_FRONT_BORDER_RIGHT)]
# Resize the image to the size of the neural network input layer
image = scipy.misc.imresize(main, [66, 200]) / 255.0
# Let the neural network predict the new steering angle
y_eval = model.y.eval(session=self.sess,
feed_dict={
model.x: [image],
model.keep_prob: 1.0
})[0][0]
degrees = y_eval * 180 / scipy.pi
steering = int(round((degrees + 180) / 180 * 32768 /
2)) # Value for vjoy controller
# Set the value of the vjoy joystick to the predicted steering angle
if self.autopilot:
self.controller_thread.set_angle(steering)
self.statusbar.showMessage("Autopilot active, steering: " +
str(steering))
else:
self.statusbar.showMessage("Autopilot inactive")
M = cv2.getRotationMatrix2D((cols / 2, rows / 2), -degrees, 1)
dst = cv2.warpAffine(img_wheel, M, (cols, rows))
functions.set_image(dst.copy(), self.steering_wheel_ui)
functions.set_image(main.copy(), self.image_front_ui)
wait_time = functions.current_milli_time() - start_time - 40
if wait_time < 0:
time.sleep(-wait_time / 1000)
start_time = functions.current_milli_time()
keyboard.clear_hotkey('shift+w')
self.controller_thread.set_autopilot(False)
def run(self):
# Settings instance
s = Settings()
# State of autopilot
autopilot = False
# Previous state of the autopilot button
autopilot_button_prev = 0
# Previous value of steering (gamepad)
manual_steering_prev = 0
img_wheel = cv2.imread('steering_wheel_image.jpg', 0)
rows, cols = img_wheel.shape
while AutopilotThread.running:
pygame.event.pump()
# Button to activate/deactivate autopilot
autopilot_button_act = self.joystick.get_button(self.b_autopilot)
# Button was pressed
if autopilot_button_act != autopilot_button_prev and autopilot_button_act == 1:
autopilot = not autopilot
#if autopilot and settings.AUTOPILOT_SOUND_ACTIVATE:
# autopilot_engage.play()
autopilot_button_prev = autopilot_button_act
# Read the steering value of joystick
axis = round(
(self.joystick.get_axis(self.steering_axis) + 1) * 32768 / 2)
# Interrupt autopilot if manual steering was detected
if abs(manual_steering_prev - axis) > 500 and autopilot:
img_id = Data().get_next_fileid()
sequence_id = Data().add_sequence(country=Settings().get_value(
Settings.COUNTRY_DEFAULT),
note="correction")
self.controller_thread.set_autopilot(False)
self.statusbar.showMessage("Autopilot inactive")
# TODO: Deactivate this feature in settings
# TODO: Amount of images to save in settings
# Save the next 3 images
for i in range(3):
# Get frame of game
frame_raw = ImageGrab.grab(
bbox=functions.get_screen_bbox())
frame = np.uint8(frame_raw)
frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
# Relevant image region for steering angle prediction
main = frame[
s.get_value(Settings.IMAGE_FRONT_BORDER_TOP):s.
get_value(Settings.IMAGE_FRONT_BORDER_BOTTOM),
s.get_value(Settings.IMAGE_FRONT_BORDER_LEFT):s.
get_value(Settings.IMAGE_FRONT_BORDER_RIGHT)]
# Resize image to save some space (height = 100px)
ratio = main.shape[1] / main.shape[0]
resized = cv2.resize(main, (round(ratio * 100), 100))
axis = self.joystick.get_axis(
s.get_value(Settings.STEERING_AXIS)) * 180
cv2.imwrite("captured/%d.png" % img_id, resized)
Data().add_image("%d.png" % img_id, axis, 0, 0, 0,
sequence_id)
img_id += 1
time.sleep(0.150)
autopilot = False
manual_steering_prev = axis
self.controller_thread.set_autopilot(autopilot)
# Get frame of game
frame_raw = ImageGrab.grab(bbox=functions.get_screen_bbox())
frame = np.uint8(frame_raw)
frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
# Relevant image region for steering angle prediction
main = frame[s.get_value(Settings.IMAGE_FRONT_BORDER_TOP):s.
get_value(Settings.IMAGE_FRONT_BORDER_BOTTOM),
s.get_value(Settings.IMAGE_FRONT_BORDER_LEFT):s.
get_value(Settings.IMAGE_FRONT_BORDER_RIGHT)]
# Resize the image to the size of the neural network input layer
image = scipy.misc.imresize(main, [66, 200]) / 255.0
# Let the neural network predict the new steering angle
y_eval = model.y.eval(session=self.sess,
feed_dict={
model.x: [image],
model.keep_prob: 1.0
})[0][0]
degrees = y_eval * 180 / scipy.pi
steering = int(round((degrees + 180) / 180 * 32768 /
2)) # Value for vjoy controller
# Set the value of the vjoy joystick to the predicted steering angle
if autopilot:
self.controller_thread.set_angle(steering)
self.statusbar.showMessage("Autopilot active")
else:
self.statusbar.showMessage("Autopilot inactive")
# TODO: Show steering wheel in GUI
M = cv2.getRotationMatrix2D((cols / 2, rows / 2), -degrees, 1)
dst = cv2.warpAffine(img_wheel, M, (cols, rows))
# functions.set_image(dst.copy(), self.steering_wheel)
functions.set_image(main.copy(), self.image_front)
self.controller_thread.set_autopilot(False)
class DrivingData(object):
def __init__(self):
self.settings = Settings()
self.data = Data()
country_string = self.settings.get_value(Settings.COUNTRIES_MODEL)
countries = country_string.split(",")
self.xs = []
self.ys = []
# points to the end of the last batch
self.train_batch_pointer = 0
self.val_batch_pointer = 0
# Get all images
self.image_list = []
for country in countries:
self.image_list += self.data.get_image_list_filter(country=country,
maneuver=0)
for image in self.image_list:
self.steering_deg = float(image[2]) * scipy.pi / 180
# higher steering angles are rare, so add four times
# if abs(steering_deg) > 40:
# for i in range(int(steering_deg/10-2)*4):
# xs.append("../captured/" + line.split()[0])
# ys.append(steering_deg)
self.xs.append(os.path.join("captured/", image[1]))
# the paper by Nvidia uses the inverse of the turning radius,
# but steering wheel angle is proportional to the inverse of turning radius
# so the steering wheel angle in radians is used as the output
self.ys.append(self.steering_deg)
# get number of images
self.num_images = len(self.xs)
# shuffle list of images
self.c = list(zip(self.xs, self.ys))
random.shuffle(self.c)
self.xs, self.ys = zip(*self.c)
# Training data
self.train_xs = self.xs[:int(len(self.xs) * 0.8)]
self.train_ys = self.ys[:int(len(self.xs) * 0.8)]
# Validation data
self.val_xs = self.xs[-int(len(self.xs) * 0.2):]
self.val_ys = self.ys[-int(len(self.xs) * 0.2):]
self.num_train_images = len(self.train_xs)
self.num_val_images = len(self.val_xs)
print("Total data:", len(self.xs), self.num_images)
print("Training data:", len(self.train_xs))
print("Validation data:", len(self.val_xs))
def LoadTrainBatch(self, batch_size):
x_out = []
y_out = []
for i in range(0, batch_size):
x_out.append(
scipy.misc.imresize(
scipy.misc.imread(self.train_xs[
(self.train_batch_pointer + i) %
self.num_train_images])[-150:], [66, 200]) / 255.0)
y_out.append([
self.train_ys[(self.train_batch_pointer + i) %
self.num_train_images]
])
self.train_batch_pointer += batch_size
return x_out, y_out
def LoadValBatch(self, batch_size):
x_out = []
y_out = []
for i in range(0, batch_size):
x_out.append(
scipy.misc.imresize(
scipy.misc.imread(
self.val_xs[(self.val_batch_pointer + i) %
self.num_val_images])[-150:], [66, 200]) /
255.0)
y_out.append([
self.val_ys[(self.val_batch_pointer + i) % self.num_val_images]
])
self.val_batch_pointer += batch_size
return x_out, y_out
import sys
import os
from PyQt5.QtWidgets import QApplication, QMainWindow
print("Loading...")
from UI.main import MainUI
from database import Settings
dbs = Settings()
# MIGRATE
if not dbs.get_value("migrated"):
print("Migrating data. This may take a while...")
dbs.set_value(dbs.COUNTRY_DEFAULT, "DE")
dbs.set_value(dbs.COUNTRIES_MODEL, "DE")
if os.path.exists("captured/data.txt") and os.path.exists(
"captured/sequence.txt"):
import migrate
migrate.migrate()
if os.path.exists("settings.py"):
import settings as s
dbs.set_value(dbs.CONTROLLER, s.JOYSTICK)
dbs.set_value(dbs.VJOY_DEVICE, s.VJOY_DEVICE)
dbs.set_value(dbs.AUTOPILOT_SOUND_ACTIVATE, s.AUTOPILOT_SOUND_ACTIVATE)
dbs.set_value(dbs.ADAPTIVE_STEERING, s.ADAPTIVE_STEERING)
dbs.set_value(dbs.AUTOPILOT, s.AUTOPILOT_BUTTON)
dbs.set_value(dbs.STEERING_AXIS, s.STEERING_AXIS)
dbs.set_value(dbs.THROTTLE_AXIS, s.THROTTLE_AXIS)
dbs.set_value(dbs.LEFT_INDICATOR, s.INDICATOR_LEFT_BUTTON)
def run(self):
s = Settings()
d = Data(batch=True)
img_id = d.get_next_fileid()
recording = False
recording_button_prev = 0
maneuver = 0 # 0 - normal, 1 - indicator left, 2 - indicator right
indicator_left = False
indicator_left_prev = 0
indicator_right = False
indicator_right_prev = 0
last_record = 0
while RecordingThread.running:
pygame.event.pump()
recording_button_act = self.joystick.get_button(s.get_value(Settings.AUTOPILOT))
if recording_button_act != recording_button_prev and recording_button_act == 1:
recording = not recording
if recording: # started recording
sequence_id = d.add_sequence()
else: # stopped recording
self.fill_sequence_list()
recording_button_prev = recording_button_act
indicator_left_act = self.joystick.get_button(s.get_value(Settings.LEFT_INDICATOR))
if indicator_left_act != indicator_left_prev and indicator_left_act == 1:
indicator_left = not indicator_left
# Switch indicator
if indicator_left and indicator_right:
indicator_right = False
indicator_left_prev = indicator_left_act
indicator_right_act = self.joystick.get_button(s.get_value(Settings.RIGHT_INDICATOR))
if indicator_right_act != indicator_right_prev and indicator_right_act == 1:
indicator_right = not indicator_right
# Switch indicator
if indicator_right and indicator_left:
indicator_left = False
indicator_right_prev = indicator_right_act
if indicator_left:
maneuver = 1
elif indicator_right:
maneuver = 2
else:
maneuver = 0
if recording:
self.statusbar.showMessage("Recording: active | Indicator: %s" % functions.get_indicator(maneuver))
else:
self.statusbar.showMessage("Recording: inactive | Indicator: %s" % functions.get_indicator(maneuver))
# Capture the whole game
frame_raw = ImageGrab.grab(bbox=functions.get_screen_bbox())
frame = np.uint8(frame_raw)
frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
main = frame[s.get_value(Settings.IMAGE_FRONT_BORDER_TOP):s.get_value(Settings.IMAGE_FRONT_BORDER_BOTTOM),
s.get_value(Settings.IMAGE_FRONT_BORDER_LEFT): s.get_value(Settings.IMAGE_FRONT_BORDER_RIGHT)]
# gray = cv2.cvtColor(main, cv2.COLOR_BGR2GRAY)
# blur_gray = cv2.GaussianBlur(gray, (3, 3), 0)
# edges = cv2.Canny(blur_gray, 50, 150)
# dilated = cv2.dilate(edges, (3,3), iterations=2)
# Resize image to save some space (height = 100px)
ratio = main.shape[1] / main.shape[0]
resized = cv2.resize(main, (round(ratio * 100), 100))
# cv2.imshow('cap', dilated)
# cv2.imshow('resized', resized)
functions.set_image(main.copy(), self.image_front)
axis = self.joystick.get_axis(s.get_value(Settings.STEERING_AXIS)) * 180 # -180 to 180 "degrees"
throttle = self.joystick.get_axis(s.get_value(Settings.THROTTLE_AXIS)) * 100 # -100=full throttle, 100=full brake
speed = speed_detection.get_speed(frame)
# Save frame every 150ms
if recording and (functions.current_milli_time() - last_record) >= 150:
last_record = functions.current_milli_time()
cv2.imwrite("captured/%d.png" % img_id, resized)
d.add_image("%d.png" % img_id, axis, speed, throttle, maneuver, sequence_id)
img_id += 1
d.append()
def run(self):
# Settings instance
s = Settings()
# State of autopilot
autopilot = False
# Previous state of the autopilot button
autopilot_button_prev = 0
# Previous value of steering (gamepad)
manual_steering_prev = 0
while AutopilotThread.running:
pygame.event.pump()
# Button to activate/deactivate autopilot
autopilot_button_act = self.joystick.get_button(self.b_autopilot)
# Button was pressed
if autopilot_button_act != autopilot_button_prev and autopilot_button_act == 1:
autopilot = not autopilot
# if autopilot and settings.AUTOPILOT_SOUND_ACTIVATE:
# autopilot_engage.play()
autopilot_button_prev = autopilot_button_act
# Read the steering value of joystick
axis = round((self.joystick.get_axis(self.steering_axis) + 1) * 32768 / 2)
# Interrupt autopilot if manual steering was detected
if abs(manual_steering_prev - axis) > 1000 and autopilot:
self.controller_thread.set_autopilot(False)
autopilot = False
manual_steering_prev = axis
self.controller_thread.set_autopilot(autopilot)
# Get frame of game
frame_raw = ImageGrab.grab(bbox=functions.get_screen_bbox())
frame = np.uint8(frame_raw)
frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
# Relevant image region for steering angle prediction
main = frame[s.get_value(Settings.IMAGE_FRONT_BORDER_TOP):s.get_value(Settings.IMAGE_FRONT_BORDER_BOTTOM),
s.get_value(Settings.IMAGE_FRONT_BORDER_LEFT):s.get_value(Settings.IMAGE_FRONT_BORDER_RIGHT)]
# Resize the image to the size of the neural network input layer
image = scipy.misc.imresize(main, [66, 200]) / 255.0
### Detect lane and steer ###
# Do a perspective transformation of the lane.
M, Minv = get_perspective_transform_matrix(main, 1, 0.2, 0.4, 0, [300, 300])
image_warped = cv2.warpPerspective(main.copy(), M, (300, 300), flags=cv2.INTER_LINEAR)
# Filter lane markings.
lower_white = np.array([180, 180, 180])
upper_white = np.array([255, 255, 255])
mask = cv2.inRange(image_warped, lower_white, upper_white)
image_warped_filtered = cv2.bitwise_and(image_warped, image_warped, mask=mask)
_, image_warped_filtered_binary = cv2.threshold(image_warped_filtered, 1, 255, cv2.THRESH_BINARY)
# # Find position of left and right markings.
# histogram = generate_column_histogram(image_warped_filtered_binary)
# left_markings = histogram.index(max(histogram[:150]))
# right_markings = histogram.index(max(histogram[150:]))
# log.debug((left_markings, right_markings))
window_width = 75
window_height = 50
# First half (left markings)
column_count = int(image_warped_filtered_binary.shape[1]/window_width)
left_prediction = []
right_prediction = []
for column in range(0, int(column_count/2)):
left_predicted_center = int(window_width/2 + column*window_width)
left_prediction.append(get_centered_sliding_window(image_warped_filtered_binary,
left_predicted_center,
window_width, window_height))
right_predicted_center = int(window_width / 2 + (column + column_count/2) * window_width)
right_prediction.append(get_centered_sliding_window(image_warped_filtered_binary,
right_predicted_center,
window_width, window_height))
# Select the sector with the highest maximum.
left_markings_histogram_max = [x[2] for x in left_prediction]
left_markings_center = [x[1] for x in left_prediction]
left_markings = left_markings_center[left_markings_histogram_max.index(max(left_markings_histogram_max))]
right_markings_histogram_max = [x[2] for x in right_prediction]
right_markings_center = [x[1] for x in right_prediction]
right_markings = right_markings_center[right_markings_histogram_max.index(max(right_markings_histogram_max))]
log.debug(('LEFT', left_markings_center, left_markings_histogram_max))
log.debug(('RIGHT', right_markings_center, right_markings_histogram_max))
log.debug(('CHOSE', left_markings, right_markings))
left_centers = [None]
right_centers = [None]
if left_markings > 0 and right_markings > 0:
# Apply sliding window technique.
window_count = int(image_warped_filtered_binary.shape[0]/window_height)
left_centers = [left_markings]
right_centers = [right_markings]
# Go through all rows (from bottom to top of the image).
for row in range(1, window_count):
if row > 0:
last_value = row-1
else:
last_value = 0
# Take the center (global position) of the last row and use it as entry point.
# Then look window_width/2 to the left and to the right and determine the more precise
# center in that area.
_, corrected_center_left, _ = get_centered_sliding_window(image_warped_filtered_binary,
left_centers[last_value], window_width,
window_height, row)
_, corrected_center_right, _ = get_centered_sliding_window(image_warped_filtered_binary,
right_centers[last_value], window_width,
window_height, row)
if row == 0:
left_centers = []
right_centers = []
left_centers.append(corrected_center_left)
right_centers.append(corrected_center_right)
log.debug(('LEFT_CENTERS', left_centers))
log.debug(('RIGHT_CENTERS', right_centers))
lane_final_image = image_warped_filtered_binary.copy()
if left_centers[0]:
lane_final_image = cv2.line(lane_final_image, (left_centers[0], 0), (left_centers[0], 300), (255, 0, 0), 5)
if right_centers[0]:
lane_final_image = cv2.line(lane_final_image, (right_centers[0], 0), (right_centers[0], 300), (0, 255, 0), 5)
# TODO: Determine center of lane and calculate degrees to reach this center.
y_eval = 0
degrees = y_eval * 180 / scipy.pi
steering = int(round((degrees + 180) / 180 * 32768 / 2)) # Value for vjoy controller
# Set the value of the vjoy joystick to the predicted steering angle
if autopilot:
self.controller_thread.set_angle(steering)
self.statusbar.showMessage("Autopilot active")
else:
self.statusbar.showMessage("Autopilot inactive")
# functions.set_image(main.copy(), self.image_front)
functions.set_image(lane_final_image.copy(), self.image_front)
def run(self):
s = Settings()
d = Data(batch=True)
img_id = d.get_next_fileid()
self.recording = False
maneuver = 0 # 0 - normal, 1 - indicator left, 2 - indicator right
last_record = functions.current_milli_time()
keyboard.add_hotkey('shift+w', self.record_callback, args=[d])
while RecordingThread.running:
# Capture the whole game
frame_raw = ImageGrab.grab(bbox=functions.get_screen_bbox())
frame = np.uint8(frame_raw)
frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
main = frame[s.get_value(Settings.IMAGE_FRONT_BORDER_TOP):s.
get_value(Settings.IMAGE_FRONT_BORDER_BOTTOM),
s.get_value(Settings.IMAGE_FRONT_BORDER_LEFT):s.
get_value(Settings.IMAGE_FRONT_BORDER_RIGHT)]
# gray = cv2.cvtColor(main, cv2.COLOR_BGR2GRAY)
# blur_gray = cv2.GaussianBlur(gray, (3, 3), 0)
# edges = cv2.Canny(blur_gray, 50, 150)
# dilated = cv2.dilate(edges, (3,3), iterations=2)
# Resize image to save some space (height = 100px)
ratio = main.shape[1] / main.shape[0]
resized = cv2.resize(main, (round(ratio * 100), 100))
# cv2.imshow('cap', dilated)
# cv2.imshow('resized', resized)
functions.set_image(main.copy(), self.image_front)
axis, throttle, speed = get_steering_throttle_speed()
if axis == 0:
maneuver = 0
elif axis > 0:
maneuver = 1
else:
maneuver = 2
if self.recording:
self.statusbar.showMessage(
"Recording: active | Indicator: %s" %
functions.get_indicator(maneuver))
else:
self.statusbar.showMessage(
"Recording: inactive | Indicator: %s" %
functions.get_indicator(maneuver))
# Save frame every 150ms
if self.recording:
cv2.imwrite("captured/%d.png" % img_id, resized)
d.add_image("%d.png" % img_id, axis, speed, throttle, maneuver,
self.sequence_id)
img_id += 1
# at least wait 150ms
wait_milli_time = functions.current_milli_time(
) - last_record - 150
if wait_milli_time < 0:
time.sleep(-wait_milli_time / 1000)
last_record = functions.current_milli_time()
else:
time.sleep(0.15)
keyboard.clear_hotkey('shift+w')
d.append()
