def main():
for i in list(range(4))[::-1]:
print(i + 1)
time.sleep(1)
paused = False
while (True):
if not paused:
# 800x600 windowed mode
screen = grab_screen(region=(0, 40, 800, 600))
last_time = time.time()
screen = cv2.cvtColor(screen, cv2.COLOR_BGR2GRAY)
screen = cv2.resize(screen, (160, 120))
# resize to something a bit more acceptable for a CNN
keys = key_check()
output = keys_to_output(keys)
training_data.append([screen, output])
if len(training_data) % 1000 == 0:
print(len(training_data))
np.save(file_name, training_data, allow_pickle=True)
keys = key_check()
if 'T' in keys:
if paused:
paused = False
print('unpaused!')
time.sleep(1)
else:
print('Pausing!')
paused = True
time.sleep(1)
def main():
for i in list(range(2))[::-1]:
print(i+1)
time.sleep(1)
paused = False
while True:
if not paused:
screen = grab_screen(region=(0,40,800,640))
last_time = time.time()
screen = cv2.cvtColor(screen, cv2.COLOR_BGR2GRAY)
screen = cv2.resize(screen, (160,120))
# resize for something acceptable for CNN
keys = key_check()
output = keys_to_output(keys)
training_data.append([screen, output])
if len(training_data) % 100 == 0:
print(f"{len(training_data)} Reached!")
if len(training_data) % 1000 == 0:
print(len(training_data))
np.save(file_name, training_data)
keys = key_check()
if 'T' in keys:
if paused:
paused = False
print('unpaused!')
time.sleep(1)
else:
print('Pausing!')
paused = True
time.sleep(1)
#print('Frame took {} seconds'.format(time.time()-last_time))
last_time = time.time()
# #new_screen,original_image, m1, m2 = process_img(screen)
# new_screen = annotate_image(screen)
# cv2.imshow('Final view', new_screen)
# #cv2.imshow('Module View',cv2.cvtColor(original_image, cv2.COLOR_BGR2RGB))
#cv2.imshow('window',cv2.cvtColor(screen, cv2.COLOR_BGR2RGB))
if cv2.waitKey(25) & 0xFF == ord('q'):
cv2.destroyAllWindows()
break
def main():
#countdown
for i in list(range(4))[::-1]:
print(i + 1)
time.sleep(1)
vertices = np.array(
[[5, 600], [5, 300], [250, 180], [550, 180], [795, 300], [795, 600]],
np.int32)
while True:
#record screen and convert
screen = grab_screen(region=(17, 33, 800, 600))
screen = cv2.cvtColor(screen, cv2.COLOR_BGR2GRAY)
#roi
screen = roi(screen, [vertices])
screen = cv2.resize(screen, (120, 90))
#record key input
keys = key_check()
output = keys_to_output(keys)
#save training data
training_data.append([screen, output])
if len(training_data) % 4000 == 0:
print("Saving... Size: " + str(len(training_data)))
np.save(file_name, training_data)
if len(training_data) % 10000 == 0:
print("Backing up... Size: " + str(len(training_data)))
np.save(file_name + "-{}".format(int(time.time())), training_data)
def generateData(file_name, idx):
train_data = []
print("Starting in 5 secs")
time.sleep(5)
print("Starting!!!!!!")
last_time = time.time()
paused = True
while True:
keys = key_check()
if 'P' in keys:
paused = not paused
print("Paused: ", paused)
time.sleep(1)
if not paused:
last_time = time.time()
screen = np.array(ImageGrab.grab(bbox=(0, 40, 800, 640)))[:, :,
0:3]
screen = cv2.resize(screen, (299, 299))
screen = cv2.cvtColor(screen, cv2.COLOR_BGR2RGB)
keys_pressed = key_check()
#if 'A' in keys_pressed or 'D' in keys_pressed or 'S' in keys_pressed:
output, output2, to_add = keys_to_output(keys_pressed)
if len(last_hundred) == max_last_hundred:
last_hundred.pop(0)
last_hundred.append(to_add)
train_data.append([screen, output, output2, last_hundred.copy()])
if len(train_data) % 10 == 0:
print("length ", len(train_data), end='\r')
if len(train_data) == 500:
print("Saving")
np.save(file_name, train_data)
print("Saved to file - ", file_name)
idx += 1
train_data = []
file_name = 'train_data_{}.npy'.format(idx)
file_name = DATA_PATH + file_name
while os.path.isfile(file_name):
print(os.path.abspath(file_name))
print("File exsits for idx ", STARTING_IDX)
idx += 1
file_name = 'train_data_{}.npy'.format(idx)
file_name = DATA_PATH + file_name
def start_collecting_data(self):
while True:
screen = grab_screen()
# print('Frame took {} seconds'.format(time.time() - last_time))
# last_time = time.time()
processed_img = process_img(screen)
keys = key_check()
output = self.keys_to_output(keys)
if not self.pause:
self.training_data.append([processed_img, output])
if len(self.training_data) % 500 == 0:
print(len(self.training_data))
np.save(self.file_name, self.training_data)
def get_input():
global prev_act
keys = get_keys.key_check()
action = prev_act
if "A" in keys:
action = 3
elif "D" in keys:
action = 1
elif "W" in keys:
action = 0
elif "S" in keys:
action = 2
prev_act = action
return action
def predictor(model):
global QUIT
global PAUSED
while not QUIT:
keys = key_check()
#keys = 'W'A
if 'Q' in keys:
QUIT = True
if 'P' in keys:
PAUSED = not PAUSED
releaseAll()
print("PAUSED:", PAUSED)
time.sleep(1)
if PAUSED:
continue
x1, x2, x3 = get_input_from_screen()
x4 = torch.from_numpy(
np.array(last_hundred).astype(dtype=np.float32).reshape(
(1, 100, 5)))
#print(x1.shape,type(x1))
#cv2.imshow('image',x1.detach().numpy().reshape(299,299,3))
#cv2.imshow('map',x2.detach().numpy().reshape(64,64,1))
#cv2.imshow('speed',x3.detach().numpy().reshape(64,64,1))
#if(cv2.waitKey(25) & 0xFF == ord('q')):
# cv2.destroyAllWindows()
# break
#input("Press Enter to continue...")
prediction = model([x1, x2, x3, x4])
if not THREADING:
key_pressed = perform_action(prediction.max(1)[1].numpy()[0])
else:
key_pressed = perform_action_with_threading(
prediction.max(1)[1].numpy()[0])
last_action = [0, 0, 0, 0, 0]
last_action[prediction.max(1)[1].numpy()[0]] = 1
last_hundred.pop(0)
last_hundred.append(last_action)
print(prediction.detach().numpy()[0], "\n", "Action = ", key_pressed)
def main():
paused = True
last_time = time.time()
while True:
keys = key_check()
if not paused:
fps = round(1 / max((time.time() - last_time), 0.01))
last_time = time.time()
screen = grab_screen(region=(0, 0, 1920, 1080))
# only resize the picture, processing the picture
# while balancing the data allows to later on
# experiment with the original (only resized) training data
# 1920*0.2 und 1080*0.2
# just because i can! 1060 gtx!
processed_img = cv2.resize(screen, (384, 216))
# visualize captured stream
# put in comment for more frames
# cv2.imshow("win", processed_img)
#
# if cv2.waitKey(25) & 0xFF == ord('q'):
# cv2.destroyAllWindows()
# break
output = keys_to_output(keys)
training_data.append([processed_img, output])
print(str(fps) + "fps")
# t for pause script so you can get in position again
if 'T' in keys:
if paused:
paused = False
time.sleep(1)
else:
paused = True
time.sleep(1)
elif 'Q' in keys:
np.save(file_name, training_data)
break
def main():
last_time = time.time()
for i in list(range(4))[::-1]:
print(i + 1)
time.sleep(1)
paused = False
while (True):
if not paused:
# 800x600 windowed mode
screen = np.array(ImageGrab.grab(bbox=(0, 40, 800, 640)))
print('loop took {} seconds'.format(time.time() - last_time))
last_time = time.time()
screen = cv2.cvtColor(screen, cv2.COLOR_BGR2GRAY)
screen = cv2.resize(screen, (80, 60))
moves = list(
np.around(model.predict([screen.reshape(80, 60, 1)])[0]))
if moves == [1, 0, 0]:
left()
elif moves == [0, 1, 0]:
straight()
elif moves == [0, 0, 1]:
right()
keys = key_check()
# p pauses game and can get annoying.
if 'T' in keys:
if paused:
paused = False
time.sleep(1)
else:
paused = True
ReleaseKey(A)
ReleaseKey(W)
ReleaseKey(D)
time.sleep(1)
def main():
for i in list(range(4))[::-1]:
print(i + 1)
time.sleep(1)
last_time = time.time()
while True:
screen = np.array(ImageGrab.grab(bbox=(0, 40, 800, 640)))
screen = cv2.cvtColor(screen, cv2.COLOR_BGR2GRAY)
screen = cv2.resize(screen, (80, 60))
keys = key_check()
output = keys_to_output(keys)
training_data.append([screen, output])
print('Frame took {} seconds'.format(time.time() - last_time))
last_time = time.time()
if len(training_data) % 500 == 0:
print(len(training_data))
np.save(file_name, training_data)
def train():
global last_screen_img
global old_p_info
global last_p_info
move_model = create_move_model()
action_model = create_action_model()
move_model.compile(optimizer='adam', loss=game_loss)
action_model.compile(optimizer='adam', loss=game_loss)
paused = True
r = 0
while True:
keys = key_check()
if 'return' in keys:
if paused:
paused = False
print('Started !!!')
time.sleep(1)
else:
paused = True
print('Stopped !!!')
time.sleep(1)
continue
if paused:
time.sleep(1)
continue
elif 'Z' in keys:
print('Exit!!')
break
# step 1. get screen capture
game_img = grab_screen(game_position)
p_info = person_info(game_img)
print(p_info)
# step2. check availability
if p_info is None:
time.sleep(1)
continue
# step3. get prediction and take action
(s_width, s_height) = input_img_size
img_small = game_img.resize(input_img_size)
np_img_small = np.array(img_small).reshape((s_width * s_height, 1))
if r == 0:
move = np.random.randint(5)
action = np.random.randint(6)
else:
move = move_model.predict(x=[[np_img_small]])
action = action_model.predict(x=[[np_img_small]])
move_index = np.argmax(move)
action_index = np.argmax(action)
take_action(move_index, action_index)
# step4. get screen capture again
last_screen_img = grab_screen(game_position)
last_p_info = person_info(last_screen_img)
move_model.fit([[np_img_small]], [[np.random.random(5)]])
action_model.fit([[np_img_small]], [[np.random.random(5)]])
old_p_info = last_p_info
with detection_graph.as_default():
with tf.Session(graph=detection_graph) as sess:
# Definite input and output Tensors for detection_graph
image_tensor = detection_graph.get_tensor_by_name('image_tensor:0')
feature_vector = detection_graph.get_tensor_by_name(
"FeatureExtractor/MobilenetV1/Conv2d_13_pointwise_2_Conv2d_5_3x3_s2_128/Relu6:0")
for i in list(range(3))[::-1]:
print(i + 1)
time.sleep(1)
paused = False
while True:
if not paused:
keys = key_check()
print('keys: ' + str(keys))
if not keys:
continue
screen = grab_screen(region=None)
screen = screen[20:1000, :1910]
image_np = cv2.resize(screen, (900, 400))
# the array based representation of the image will be used later in order to prepare the
# result image with boxes and labels on it.
# Expand dimensions since the model expects images to have shape: [1, None, None, 3]
image_np_expanded = np.expand_dims(image_np, axis=0)
# Actual detection.
(rep) = sess.run(
[feature_vector],
feed_dict={image_tensor: image_np_expanded})
UPDATE_TARGET_EVERY = settings.UPDATE_TARGET_EVERY # Terminal states (end of episodes)
MODEL_NAME = settings.MODEL_NAME
NORMALIZE_BY = settings.NORMALIZE_BY
MEMORY_FRACTION = settings.MEMORY_FRACTION # 0.20
LEARNING_RATE = settings.LEARNING_RATE
# For more repetitive results
random.seed(1)
np.random.seed(1)
# SAVE / LOAD
LOAD_PREV_MODEL = settings.LOAD_PREV_MODEL
key_check()
keys = []
def extend_image(source, image):
source = list(source)
image = list(image)
source.extend(image)
source = np.array(source)
return source
class DQNAgent:
def __init__(self, num_actions, sample_state):
self.memory = deque([
np.zeros(sample_state.shape) for _ in range(settings.NUM_STACKED)
def main():
global bytes,stream
for i in list(range(4))[:: -1]:
print(i+1)
time.sleep(0.25)
last_time = time.time()
try:
print ("Streaming...")
print ("Press 'q' to exit")
with open(recordfile,'r') as getFrame:
frame = int(getFrame.read())
getFrame.close()
while True:
bytes += stream.read(1024)
a = bytes.find(b'\xff\xd8')
b = bytes.find(b'\xff\xd9')
if a != -1 and b != -1:
jpg = bytes[a:b + 2]
bytes = bytes[b + 2:]
image = cv2.imdecode(np.fromstring(jpg, dtype=np.uint8), cv2.IMREAD_COLOR)
roi = image[50:,]
if 'W' in key_check():
frame +=1
s.send("forw".encode('utf-8'))
print("moving forward")
cv2.imwrite('training_data_forward/frame{}_{}.jpg'.format(frame,'0'), roi)
elif 'D' in key_check():
frame +=1
s.send("righ".encode('utf-8'))
print("moving right")
cv2.imwrite('training_data_right/frame{}_{}.jpg'.format(frame,'1'), roi)
elif 'A' in key_check():
frame +=1
s.send("left".encode('utf-8'))
print("moving left")
cv2.imwrite('training_data_left/frame{}_{}.jpg'.format(frame, '2'), roi)
elif 'S' in key_check():
s.send("reve".encode('utf-8'))
#cv2.imwrite('training_data_forward/frame{}_{}.jpg'.format(frame,'0'), roi)
print("moving reverser")
else:
s.send('brak'.encode('utf-8'))
cv2.imshow('video', roi)
file = open(recordfile,'w')
file.write(str(frame))
file.close()
print("Frame took {}seconds".format(time.time()-last_time))
last_time = time.time()
if cv2.waitKey(1) & 0xFF == ord('q'):
break
finally:
s.close()
with detection_graph.as_default():
with tf.Session(graph=detection_graph) as sess:
# Definite input and output Tensors for detection_graph
image_tensor = detection_graph.get_tensor_by_name('image_tensor:0')
feature_vector = detection_graph.get_tensor_by_name(
"FeatureExtractor/MobilenetV1/Conv2d_13_pointwise_2_Conv2d_5_3x3_s2_128/Relu6:0")
for i in list(range(3))[::-1]:
print(i + 1)
time.sleep(1)
paused = False
while True:
if not paused:
keys = key_check()
print('keys: ' + str(keys))
if not keys:
continue
screen = grab_screen(region=None)
screen = screen[20:1000, :1910]
image_np = cv2.resize(screen, (900, 400))
# the array based representation of the image will be used later in order to prepare the
# result image with boxes and labels on it.
# Expand dimensions since the model expects images to have shape: [1, None, None, 3]
image_np_expanded = np.expand_dims(image_np, axis=0)
# Actual detection.
(rep) = sess.run(
[feature_vector],
feed_dict={image_tensor: image_np_expanded})
def get_data():
# Create image folder
try:
os.mkdir(os.path.join("data", "IMG"))
except:
pass
print("Collecting data after:")
for i in list(range(10))[::-1]:
print(i + 1)
time.sleep(1)
paused = False
print("Starting!")
# Fill csv file
# TODO: If this file doesn't exist create it and start new
# If it exists keep it and append
df = pd.DataFrame(columns=["image", "forward", "left", "right", "stop"])
while True:
if not paused:
image_name = f"center_{datetime.today().strftime('%Y_%m_%d_%H_%M_%S_%f')}.jpg"
screen = np.array(ImageGrab.grab(bbox=(0, 40, 800, 640)))
cv2.imshow("Window", cv2.cvtColor(screen, cv2.COLOR_BGR2RGB))
cv2.imwrite(os.path.join(image_folder, image_name), screen)
keys = key_check()
output = keys_to_output(keys)
# print(output)
df = df.append(
{
"image": os.path.join("IMG/", image_name),
"forward": output[0],
"left": output[1],
"right": output[2],
"stop": output[3]
},
ignore_index=True)
# print the number of images each 1000 steps
folder_len = (len([
name for name in os.listdir(image_folder)
if os.path.isfile(os.path.join(image_folder, name))
]))
if folder_len % 1000 == 0:
print(folder_len, "Reached!")
if cv2.waitKey(25) & 0xFF == ord("q"):
cv2.destroyAllWindows()
break
# print(df)
check_paused = key_check()
if "T" in check_paused:
if paused:
paused = False
print("Unpaused!")
time.sleep(1)
else:
print("Pausing")
paused = True
time.sleep(1)
df.to_csv(os.path.join(DATA_DIR, "driving_log.csv"), index=False)
def main():
for i in list(range(4))[::-1]:
print(i + 1)
time.sleep(0.25)
last_time = time.time()
try:
print("Connection from: ", client_address)
print("Streaming...")
print("Press 'q' to exit")
stream_bytes = bytes()
with open(recordfile, 'r') as getFrame:
frame = int(getFrame.read())
getFrame.close()
while True:
stream_bytes += connection.read(1024)
first = stream_bytes.find(b'\xff\xd8')
last = stream_bytes.find(b'\xff\xd9')
if first != -1 and last != -1:
jpg = stream_bytes[first:last + 2]
stream_bytes = stream_bytes[last + 2:]
image = cv2.imdecode(np.fromstring(jpg, dtype=np.uint8),
cv2.IMREAD_COLOR)
#roi = image[130:320, :]
height, width, _ = image.shape
roi_ver = [
(0, height),
((260), (75)),
(width, height),
]
cropped_image = region_of_interest(
image,
np.array([roi_ver], np.int32),
)
cropped_image = cropped_image[187:, :]
if 'W' in key_check():
frame += 1
s.send("forw".encode('utf-8'))
print("moving forward")
cv2.imwrite(
'training_data_forward/frame{}_{}.jpg'.format(
frame, '0'), cropped_image)
elif 'D' in key_check():
frame += 1
s.send("righ".encode('utf-8'))
print("moving right")
cv2.imwrite(
'training_data_right/frame{}_{}.jpg'.format(
frame, '1'), cropped_image)
elif 'A' in key_check():
frame += 1
s.send("left".encode('utf-8'))
print("moving left")
cv2.imwrite(
'training_data_left/frame{}_{}.jpg'.format(
frame, '2'), cropped_image)
elif 'S' in key_check():
s.send("reve".encode('utf-8'))
cv2.imwrite(
'training_data_right/left{}_{}.jpg'.format(
frame, '2'), cropped_image)
print("moving reverser")
else:
s.send('brak'.encode('utf-8'))
cv2.imshow('video', cropped_image)
file = open(recordfile, 'w')
file.write(str(frame))
file.close()
print("Frame took {}seconds".format(time.time() -
last_time))
last_time = time.time()
if cv2.waitKey(1) & 0xFF == ord('q'):
break
finally:
connection.close()
server_socket.close()
s.close()
def main():
#countdown
for i in list(range(4))[::-1]:
print(i + 1)
time.sleep(1)
last_time = time.time()
paused = False
vertices = np.array(
[[5, 600], [5, 300], [250, 180], [550, 180], [795, 300], [795, 600]],
np.int32)
while True:
keys = key_check()
if 'T' in keys:
if paused:
paused = False
time.sleep(1)
else:
paused = True
ReleaseKey(A)
ReleaseKey(W)
ReleaseKey(D)
time.sleep(1)
if paused:
continue
#record screen and convert
#record screen and convert
screen = grab_screen(region=(17, 33, 800, 600))
screen = cv2.cvtColor(screen, cv2.COLOR_BGR2GRAY)
#roi
screen = roi(screen, [vertices])
screen = cv2.resize(screen, (120, 90))
#cut
height, width = screen.shape
screen = screen[28:height, 1:width - 1]
#predict
prediction = model.predict([screen.reshape(WIDTH, HEIGHT, 1)])[0]
moves = list(np.around(prediction))
print(moves)
if moves == [1, 0, 0]:
left()
#print("left")
choices[0] += 1
elif moves == [0, 1, 0]:
straight()
#print("straight")
choices[1] += 1
elif moves == [0, 0, 1]:
right()
#print("right")
choices[2] += 1
elif moves == [0, 0, 0]:
nothing()
choices[3] += 1
else:
print("Something went wrong")
height, width = screen.shape
cv2.imshow('AI', cv2.resize(screen, (width * 10, height * 10)))
if cv2.waitKey(25) % 0xFF == ord('q'):
cv2.destroyAllWindows()
break
#print time the frame took
print('Frame took {} seconds'.format(time.time() - last_time))
last_time = time.time()
