def main(): for i in list(range(4))[::-1]: print(i + 1) time.sleep(1) paused = False while (True): if not paused: # 800x600 windowed mode screen = grab_screen(region=(0, 40, 800, 600)) last_time = time.time() screen = cv2.cvtColor(screen, cv2.COLOR_BGR2GRAY) screen = cv2.resize(screen, (160, 120)) # resize to something a bit more acceptable for a CNN keys = key_check() output = keys_to_output(keys) training_data.append([screen, output]) if len(training_data) % 1000 == 0: print(len(training_data)) np.save(file_name, training_data, allow_pickle=True) keys = key_check() if 'T' in keys: if paused: paused = False print('unpaused!') time.sleep(1) else: print('Pausing!') paused = True time.sleep(1)
def main(): for i in list(range(2))[::-1]: print(i+1) time.sleep(1) paused = False while True: if not paused: screen = grab_screen(region=(0,40,800,640)) last_time = time.time() screen = cv2.cvtColor(screen, cv2.COLOR_BGR2GRAY) screen = cv2.resize(screen, (160,120)) # resize for something acceptable for CNN keys = key_check() output = keys_to_output(keys) training_data.append([screen, output]) if len(training_data) % 100 == 0: print(f"{len(training_data)} Reached!") if len(training_data) % 1000 == 0: print(len(training_data)) np.save(file_name, training_data) keys = key_check() if 'T' in keys: if paused: paused = False print('unpaused!') time.sleep(1) else: print('Pausing!') paused = True time.sleep(1) #print('Frame took {} seconds'.format(time.time()-last_time)) last_time = time.time() # #new_screen,original_image, m1, m2 = process_img(screen) # new_screen = annotate_image(screen) # cv2.imshow('Final view', new_screen) # #cv2.imshow('Module View',cv2.cvtColor(original_image, cv2.COLOR_BGR2RGB)) #cv2.imshow('window',cv2.cvtColor(screen, cv2.COLOR_BGR2RGB)) if cv2.waitKey(25) & 0xFF == ord('q'): cv2.destroyAllWindows() break
def main(): #countdown for i in list(range(4))[::-1]: print(i + 1) time.sleep(1) vertices = np.array( [[5, 600], [5, 300], [250, 180], [550, 180], [795, 300], [795, 600]], np.int32) while True: #record screen and convert screen = grab_screen(region=(17, 33, 800, 600)) screen = cv2.cvtColor(screen, cv2.COLOR_BGR2GRAY) #roi screen = roi(screen, [vertices]) screen = cv2.resize(screen, (120, 90)) #record key input keys = key_check() output = keys_to_output(keys) #save training data training_data.append([screen, output]) if len(training_data) % 4000 == 0: print("Saving... Size: " + str(len(training_data))) np.save(file_name, training_data) if len(training_data) % 10000 == 0: print("Backing up... Size: " + str(len(training_data))) np.save(file_name + "-{}".format(int(time.time())), training_data)
def generateData(file_name, idx): train_data = [] print("Starting in 5 secs") time.sleep(5) print("Starting!!!!!!") last_time = time.time() paused = True while True: keys = key_check() if 'P' in keys: paused = not paused print("Paused: ", paused) time.sleep(1) if not paused: last_time = time.time() screen = np.array(ImageGrab.grab(bbox=(0, 40, 800, 640)))[:, :, 0:3] screen = cv2.resize(screen, (299, 299)) screen = cv2.cvtColor(screen, cv2.COLOR_BGR2RGB) keys_pressed = key_check() #if 'A' in keys_pressed or 'D' in keys_pressed or 'S' in keys_pressed: output, output2, to_add = keys_to_output(keys_pressed) if len(last_hundred) == max_last_hundred: last_hundred.pop(0) last_hundred.append(to_add) train_data.append([screen, output, output2, last_hundred.copy()]) if len(train_data) % 10 == 0: print("length ", len(train_data), end='\r') if len(train_data) == 500: print("Saving") np.save(file_name, train_data) print("Saved to file - ", file_name) idx += 1 train_data = [] file_name = 'train_data_{}.npy'.format(idx) file_name = DATA_PATH + file_name while os.path.isfile(file_name): print(os.path.abspath(file_name)) print("File exsits for idx ", STARTING_IDX) idx += 1 file_name = 'train_data_{}.npy'.format(idx) file_name = DATA_PATH + file_name
def start_collecting_data(self): while True: screen = grab_screen() # print('Frame took {} seconds'.format(time.time() - last_time)) # last_time = time.time() processed_img = process_img(screen) keys = key_check() output = self.keys_to_output(keys) if not self.pause: self.training_data.append([processed_img, output]) if len(self.training_data) % 500 == 0: print(len(self.training_data)) np.save(self.file_name, self.training_data)
def get_input(): global prev_act keys = get_keys.key_check() action = prev_act if "A" in keys: action = 3 elif "D" in keys: action = 1 elif "W" in keys: action = 0 elif "S" in keys: action = 2 prev_act = action return action
def predictor(model): global QUIT global PAUSED while not QUIT: keys = key_check() #keys = 'W'A if 'Q' in keys: QUIT = True if 'P' in keys: PAUSED = not PAUSED releaseAll() print("PAUSED:", PAUSED) time.sleep(1) if PAUSED: continue x1, x2, x3 = get_input_from_screen() x4 = torch.from_numpy( np.array(last_hundred).astype(dtype=np.float32).reshape( (1, 100, 5))) #print(x1.shape,type(x1)) #cv2.imshow('image',x1.detach().numpy().reshape(299,299,3)) #cv2.imshow('map',x2.detach().numpy().reshape(64,64,1)) #cv2.imshow('speed',x3.detach().numpy().reshape(64,64,1)) #if(cv2.waitKey(25) & 0xFF == ord('q')): # cv2.destroyAllWindows() # break #input("Press Enter to continue...") prediction = model([x1, x2, x3, x4]) if not THREADING: key_pressed = perform_action(prediction.max(1)[1].numpy()[0]) else: key_pressed = perform_action_with_threading( prediction.max(1)[1].numpy()[0]) last_action = [0, 0, 0, 0, 0] last_action[prediction.max(1)[1].numpy()[0]] = 1 last_hundred.pop(0) last_hundred.append(last_action) print(prediction.detach().numpy()[0], "\n", "Action = ", key_pressed)
def main(): paused = True last_time = time.time() while True: keys = key_check() if not paused: fps = round(1 / max((time.time() - last_time), 0.01)) last_time = time.time() screen = grab_screen(region=(0, 0, 1920, 1080)) # only resize the picture, processing the picture # while balancing the data allows to later on # experiment with the original (only resized) training data # 1920*0.2 und 1080*0.2 # just because i can! 1060 gtx! processed_img = cv2.resize(screen, (384, 216)) # visualize captured stream # put in comment for more frames # cv2.imshow("win", processed_img) # # if cv2.waitKey(25) & 0xFF == ord('q'): # cv2.destroyAllWindows() # break output = keys_to_output(keys) training_data.append([processed_img, output]) print(str(fps) + "fps") # t for pause script so you can get in position again if 'T' in keys: if paused: paused = False time.sleep(1) else: paused = True time.sleep(1) elif 'Q' in keys: np.save(file_name, training_data) break
def main(): last_time = time.time() for i in list(range(4))[::-1]: print(i + 1) time.sleep(1) paused = False while (True): if not paused: # 800x600 windowed mode screen = np.array(ImageGrab.grab(bbox=(0, 40, 800, 640))) print('loop took {} seconds'.format(time.time() - last_time)) last_time = time.time() screen = cv2.cvtColor(screen, cv2.COLOR_BGR2GRAY) screen = cv2.resize(screen, (80, 60)) moves = list( np.around(model.predict([screen.reshape(80, 60, 1)])[0])) if moves == [1, 0, 0]: left() elif moves == [0, 1, 0]: straight() elif moves == [0, 0, 1]: right() keys = key_check() # p pauses game and can get annoying. if 'T' in keys: if paused: paused = False time.sleep(1) else: paused = True ReleaseKey(A) ReleaseKey(W) ReleaseKey(D) time.sleep(1)
def main(): for i in list(range(4))[::-1]: print(i + 1) time.sleep(1) last_time = time.time() while True: screen = np.array(ImageGrab.grab(bbox=(0, 40, 800, 640))) screen = cv2.cvtColor(screen, cv2.COLOR_BGR2GRAY) screen = cv2.resize(screen, (80, 60)) keys = key_check() output = keys_to_output(keys) training_data.append([screen, output]) print('Frame took {} seconds'.format(time.time() - last_time)) last_time = time.time() if len(training_data) % 500 == 0: print(len(training_data)) np.save(file_name, training_data)
def train(): global last_screen_img global old_p_info global last_p_info move_model = create_move_model() action_model = create_action_model() move_model.compile(optimizer='adam', loss=game_loss) action_model.compile(optimizer='adam', loss=game_loss) paused = True r = 0 while True: keys = key_check() if 'return' in keys: if paused: paused = False print('Started !!!') time.sleep(1) else: paused = True print('Stopped !!!') time.sleep(1) continue if paused: time.sleep(1) continue elif 'Z' in keys: print('Exit!!') break # step 1. get screen capture game_img = grab_screen(game_position) p_info = person_info(game_img) print(p_info) # step2. check availability if p_info is None: time.sleep(1) continue # step3. get prediction and take action (s_width, s_height) = input_img_size img_small = game_img.resize(input_img_size) np_img_small = np.array(img_small).reshape((s_width * s_height, 1)) if r == 0: move = np.random.randint(5) action = np.random.randint(6) else: move = move_model.predict(x=[[np_img_small]]) action = action_model.predict(x=[[np_img_small]]) move_index = np.argmax(move) action_index = np.argmax(action) take_action(move_index, action_index) # step4. get screen capture again last_screen_img = grab_screen(game_position) last_p_info = person_info(last_screen_img) move_model.fit([[np_img_small]], [[np.random.random(5)]]) action_model.fit([[np_img_small]], [[np.random.random(5)]]) old_p_info = last_p_info
with detection_graph.as_default(): with tf.Session(graph=detection_graph) as sess: # Definite input and output Tensors for detection_graph image_tensor = detection_graph.get_tensor_by_name('image_tensor:0') feature_vector = detection_graph.get_tensor_by_name( "FeatureExtractor/MobilenetV1/Conv2d_13_pointwise_2_Conv2d_5_3x3_s2_128/Relu6:0") for i in list(range(3))[::-1]: print(i + 1) time.sleep(1) paused = False while True: if not paused: keys = key_check() print('keys: ' + str(keys)) if not keys: continue screen = grab_screen(region=None) screen = screen[20:1000, :1910] image_np = cv2.resize(screen, (900, 400)) # the array based representation of the image will be used later in order to prepare the # result image with boxes and labels on it. # Expand dimensions since the model expects images to have shape: [1, None, None, 3] image_np_expanded = np.expand_dims(image_np, axis=0) # Actual detection. (rep) = sess.run( [feature_vector], feed_dict={image_tensor: image_np_expanded})
UPDATE_TARGET_EVERY = settings.UPDATE_TARGET_EVERY # Terminal states (end of episodes) MODEL_NAME = settings.MODEL_NAME NORMALIZE_BY = settings.NORMALIZE_BY MEMORY_FRACTION = settings.MEMORY_FRACTION # 0.20 LEARNING_RATE = settings.LEARNING_RATE # For more repetitive results random.seed(1) np.random.seed(1) # SAVE / LOAD LOAD_PREV_MODEL = settings.LOAD_PREV_MODEL key_check() keys = [] def extend_image(source, image): source = list(source) image = list(image) source.extend(image) source = np.array(source) return source class DQNAgent: def __init__(self, num_actions, sample_state): self.memory = deque([ np.zeros(sample_state.shape) for _ in range(settings.NUM_STACKED)
def main(): global bytes,stream for i in list(range(4))[:: -1]: print(i+1) time.sleep(0.25) last_time = time.time() try: print ("Streaming...") print ("Press 'q' to exit") with open(recordfile,'r') as getFrame: frame = int(getFrame.read()) getFrame.close() while True: bytes += stream.read(1024) a = bytes.find(b'\xff\xd8') b = bytes.find(b'\xff\xd9') if a != -1 and b != -1: jpg = bytes[a:b + 2] bytes = bytes[b + 2:] image = cv2.imdecode(np.fromstring(jpg, dtype=np.uint8), cv2.IMREAD_COLOR) roi = image[50:,] if 'W' in key_check(): frame +=1 s.send("forw".encode('utf-8')) print("moving forward") cv2.imwrite('training_data_forward/frame{}_{}.jpg'.format(frame,'0'), roi) elif 'D' in key_check(): frame +=1 s.send("righ".encode('utf-8')) print("moving right") cv2.imwrite('training_data_right/frame{}_{}.jpg'.format(frame,'1'), roi) elif 'A' in key_check(): frame +=1 s.send("left".encode('utf-8')) print("moving left") cv2.imwrite('training_data_left/frame{}_{}.jpg'.format(frame, '2'), roi) elif 'S' in key_check(): s.send("reve".encode('utf-8')) #cv2.imwrite('training_data_forward/frame{}_{}.jpg'.format(frame,'0'), roi) print("moving reverser") else: s.send('brak'.encode('utf-8')) cv2.imshow('video', roi) file = open(recordfile,'w') file.write(str(frame)) file.close() print("Frame took {}seconds".format(time.time()-last_time)) last_time = time.time() if cv2.waitKey(1) & 0xFF == ord('q'): break finally: s.close()
def get_data(): # Create image folder try: os.mkdir(os.path.join("data", "IMG")) except: pass print("Collecting data after:") for i in list(range(10))[::-1]: print(i + 1) time.sleep(1) paused = False print("Starting!") # Fill csv file # TODO: If this file doesn't exist create it and start new # If it exists keep it and append df = pd.DataFrame(columns=["image", "forward", "left", "right", "stop"]) while True: if not paused: image_name = f"center_{datetime.today().strftime('%Y_%m_%d_%H_%M_%S_%f')}.jpg" screen = np.array(ImageGrab.grab(bbox=(0, 40, 800, 640))) cv2.imshow("Window", cv2.cvtColor(screen, cv2.COLOR_BGR2RGB)) cv2.imwrite(os.path.join(image_folder, image_name), screen) keys = key_check() output = keys_to_output(keys) # print(output) df = df.append( { "image": os.path.join("IMG/", image_name), "forward": output[0], "left": output[1], "right": output[2], "stop": output[3] }, ignore_index=True) # print the number of images each 1000 steps folder_len = (len([ name for name in os.listdir(image_folder) if os.path.isfile(os.path.join(image_folder, name)) ])) if folder_len % 1000 == 0: print(folder_len, "Reached!") if cv2.waitKey(25) & 0xFF == ord("q"): cv2.destroyAllWindows() break # print(df) check_paused = key_check() if "T" in check_paused: if paused: paused = False print("Unpaused!") time.sleep(1) else: print("Pausing") paused = True time.sleep(1) df.to_csv(os.path.join(DATA_DIR, "driving_log.csv"), index=False)
def main(): for i in list(range(4))[::-1]: print(i + 1) time.sleep(0.25) last_time = time.time() try: print("Connection from: ", client_address) print("Streaming...") print("Press 'q' to exit") stream_bytes = bytes() with open(recordfile, 'r') as getFrame: frame = int(getFrame.read()) getFrame.close() while True: stream_bytes += connection.read(1024) first = stream_bytes.find(b'\xff\xd8') last = stream_bytes.find(b'\xff\xd9') if first != -1 and last != -1: jpg = stream_bytes[first:last + 2] stream_bytes = stream_bytes[last + 2:] image = cv2.imdecode(np.fromstring(jpg, dtype=np.uint8), cv2.IMREAD_COLOR) #roi = image[130:320, :] height, width, _ = image.shape roi_ver = [ (0, height), ((260), (75)), (width, height), ] cropped_image = region_of_interest( image, np.array([roi_ver], np.int32), ) cropped_image = cropped_image[187:, :] if 'W' in key_check(): frame += 1 s.send("forw".encode('utf-8')) print("moving forward") cv2.imwrite( 'training_data_forward/frame{}_{}.jpg'.format( frame, '0'), cropped_image) elif 'D' in key_check(): frame += 1 s.send("righ".encode('utf-8')) print("moving right") cv2.imwrite( 'training_data_right/frame{}_{}.jpg'.format( frame, '1'), cropped_image) elif 'A' in key_check(): frame += 1 s.send("left".encode('utf-8')) print("moving left") cv2.imwrite( 'training_data_left/frame{}_{}.jpg'.format( frame, '2'), cropped_image) elif 'S' in key_check(): s.send("reve".encode('utf-8')) cv2.imwrite( 'training_data_right/left{}_{}.jpg'.format( frame, '2'), cropped_image) print("moving reverser") else: s.send('brak'.encode('utf-8')) cv2.imshow('video', cropped_image) file = open(recordfile, 'w') file.write(str(frame)) file.close() print("Frame took {}seconds".format(time.time() - last_time)) last_time = time.time() if cv2.waitKey(1) & 0xFF == ord('q'): break finally: connection.close() server_socket.close() s.close()
def main(): #countdown for i in list(range(4))[::-1]: print(i + 1) time.sleep(1) last_time = time.time() paused = False vertices = np.array( [[5, 600], [5, 300], [250, 180], [550, 180], [795, 300], [795, 600]], np.int32) while True: keys = key_check() if 'T' in keys: if paused: paused = False time.sleep(1) else: paused = True ReleaseKey(A) ReleaseKey(W) ReleaseKey(D) time.sleep(1) if paused: continue #record screen and convert #record screen and convert screen = grab_screen(region=(17, 33, 800, 600)) screen = cv2.cvtColor(screen, cv2.COLOR_BGR2GRAY) #roi screen = roi(screen, [vertices]) screen = cv2.resize(screen, (120, 90)) #cut height, width = screen.shape screen = screen[28:height, 1:width - 1] #predict prediction = model.predict([screen.reshape(WIDTH, HEIGHT, 1)])[0] moves = list(np.around(prediction)) print(moves) if moves == [1, 0, 0]: left() #print("left") choices[0] += 1 elif moves == [0, 1, 0]: straight() #print("straight") choices[1] += 1 elif moves == [0, 0, 1]: right() #print("right") choices[2] += 1 elif moves == [0, 0, 0]: nothing() choices[3] += 1 else: print("Something went wrong") height, width = screen.shape cv2.imshow('AI', cv2.resize(screen, (width * 10, height * 10))) if cv2.waitKey(25) % 0xFF == ord('q'): cv2.destroyAllWindows() break #print time the frame took print('Frame took {} seconds'.format(time.time() - last_time)) last_time = time.time()