def relaunch(self, before_relaunch=None, after_relaunch=None): clear_terminal() print("") print("Relaunching the game...") self.stop_frame_grabber() time.sleep(1) if before_relaunch is not None: before_relaunch() time.sleep(1) subprocess.call(shlex.split(f"serpent launch {self.game_name}")) self.launch(dry_run=True) self.start_frame_grabber() self.redis_client.delete(config["frame_grabber"]["redis_key"]) while self.redis_client.llen( config["frame_grabber"]["redis_key"]) == 0: time.sleep(0.1) self.window_controller.focus_window(self.window_id) if after_relaunch is not None: after_relaunch()
def modules(): import importlib exists = importlib.util.find_spec serpent_modules = { "OCR": (exists("tesserocr") or exists("pytesseract")) is not None, "GUI": exists("kivy") is not None, "ML": exists("keras") is not None and exists("tensorforce") is not None } clear_terminal() display_serpent_logo() print("") print("Installed Serpent.AI Modules:") print("") print( f"OCR => {'Yes' if serpent_modules['OCR'] else 'No; Install with `serpent setup ocr` if needed'}" ) print( f"GUI => {'Yes' if serpent_modules['GUI'] else 'No; Install with `serpent setup gui` if needed'}" ) print( f"ML => {'Yes' if serpent_modules['ML'] else 'No; Install with `serpent setup ml` if needed'}" ) print("")
def handle_collect_frames_for_context(self, game_frame, game_frame_pipeline, **kwargs): context = kwargs.get("context") or config["frame_handlers"][ "COLLECT_FRAMES_FOR_CONTEXT"]["context"] interval = kwargs.get("interval") or config["frame_handlers"][ "COLLECT_FRAMES_FOR_CONTEXT"]["interval"] screen_region = kwargs.get("screen_region") if screen_region is not None: if screen_region not in self.game.screen_regions: raise GameAgentError("Invalid game screen region...") frame_region = serpent.cv.extract_region_from_image( game_frame.frame, self.game.screen_regions[screen_region]) game_frame = GameFrame(frame_region) self.game_frames.append(game_frame) self.collected_frame_count += 1 clear_terminal() print( f"Collected Frame #{self.collected_frame_count} for Context: {context}" ) time.sleep(interval)
def handle_collect_frames(self, game_frame, **kwargs): self.game_frames.append(game_frame) self.collected_frame_count += 1 clear_terminal() print(f"Collected Frame #{self.collected_frame_count}") time.sleep( kwargs.get("interval") or self.config.get("collect_frames_interval") or 1)
def setup_handle_record(self, **kwargs): self.game_frame_buffers = list() self.input_recorder_process = None self.frame_offsets = list( range(0, (self.kwargs["frame_count"] * self.kwargs["frame_spacing"]) - 1, self.kwargs["frame_spacing"])) self._start_input_recorder() clear_terminal() print( "Start playing the game! Focus out when you are done or want to save the collected data to that point." )
def setup(module=None): clear_terminal() display_serpent_logo() print("") if module is None: setup_base() elif module == "ocr": setup_ocr() elif module == "gui": setup_gui() elif module == "ml": setup_ml() else: print(f"Invalid Setup Module: {module}")
def handle_collect_frame_regions(self, game_frame, game_frame_pipeline, **kwargs): region = kwargs.get("region") self.game_frames.append(game_frame) self.collected_frame_count += 1 clear_terminal() print( f"Collected Frame #{self.collected_frame_count} for Region: {region}" ) time.sleep( kwargs.get("interval") or self.config.get("collect_frames_interval") or 1)
def window_name(): clear_terminal() print("Open the Game manually.") input("\nPress Enter and then focus the game window...") window_controller = WindowController() time.sleep(5) focused_window_name = window_controller.get_focused_window_name() print( f"\nGame Window Detected! Please set the kwargs['window_name'] value in the Game plugin to:" ) print("\n" + focused_window_name + "\n")
def generate_game_agent_plugin(): clear_terminal() display_serpent_logo() print("") game_agent_name = input( "What is the name of the game agent? (Titleized, No Spaces i.e. AwesomeGameAgent): \n" ) if game_agent_name in [None, ""]: raise Exception("Invalid game agent name.") prepare_game_agent_plugin(game_agent_name) subprocess.call( shlex.split( f"serpent activate Serpent{game_agent_name}GameAgentPlugin"))
def update(): clear_terminal() display_serpent_logo() print("") print("Updating Serpent.AI to the latest version...") subprocess.call(shlex.split("pip install --upgrade SerpentAI")) if is_linux(): shutil.copy( os.path.join(os.path.dirname(__file__), "requirements.linux.txt"), os.path.join(os.getcwd(), "requirements.txt")) elif is_macos(): shutil.copy( os.path.join(os.path.dirname(__file__), "requirements.darwin.txt"), os.path.join(os.getcwd(), "requirements.txt")) elif is_windows(): shutil.copy( os.path.join(os.path.dirname(__file__), "requirements.win32.txt"), os.path.join(os.getcwd(), "requirements.txt")) subprocess.call(shlex.split("pip install -r requirements.txt")) import yaml with open(os.path.join(os.path.dirname(__file__), "config", "config.yml"), "r") as f: serpent_config = yaml.safe_load(f) or {} with open(os.path.join(os.getcwd(), "config", "config.yml"), "r") as f: user_config = yaml.safe_load(f) or {} config_changed = False for key, value in serpent_config.items(): if key not in user_config: user_config[key] = value config_changed = True if config_changed: with open(os.path.join(os.getcwd(), "config", "config.yml"), "w") as f: f.write(yaml.dump(user_config))
def generate_game_plugin(): clear_terminal() display_serpent_logo() print("") game_name = input( "What is the name of the game? (Titleized, No Spaces i.e. AwesomeGame): \n" ) game_platform = input( "How is the game launched? (One of: 'steam', 'executable', 'web_browser'): \n" ) if game_name in [None, ""]: raise Exception("Invalid game name.") if game_platform not in ["steam", "executable", "web_browser"]: raise Exception("Invalid game platform.") prepare_game_plugin(game_name, game_platform) subprocess.call( shlex.split(f"serpent activate Serpent{game_name}GamePlugin"))
def flush(self): clear_terminal() print("\n".join(self.lines)) self.clear()
def on_record_pause(self, **kwargs): InputRecorder.pause_input_recording() input_events = list() input_event_count = self.redis_client.llen( config["input_recorder"]["redis_key"]) for i in range(input_event_count): input_events.append( pickle.loads( self.redis_client.lpop( config["input_recorder"]["redis_key"]))) data = self._merge_frames_and_input_events(input_events) if not len(data): time.sleep(1) return None latest_game_frame_buffer = None active_keys = set() down_keys = dict() observations = dict() compute_reward = "reward_function" in self.config and self.config[ "reward_function"] in self.reward_functions reward_func = None if compute_reward: reward_func = self.reward_functions[self.config["reward_function"]] for item in data: if isinstance(item, GameFrameBuffer): latest_game_frame_buffer = item reward_score = 0 if compute_reward: reward_score = reward_func(item.frames) timestamp = item.frames[-2].timestamp observations[timestamp] = [ item, dict(), list(active_keys), list(), reward_score ] elif item["type"] == "keyboard": key_name, key_event = item["name"].split("-") if key_event == "DOWN": active_keys.add(key_name) if latest_game_frame_buffer is not None: timestamp = latest_game_frame_buffer.frames[ -2].timestamp observations[timestamp][1][key_name] = item[ "timestamp"] down_keys[key_name] = timestamp elif key_event == "UP": active_keys.remove(key_name) if key_name in down_keys: timestamp = down_keys[key_name] duration = item["timestamp"] - observations[timestamp][ 1][key_name] observations[timestamp][1][key_name] = duration del down_keys[key_name] elif item["type"] == "mouse": if latest_game_frame_buffer is not None: timestamp = latest_game_frame_buffer.frames[-2].timestamp observations[timestamp][3].append(item) print( f"Writing Recorded Input Data to 'datasets/input_recording.h5'... (0/{len(observations)})" ) with h5py.File("datasets/input_recording.h5", "a") as f: i = 0 for timestamp, observation in observations.items(): clear_terminal() print( f"Writing Recorded Input Data to 'datasets/input_recording.h5'... ({i + 1}/{len(observations)})" ) game_frame_buffer, keyboard_inputs, keyboard_inputs_active, mouse_inputs, reward_score = observation f.create_dataset(f"{timestamp}-frames", data=[ game_frame.frame_bytes for game_frame in game_frame_buffer.frames ]) f.create_dataset( f"{timestamp}-keyboard-inputs", data=[(key_name.encode("utf-8"), str(duration).encode("utf-8")) for key_name, duration in keyboard_inputs.items()]) f.create_dataset(f"{timestamp}-keyboard-inputs-active", data=[ key_name.encode("utf-8") for key_name in keyboard_inputs_active ]) filtered_mouse_inputs = list() mouse_move_index = None valid_game_window_x = range( self.game.window_geometry["x_offset"], self.game.window_geometry["x_offset"] + self.game.window_geometry["width"] + 1) valid_game_window_y = range( self.game.window_geometry["y_offset"], self.game.window_geometry["y_offset"] + self.game.window_geometry["height"] + 1) for mouse_input in mouse_inputs: if mouse_input["x"] in valid_game_window_x and mouse_input[ "y"] in valid_game_window_y: if mouse_input["name"] == "MOVE": mouse_move_index = len(filtered_mouse_inputs) filtered_mouse_inputs.append(mouse_input) mouse_input_data = list() for i, mouse_input in enumerate(filtered_mouse_inputs): if mouse_input["name"] == "MOVE" and i != mouse_move_index: continue mouse_input_data.append( (mouse_input["name"].encode("utf-8"), mouse_input["button"].encode("utf-8") if mouse_input["button"] else b"", mouse_input["direction"].encode("utf-8") if mouse_input["direction"] else b"", mouse_input["velocity"] or b"", mouse_input["x"], mouse_input["y"], mouse_input["timestamp"])) f.create_dataset(f"{timestamp}-mouse-inputs", data=mouse_input_data) f.create_dataset(f"{timestamp}-reward", data=reward_score) i += 1 self.game_frame_buffers = list() clear_terminal() print( f"Writing Frame/Input Data to 'datasets/input_recording.h5'... DONE" ) time.sleep(1)
def play(self, game_agent_class_name="GameAgent", frame_handler=None, **kwargs): if not self.is_launched: raise GameError( f"Game '{self.__class__.__name__}' is not running...") game_agent_class = offshoot.discover( "GameAgent", selection=game_agent_class_name).get(game_agent_class_name, GameAgent) if game_agent_class is None: raise GameError( "The provided Game Agent class name does not map to an existing class..." ) game_agent = game_agent_class(game=self, input_controller=InputController( game=self, backend=self.input_controller), **kwargs) # Look if we need to auto-append PNG to frame transformation pipeline based on given frame_handler png_frame_handlers = ["RECORD"] if frame_handler in png_frame_handlers and self.frame_transformation_pipeline_string is not None: if not self.frame_transformation_pipeline_string.endswith("|PNG"): self.frame_transformation_pipeline_string += "|PNG" self.start_frame_grabber() self.redis_client.delete(config["frame_grabber"]["redis_key"]) while self.redis_client.llen( config["frame_grabber"]["redis_key"]) == 0: time.sleep(0.1) self.window_controller.focus_window(self.window_id) frame_type = "FULL" pipeline_frame_handlers = [ "COLLECT_FRAMES", "COLLECT_FRAME_REGIONS", "COLLECT_FRAMES_FOR_CONTEXT", "RECORD" ] if frame_handler in pipeline_frame_handlers and self.frame_transformation_pipeline_string is not None: frame_type = "PIPELINE" # Override FPS Config? if frame_handler == "RECORD": self.game_frame_limiter = GameFrameLimiter(fps=10) try: while True: self.game_frame_limiter.start() game_frame = self.grab_latest_frame(frame_type=frame_type) try: if self.is_focused: game_agent.on_game_frame(game_frame, frame_handler=frame_handler, **kwargs) else: clear_terminal() print("PAUSED\n") game_agent.on_pause(frame_handler=frame_handler, **kwargs) time.sleep(1) except Exception as e: raise e # print(e) # time.sleep(0.1) self.game_frame_limiter.stop_and_delay() finally: self.stop_frame_grabber()