def relaunch(self, before_relaunch=None, after_relaunch=None):
clear_terminal()
print("")
print("Relaunching the game...")
self.stop_frame_grabber()
time.sleep(1)
if before_relaunch is not None:
before_relaunch()
time.sleep(1)
subprocess.call(shlex.split(f"serpent launch {self.game_name}"))
self.launch(dry_run=True)
self.start_frame_grabber()
self.redis_client.delete(config["frame_grabber"]["redis_key"])
while self.redis_client.llen(
config["frame_grabber"]["redis_key"]) == 0:
time.sleep(0.1)
self.window_controller.focus_window(self.window_id)
if after_relaunch is not None:
after_relaunch()
def modules():
import importlib
exists = importlib.util.find_spec
serpent_modules = {
"OCR": (exists("tesserocr") or exists("pytesseract")) is not None,
"GUI": exists("kivy") is not None,
"ML": exists("keras") is not None and exists("tensorforce") is not None
}
clear_terminal()
display_serpent_logo()
print("")
print("Installed Serpent.AI Modules:")
print("")
print(
f"OCR => {'Yes' if serpent_modules['OCR'] else 'No; Install with `serpent setup ocr` if needed'}"
)
print(
f"GUI => {'Yes' if serpent_modules['GUI'] else 'No; Install with `serpent setup gui` if needed'}"
)
print(
f"ML => {'Yes' if serpent_modules['ML'] else 'No; Install with `serpent setup ml` if needed'}"
)
print("")
def handle_collect_frames_for_context(self, game_frame,
game_frame_pipeline, **kwargs):
context = kwargs.get("context") or config["frame_handlers"][
"COLLECT_FRAMES_FOR_CONTEXT"]["context"]
interval = kwargs.get("interval") or config["frame_handlers"][
"COLLECT_FRAMES_FOR_CONTEXT"]["interval"]
screen_region = kwargs.get("screen_region")
if screen_region is not None:
if screen_region not in self.game.screen_regions:
raise GameAgentError("Invalid game screen region...")
frame_region = serpent.cv.extract_region_from_image(
game_frame.frame, self.game.screen_regions[screen_region])
game_frame = GameFrame(frame_region)
self.game_frames.append(game_frame)
self.collected_frame_count += 1
clear_terminal()
print(
f"Collected Frame #{self.collected_frame_count} for Context: {context}"
)
time.sleep(interval)
def handle_collect_frames(self, game_frame, **kwargs):
self.game_frames.append(game_frame)
self.collected_frame_count += 1
clear_terminal()
print(f"Collected Frame #{self.collected_frame_count}")
time.sleep(
kwargs.get("interval")
or self.config.get("collect_frames_interval") or 1)
def setup_handle_record(self, **kwargs):
self.game_frame_buffers = list()
self.input_recorder_process = None
self.frame_offsets = list(
range(0,
(self.kwargs["frame_count"] * self.kwargs["frame_spacing"]) -
1, self.kwargs["frame_spacing"]))
self._start_input_recorder()
clear_terminal()
print(
"Start playing the game! Focus out when you are done or want to save the collected data to that point."
)
def setup(module=None):
clear_terminal()
display_serpent_logo()
print("")
if module is None:
setup_base()
elif module == "ocr":
setup_ocr()
elif module == "gui":
setup_gui()
elif module == "ml":
setup_ml()
else:
print(f"Invalid Setup Module: {module}")
def handle_collect_frame_regions(self, game_frame, game_frame_pipeline,
**kwargs):
region = kwargs.get("region")
self.game_frames.append(game_frame)
self.collected_frame_count += 1
clear_terminal()
print(
f"Collected Frame #{self.collected_frame_count} for Region: {region}"
)
time.sleep(
kwargs.get("interval")
or self.config.get("collect_frames_interval") or 1)
def window_name():
clear_terminal()
print("Open the Game manually.")
input("\nPress Enter and then focus the game window...")
window_controller = WindowController()
time.sleep(5)
focused_window_name = window_controller.get_focused_window_name()
print(
f"\nGame Window Detected! Please set the kwargs['window_name'] value in the Game plugin to:"
)
print("\n" + focused_window_name + "\n")
def generate_game_agent_plugin():
clear_terminal()
display_serpent_logo()
print("")
game_agent_name = input(
"What is the name of the game agent? (Titleized, No Spaces i.e. AwesomeGameAgent): \n"
)
if game_agent_name in [None, ""]:
raise Exception("Invalid game agent name.")
prepare_game_agent_plugin(game_agent_name)
subprocess.call(
shlex.split(
f"serpent activate Serpent{game_agent_name}GameAgentPlugin"))
def update():
clear_terminal()
display_serpent_logo()
print("")
print("Updating Serpent.AI to the latest version...")
subprocess.call(shlex.split("pip install --upgrade SerpentAI"))
if is_linux():
shutil.copy(
os.path.join(os.path.dirname(__file__), "requirements.linux.txt"),
os.path.join(os.getcwd(), "requirements.txt"))
elif is_macos():
shutil.copy(
os.path.join(os.path.dirname(__file__), "requirements.darwin.txt"),
os.path.join(os.getcwd(), "requirements.txt"))
elif is_windows():
shutil.copy(
os.path.join(os.path.dirname(__file__), "requirements.win32.txt"),
os.path.join(os.getcwd(), "requirements.txt"))
subprocess.call(shlex.split("pip install -r requirements.txt"))
import yaml
with open(os.path.join(os.path.dirname(__file__), "config", "config.yml"),
"r") as f:
serpent_config = yaml.safe_load(f) or {}
with open(os.path.join(os.getcwd(), "config", "config.yml"), "r") as f:
user_config = yaml.safe_load(f) or {}
config_changed = False
for key, value in serpent_config.items():
if key not in user_config:
user_config[key] = value
config_changed = True
if config_changed:
with open(os.path.join(os.getcwd(), "config", "config.yml"), "w") as f:
f.write(yaml.dump(user_config))
def generate_game_plugin():
clear_terminal()
display_serpent_logo()
print("")
game_name = input(
"What is the name of the game? (Titleized, No Spaces i.e. AwesomeGame): \n"
)
game_platform = input(
"How is the game launched? (One of: 'steam', 'executable', 'web_browser'): \n"
)
if game_name in [None, ""]:
raise Exception("Invalid game name.")
if game_platform not in ["steam", "executable", "web_browser"]:
raise Exception("Invalid game platform.")
prepare_game_plugin(game_name, game_platform)
subprocess.call(
shlex.split(f"serpent activate Serpent{game_name}GamePlugin"))
def flush(self):
clear_terminal()
print("\n".join(self.lines))
self.clear()
def on_record_pause(self, **kwargs):
InputRecorder.pause_input_recording()
input_events = list()
input_event_count = self.redis_client.llen(
config["input_recorder"]["redis_key"])
for i in range(input_event_count):
input_events.append(
pickle.loads(
self.redis_client.lpop(
config["input_recorder"]["redis_key"])))
data = self._merge_frames_and_input_events(input_events)
if not len(data):
time.sleep(1)
return None
latest_game_frame_buffer = None
active_keys = set()
down_keys = dict()
observations = dict()
compute_reward = "reward_function" in self.config and self.config[
"reward_function"] in self.reward_functions
reward_func = None
if compute_reward:
reward_func = self.reward_functions[self.config["reward_function"]]
for item in data:
if isinstance(item, GameFrameBuffer):
latest_game_frame_buffer = item
reward_score = 0
if compute_reward:
reward_score = reward_func(item.frames)
timestamp = item.frames[-2].timestamp
observations[timestamp] = [
item,
dict(),
list(active_keys),
list(), reward_score
]
elif item["type"] == "keyboard":
key_name, key_event = item["name"].split("-")
if key_event == "DOWN":
active_keys.add(key_name)
if latest_game_frame_buffer is not None:
timestamp = latest_game_frame_buffer.frames[
-2].timestamp
observations[timestamp][1][key_name] = item[
"timestamp"]
down_keys[key_name] = timestamp
elif key_event == "UP":
active_keys.remove(key_name)
if key_name in down_keys:
timestamp = down_keys[key_name]
duration = item["timestamp"] - observations[timestamp][
1][key_name]
observations[timestamp][1][key_name] = duration
del down_keys[key_name]
elif item["type"] == "mouse":
if latest_game_frame_buffer is not None:
timestamp = latest_game_frame_buffer.frames[-2].timestamp
observations[timestamp][3].append(item)
print(
f"Writing Recorded Input Data to 'datasets/input_recording.h5'... (0/{len(observations)})"
)
with h5py.File("datasets/input_recording.h5", "a") as f:
i = 0
for timestamp, observation in observations.items():
clear_terminal()
print(
f"Writing Recorded Input Data to 'datasets/input_recording.h5'... ({i + 1}/{len(observations)})"
)
game_frame_buffer, keyboard_inputs, keyboard_inputs_active, mouse_inputs, reward_score = observation
f.create_dataset(f"{timestamp}-frames",
data=[
game_frame.frame_bytes
for game_frame in game_frame_buffer.frames
])
f.create_dataset(
f"{timestamp}-keyboard-inputs",
data=[(key_name.encode("utf-8"),
str(duration).encode("utf-8"))
for key_name, duration in keyboard_inputs.items()])
f.create_dataset(f"{timestamp}-keyboard-inputs-active",
data=[
key_name.encode("utf-8")
for key_name in keyboard_inputs_active
])
filtered_mouse_inputs = list()
mouse_move_index = None
valid_game_window_x = range(
self.game.window_geometry["x_offset"],
self.game.window_geometry["x_offset"] +
self.game.window_geometry["width"] + 1)
valid_game_window_y = range(
self.game.window_geometry["y_offset"],
self.game.window_geometry["y_offset"] +
self.game.window_geometry["height"] + 1)
for mouse_input in mouse_inputs:
if mouse_input["x"] in valid_game_window_x and mouse_input[
"y"] in valid_game_window_y:
if mouse_input["name"] == "MOVE":
mouse_move_index = len(filtered_mouse_inputs)
filtered_mouse_inputs.append(mouse_input)
mouse_input_data = list()
for i, mouse_input in enumerate(filtered_mouse_inputs):
if mouse_input["name"] == "MOVE" and i != mouse_move_index:
continue
mouse_input_data.append(
(mouse_input["name"].encode("utf-8"),
mouse_input["button"].encode("utf-8")
if mouse_input["button"] else b"",
mouse_input["direction"].encode("utf-8")
if mouse_input["direction"] else b"",
mouse_input["velocity"] or b"", mouse_input["x"],
mouse_input["y"], mouse_input["timestamp"]))
f.create_dataset(f"{timestamp}-mouse-inputs",
data=mouse_input_data)
f.create_dataset(f"{timestamp}-reward", data=reward_score)
i += 1
self.game_frame_buffers = list()
clear_terminal()
print(
f"Writing Frame/Input Data to 'datasets/input_recording.h5'... DONE"
)
time.sleep(1)
def play(self,
game_agent_class_name="GameAgent",
frame_handler=None,
**kwargs):
if not self.is_launched:
raise GameError(
f"Game '{self.__class__.__name__}' is not running...")
game_agent_class = offshoot.discover(
"GameAgent",
selection=game_agent_class_name).get(game_agent_class_name,
GameAgent)
if game_agent_class is None:
raise GameError(
"The provided Game Agent class name does not map to an existing class..."
)
game_agent = game_agent_class(game=self,
input_controller=InputController(
game=self,
backend=self.input_controller),
**kwargs)
# Look if we need to auto-append PNG to frame transformation pipeline based on given frame_handler
png_frame_handlers = ["RECORD"]
if frame_handler in png_frame_handlers and self.frame_transformation_pipeline_string is not None:
if not self.frame_transformation_pipeline_string.endswith("|PNG"):
self.frame_transformation_pipeline_string += "|PNG"
self.start_frame_grabber()
self.redis_client.delete(config["frame_grabber"]["redis_key"])
while self.redis_client.llen(
config["frame_grabber"]["redis_key"]) == 0:
time.sleep(0.1)
self.window_controller.focus_window(self.window_id)
frame_type = "FULL"
pipeline_frame_handlers = [
"COLLECT_FRAMES", "COLLECT_FRAME_REGIONS",
"COLLECT_FRAMES_FOR_CONTEXT", "RECORD"
]
if frame_handler in pipeline_frame_handlers and self.frame_transformation_pipeline_string is not None:
frame_type = "PIPELINE"
# Override FPS Config?
if frame_handler == "RECORD":
self.game_frame_limiter = GameFrameLimiter(fps=10)
try:
while True:
self.game_frame_limiter.start()
game_frame = self.grab_latest_frame(frame_type=frame_type)
try:
if self.is_focused:
game_agent.on_game_frame(game_frame,
frame_handler=frame_handler,
**kwargs)
else:
clear_terminal()
print("PAUSED\n")
game_agent.on_pause(frame_handler=frame_handler,
**kwargs)
time.sleep(1)
except Exception as e:
raise e
# print(e)
# time.sleep(0.1)
self.game_frame_limiter.stop_and_delay()
finally:
self.stop_frame_grabber()
