def __init__(self,
name,
game_inputs=None,
callbacks=None,
seed=None,
window_geometry=None,
logger=Loggers.NOOP,
logger_kwargs=None):
super().__init__(name,
game_inputs=game_inputs,
callbacks=callbacks,
seed=seed,
logger=logger,
logger_kwargs=logger_kwargs)
if window_geometry is None or not isinstance(window_geometry, dict):
raise SerpentError(
"RecorderAgent expects a 'window_geometry' dict kwarg.")
self.window_geometry = window_geometry
self.game_frame_buffers = list()
self.rewards = list()
self.current_step = 0
self.redis_client = StrictRedis(**config["redis"])
InputRecorder.pause_input_recording()
input_recorder_command = "serpent record_inputs"
self.input_recorder_process = subprocess.Popen(
shlex.split(input_recorder_command))
signal.signal(signal.SIGINT, self._handle_signal)
signal.signal(signal.SIGTERM, self._handle_signal)
atexit.register(self._handle_signal, 15, None, False)
def on_record_pause(self, **kwargs):
InputRecorder.pause_input_recording()
input_events = list()
input_event_count = self.redis_client.llen(
config["input_recorder"]["redis_key"])
for i in range(input_event_count):
input_events.append(
pickle.loads(
self.redis_client.lpop(
config["input_recorder"]["redis_key"])))
data = self._merge_frames_and_input_events(input_events)
if not len(data):
time.sleep(1)
return None
latest_game_frame_buffer = None
active_keys = set()
down_keys = dict()
observations = dict()
compute_reward = "reward_function" in self.config and self.config[
"reward_function"] in self.reward_functions
reward_func = None
if compute_reward:
reward_func = self.reward_functions[self.config["reward_function"]]
for item in data:
if isinstance(item, GameFrameBuffer):
latest_game_frame_buffer = item
reward_score = 0
if compute_reward:
reward_score = reward_func(item.frames)
timestamp = item.frames[-2].timestamp
observations[timestamp] = [
item,
dict(),
list(active_keys),
list(), reward_score
]
elif item["type"] == "keyboard":
key_name, key_event = item["name"].split("-")
if key_event == "DOWN":
active_keys.add(key_name)
if latest_game_frame_buffer is not None:
timestamp = latest_game_frame_buffer.frames[
-2].timestamp
observations[timestamp][1][key_name] = item[
"timestamp"]
down_keys[key_name] = timestamp
elif key_event == "UP":
active_keys.remove(key_name)
if key_name in down_keys:
timestamp = down_keys[key_name]
duration = item["timestamp"] - observations[timestamp][
1][key_name]
observations[timestamp][1][key_name] = duration
del down_keys[key_name]
elif item["type"] == "mouse":
if latest_game_frame_buffer is not None:
timestamp = latest_game_frame_buffer.frames[-2].timestamp
observations[timestamp][3].append(item)
print(
f"Writing Recorded Input Data to 'datasets/input_recording.h5'... (0/{len(observations)})"
)
with h5py.File("datasets/input_recording.h5", "a") as f:
i = 0
for timestamp, observation in observations.items():
clear_terminal()
print(
f"Writing Recorded Input Data to 'datasets/input_recording.h5'... ({i + 1}/{len(observations)})"
)
game_frame_buffer, keyboard_inputs, keyboard_inputs_active, mouse_inputs, reward_score = observation
f.create_dataset(f"{timestamp}-frames",
data=[
game_frame.frame_bytes
for game_frame in game_frame_buffer.frames
])
f.create_dataset(
f"{timestamp}-keyboard-inputs",
data=[(key_name.encode("utf-8"),
str(duration).encode("utf-8"))
for key_name, duration in keyboard_inputs.items()])
f.create_dataset(f"{timestamp}-keyboard-inputs-active",
data=[
key_name.encode("utf-8")
for key_name in keyboard_inputs_active
])
filtered_mouse_inputs = list()
mouse_move_index = None
valid_game_window_x = range(
self.game.window_geometry["x_offset"],
self.game.window_geometry["x_offset"] +
self.game.window_geometry["width"] + 1)
valid_game_window_y = range(
self.game.window_geometry["y_offset"],
self.game.window_geometry["y_offset"] +
self.game.window_geometry["height"] + 1)
for mouse_input in mouse_inputs:
if mouse_input["x"] in valid_game_window_x and mouse_input[
"y"] in valid_game_window_y:
if mouse_input["name"] == "MOVE":
mouse_move_index = len(filtered_mouse_inputs)
filtered_mouse_inputs.append(mouse_input)
mouse_input_data = list()
for i, mouse_input in enumerate(filtered_mouse_inputs):
if mouse_input["name"] == "MOVE" and i != mouse_move_index:
continue
mouse_input_data.append(
(mouse_input["name"].encode("utf-8"),
mouse_input["button"].encode("utf-8")
if mouse_input["button"] else b"",
mouse_input["direction"].encode("utf-8")
if mouse_input["direction"] else b"",
mouse_input["velocity"] or b"", mouse_input["x"],
mouse_input["y"], mouse_input["timestamp"]))
f.create_dataset(f"{timestamp}-mouse-inputs",
data=mouse_input_data)
f.create_dataset(f"{timestamp}-reward", data=reward_score)
i += 1
self.game_frame_buffers = list()
clear_terminal()
print(
f"Writing Frame/Input Data to 'datasets/input_recording.h5'... DONE"
)
time.sleep(1)
def record_inputs():
from serpent.input_recorder import InputRecorder
input_recorder = InputRecorder()
input_recorder.start()
def observe(self, reward=0, terminal=False, **kwargs):
if self.current_state is None:
return None
self.current_step += 1
print(self.current_step)
self.rewards.append((reward, terminal))
if terminal:
InputRecorder.pause_input_recording()
input_events = list()
input_event_count = self.redis_client.llen(
config["input_recorder"]["redis_key"])
for i in range(input_event_count):
input_events.append(
pickle.loads(
self.redis_client.lpop(
config["input_recorder"]["redis_key"])))
data = self._merge_frames_and_input_events(input_events)
latest_game_frame_buffer = None
rewards_index = 0
active_keys = set()
down_keys = dict()
observations = dict()
for item in data:
if isinstance(item, GameFrameBuffer):
latest_game_frame_buffer = item
reward, terminal = self.rewards[rewards_index]
rewards_index += 1
timestamp = item.frames[-2].timestamp
observations[timestamp] = [
item,
dict(),
list(active_keys),
list(), reward, terminal
]
elif item["type"] == "keyboard":
key_name, key_event = item["name"].split("-")
if key_event == "DOWN":
active_keys.add(key_name)
if latest_game_frame_buffer is not None:
timestamp = latest_game_frame_buffer.frames[
-2].timestamp
observations[timestamp][1][key_name] = item[
"timestamp"]
down_keys[key_name] = timestamp
elif key_event == "UP":
if key_name in active_keys:
active_keys.remove(key_name)
if key_name in down_keys:
timestamp = down_keys[key_name]
duration = item["timestamp"] - observations[
timestamp][1][key_name]
observations[timestamp][1][key_name] = duration
del down_keys[key_name]
elif item["type"] == "mouse":
if latest_game_frame_buffer is not None:
timestamp = latest_game_frame_buffer.frames[
-2].timestamp
observations[timestamp][3].append(item)
with h5py.File(f"datasets/{self.name}_input_recording.h5",
"a") as f:
i = 0
for timestamp, observation in observations.items():
game_frame_buffer, keyboard_inputs, keyboard_inputs_active, mouse_inputs, reward, terminal = observation
f.create_dataset(
f"{timestamp}-frames",
data=[
game_frame.to_png_bytes()
for game_frame in game_frame_buffer.frames
])
f.create_dataset(
f"{timestamp}-keyboard-inputs",
data=[
(key_name.encode("utf-8"),
str(duration).encode("utf-8"))
for key_name, duration in keyboard_inputs.items()
])
f.create_dataset(f"{timestamp}-keyboard-inputs-active",
data=[
key_name.encode("utf-8")
for key_name in keyboard_inputs_active
])
filtered_mouse_inputs = list()
mouse_move_index = None
valid_game_window_x = range(
self.window_geometry["x_offset"],
self.window_geometry["x_offset"] +
self.window_geometry["width"] + 1)
valid_game_window_y = range(
self.window_geometry["y_offset"],
self.window_geometry["y_offset"] +
self.window_geometry["height"] + 1)
for mouse_input in mouse_inputs:
if mouse_input[
"x"] in valid_game_window_x and mouse_input[
"y"] in valid_game_window_y:
if mouse_input["name"] == "MOVE":
mouse_move_index = len(filtered_mouse_inputs)
filtered_mouse_inputs.append(mouse_input)
mouse_input_data = list()
for i, mouse_input in enumerate(filtered_mouse_inputs):
if mouse_input[
"name"] == "MOVE" and i != mouse_move_index:
continue
mouse_input_data.append(
(mouse_input["name"].encode("utf-8"),
mouse_input["button"].encode("utf-8")
if mouse_input["button"] else b"",
mouse_input["direction"].encode("utf-8")
if mouse_input["direction"] else b"",
mouse_input["velocity"] or b"", mouse_input["x"],
mouse_input["y"], mouse_input["timestamp"]))
f.create_dataset(f"{timestamp}-mouse-inputs",
data=mouse_input_data)
f.create_dataset(f"{timestamp}-reward", data=reward)
f.create_dataset(f"{timestamp}-terminal", data=terminal)
i += 1
self.game_frame_buffers = list()
self.rewards = list()
else:
InputRecorder.resume_input_recording()
super().observe(reward=reward, terminal=terminal, **kwargs)