def poison_check(self, game_frame):
image_data = skimage.io.imread("plugins/SerpentSlayTheSpireGamePlugin/files/data/sprites/sprite_poison_check_0.png")[..., np.newaxis]
poison_check = Sprite("poison_check", image_data=image_data)
# # Full game frame capture
# print("-----------Full game frame capture-----------")
# full_game_frame = FrameGrabber.get_frames(
# [0],
# frame_shape=(self.game.frame_height, self.game.frame_width),
# frame_type="PIPELINE"
# ).frames[0]
sprite_locator = SpriteLocator()
poison_check_location = sprite_locator.locate(sprite=poison_check, game_frame=game_frame)
print("poison_check_location: ", poison_check_location)
if (poison_check_location != None):
self.game_state["poison_check"].insert(0, True)
print("POISON_CHECK == TRUE")
else:
self.game_state["poison_check"].insert(0, False)
print("POISON_CHECK == FALSE")
self.ddqn_setup(game_frame)
def _measure_actor_hp(self, game_frame):
hp_area_frame = serpent.cv.extract_region_from_image(
game_frame.frame, self.game.screen_regions["HP_AREA"])
hp_area_image = Image.fromarray(hp_area_frame)
actor_hp = 0
image_colors = hp_area_image.getcolors(
) # TODO: remove in favor of sprite detection and location
if image_colors:
actor_hp = len(image_colors) - 7
for name, sprite in self.game.sprites.items():
query_sprite = Sprite("QUERY", image_data=sprite.image_data)
sprite_name = self.sprite_identifier.identify(
query_sprite, mode="CONSTELLATION_OF_PIXELS"
) # Will be "UNKNOWN" if no match
print(sprite_name)
sprite_to_locate = Sprite("QUERY", image_data=sprite.image_data)
sprite_locator = SpriteLocator()
location = sprite_locator.locate(sprite=sprite_to_locate,
game_frame=game_frame)
print(location)
if location:
actor_hp = 1000000
return actor_hp
def identify_sprite(self, sprite, game_frame):
# sprite_identifier = SpriteIdentifier()
sprite_identify = self.sprite_identifier.identify(
sprite=sprite, mode="SIGNATURE_COLORS")
sprite_locator = SpriteLocator()
location = sprite_locator.locate(sprite=sprite, game_frame=game_frame)
output = {'sprite_name': sprite_identify, 'location': location}
return output
def check_game_state(self, game_frame):
#game over?
locationGO = None
sprite_to_locate = Sprite("QUERY", image_data=self.spriteGO.image_data)
sprite_locator = SpriteLocator()
locationGO = sprite_locator.locate(sprite=sprite_to_locate,
game_frame=game_frame)
print("Location Game over:", locationGO)
#won game?
locationWO = None
sprite_to_locate = Sprite("QUERY", image_data=self.spriteWO.image_data)
sprite_locator = SpriteLocator()
locationWO = sprite_locator.locate(sprite=sprite_to_locate,
game_frame=game_frame.frames)
print("Location Game won:", locationWO)
self.gamestate.girl_alive = (locationGO == None and locationWO == None)
self.gamestate.done = not self.gamestate.girl_alive
self.gamestate.victory = locationWO != None
print(f"Is alive? {self.gamestate.girl_alive}")
print(f"Game over? {self.gamestate.lose}")
print(f"Won? {self.gamestate.victory}")
def _measure_actor_hp(self, game_frame):
hp_area_frame = serpent.cv.extract_region_from_image(
game_frame.frame, self.game.screen_regions["HP_AREA"])
hp_area_image = Image.fromarray(hp_area_frame)
actor_hp = 0
image_colors = hp_area_image.getcolors(
) # TODO: remove in favor of sprite detection and location
if image_colors:
actor_hp = len(image_colors) - 7
for name, sprite in self.game.sprites.items():
sprite_to_locate = Sprite("QUERY", image_data=sprite.image_data)
sprite_locator = SpriteLocator()
location = sprite_locator.locate(sprite=sprite_to_locate,
game_frame=game_frame)
print(location)
if location:
actor_hp = 1000000
return actor_hp
class NativeWin32InputController(InputController):
def __init__(self, game=None, **kwargs):
self.game = game
self.previous_key_collection_set = set()
self.sprite_locator = SpriteLocator()
# Keyboard Actions
def handle_keys(self, key_collection, **kwargs):
key_collection_set = set(key_collection)
keys_to_press = key_collection_set - self.previous_key_collection_set
keys_to_release = self.previous_key_collection_set - key_collection_set
for key in keys_to_press:
self.press_key(key, **kwargs)
for key in keys_to_release:
self.release_key(key, **kwargs)
self.previous_key_collection_set = key_collection_set
def tap_keys(self, keys, duration=0.05, **kwargs):
if ("force" in kwargs
and kwargs["force"] is True) or self.game_is_focused:
for key in keys:
self.press_key(key, **kwargs)
time.sleep(duration)
for key in keys:
self.release_key(key, **kwargs)
def tap_key(self, key, duration=0.05, **kwargs):
if ("force" in kwargs
and kwargs["force"] is True) or self.game_is_focused:
self.press_key(key, **kwargs)
time.sleep(duration)
self.release_key(key, **kwargs)
def press_keys(self, keys, **kwargs):
for key in keys:
self.press_key(key, **kwargs)
def press_key(self, key, **kwargs):
if ("force" in kwargs
and kwargs["force"] is True) or self.game_is_focused:
extra = ctypes.c_ulong(0)
ii_ = Input_I()
if keyboard_key_mapping[key.name] >= 1024:
key = keyboard_key_mapping[key.name] - 1024
flags = 0x0008 | 0x0001
else:
key = keyboard_key_mapping[key.name]
flags = 0x0008
ii_.ki = KeyBdInput(0, key, flags, 0, ctypes.pointer(extra))
x = Input(ctypes.c_ulong(1), ii_)
ctypes.windll.user32.SendInput(1, ctypes.pointer(x),
ctypes.sizeof(x))
def release_keys(self, keys, **kwargs):
for key in keys:
self.release_key(key, **kwargs)
def release_key(self, key, **kwargs):
if ("force" in kwargs
and kwargs["force"] is True) or self.game_is_focused:
extra = ctypes.c_ulong(0)
ii_ = Input_I()
if keyboard_key_mapping[key.name] >= 1024:
key = keyboard_key_mapping[key.name] - 1024
flags = 0x0008 | 0x0001 | 0x0002
else:
key = keyboard_key_mapping[key.name]
flags = 0x0008 | 0x0002
ii_.ki = KeyBdInput(0, key, flags, 0, ctypes.pointer(extra))
x = Input(ctypes.c_ulong(1), ii_)
ctypes.windll.user32.SendInput(1, ctypes.pointer(x),
ctypes.sizeof(x))
def type_string(self, string, duration=0.05, **kwargs):
if ("force" in kwargs
and kwargs["force"] is True) or self.game_is_focused:
for character in string:
keys = character_keyboard_key_mapping.get(character)
if keys is not None:
self.tap_keys(keys, duration=duration, **kwargs)
# Mouse Actions
def move(self, x=None, y=None, duration=0.25, absolute=True, **kwargs):
if ("force" in kwargs
and kwargs["force"] is True) or self.game_is_focused:
x += self.game.window_geometry["x_offset"]
y += self.game.window_geometry["y_offset"]
current_pixel_coordinates = win32api.GetCursorPos()
start_coordinates = self._to_windows_coordinates(
*current_pixel_coordinates)
if absolute:
end_coordinates = self._to_windows_coordinates(x, y)
else:
end_coordinates = self._to_windows_coordinates(
current_pixel_coordinates[0] + x,
current_pixel_coordinates[1] + y)
coordinates = self._interpolate_mouse_movement(
start_windows_coordinates=start_coordinates,
end_windows_coordinates=end_coordinates)
for x, y in coordinates:
extra = ctypes.c_ulong(0)
ii_ = Input_I()
ii_.mi = MouseInput(x, y, 0, (0x0001 | 0x8000), 0,
ctypes.pointer(extra))
x = Input(ctypes.c_ulong(0), ii_)
ctypes.windll.user32.SendInput(1, ctypes.pointer(x),
ctypes.sizeof(x))
time.sleep(duration / 20)
def click_down(self, button=MouseButton.LEFT, **kwargs):
if ("force" in kwargs
and kwargs["force"] is True) or self.game_is_focused:
extra = ctypes.c_ulong(0)
ii_ = Input_I()
ii_.mi = MouseInput(0, 0, 0,
mouse_button_down_mapping[button.name], 0,
ctypes.pointer(extra))
x = Input(ctypes.c_ulong(0), ii_)
ctypes.windll.user32.SendInput(1, ctypes.pointer(x),
ctypes.sizeof(x))
def click_up(self, button=MouseButton.LEFT, **kwargs):
if ("force" in kwargs
and kwargs["force"] is True) or self.game_is_focused:
extra = ctypes.c_ulong(0)
ii_ = Input_I()
ii_.mi = MouseInput(0, 0, 0, mouse_button_up_mapping[button.name],
0, ctypes.pointer(extra))
x = Input(ctypes.c_ulong(0), ii_)
ctypes.windll.user32.SendInput(1, ctypes.pointer(x),
ctypes.sizeof(x))
def click(self, button=MouseButton.LEFT, duration=0.05, **kwargs):
if ("force" in kwargs
and kwargs["force"] is True) or self.game_is_focused:
self.click_down(button=button, **kwargs)
time.sleep(duration)
self.click_up(button=button, **kwargs)
def click_screen_region(self,
button=MouseButton.LEFT,
screen_region=None,
**kwargs):
if ("force" in kwargs
and kwargs["force"] is True) or self.game_is_focused:
screen_region_coordinates = self.game.screen_regions.get(
screen_region)
x = (screen_region_coordinates[1] +
screen_region_coordinates[3]) // 2
y = (screen_region_coordinates[0] +
screen_region_coordinates[2]) // 2
self.move(x, y)
self.click(button=button, **kwargs)
def click_sprite(self,
button=MouseButton.LEFT,
sprite=None,
game_frame=None,
**kwargs):
if ("force" in kwargs
and kwargs["force"] is True) or self.game_is_focused:
sprite_location = self.sprite_locator.locate(sprite=sprite,
game_frame=game_frame)
if sprite_location is None:
return False
x = (sprite_location[1] + sprite_location[3]) // 2
y = (sprite_location[0] + sprite_location[2]) // 2
self.move(x, y)
self.click(button=button, **kwargs)
return True
# Requires the Serpent OCR module
def click_string(self,
query_string,
button=MouseButton.LEFT,
game_frame=None,
fuzziness=2,
ocr_preset=None,
**kwargs):
import serpent.ocr
if ("force" in kwargs
and kwargs["force"] is True) or self.game_is_focused:
string_location = serpent.ocr.locate_string(
query_string,
game_frame.frame,
fuzziness=fuzziness,
ocr_preset=ocr_preset,
offset_x=game_frame.offset_x,
offset_y=game_frame.offset_y)
if string_location is not None:
x = (string_location[1] + string_location[3]) // 2
y = (string_location[0] + string_location[2]) // 2
self.move(x, y)
self.click(button=button, **kwargs)
return True
return False
def drag(self,
button=MouseButton.LEFT,
x0=None,
y0=None,
x1=None,
y1=None,
duration=0.25,
**kwargs):
if ("force" in kwargs
and kwargs["force"] is True) or self.game_is_focused:
self.move(x=x0, y=y0)
self.click_down(button=button)
self.move(x=x1, y=y1, duration=duration)
self.click_up(button=button)
def drag_screen_region_to_screen_region(self,
button=MouseButton.LEFT,
start_screen_region=None,
end_screen_region=None,
duration=1,
**kwargs):
if ("force" in kwargs
and kwargs["force"] is True) or self.game_is_focused:
start_screen_region_coordinates = self._extract_screen_region_coordinates(
start_screen_region)
end_screen_region_coordinates = self._extract_screen_region_coordinates(
end_screen_region)
self.drag(button=button,
x0=start_screen_region_coordinates[0],
y0=start_screen_region_coordinates[1],
x1=end_screen_region_coordinates[0],
y1=end_screen_region_coordinates[1],
duration=duration,
**kwargs)
def scroll(self, clicks=1, direction="DOWN", **kwargs):
if ("force" in kwargs
and kwargs["force"] is True) or self.game_is_focused:
clicks = clicks * (1 if direction == "UP" else -1) * 120
extra = ctypes.c_ulong(0)
ii_ = Input_I()
ii_.mi = MouseInput(0, 0, clicks, 0x0800, 0, ctypes.pointer(extra))
x = Input(ctypes.c_ulong(0), ii_)
ctypes.windll.user32.SendInput(1, ctypes.pointer(x),
ctypes.sizeof(x))
@staticmethod
def _to_windows_coordinates(x=0, y=0):
display_width = win32api.GetSystemMetrics(0)
display_height = win32api.GetSystemMetrics(1)
windows_x = (x * 65535) // display_width
windows_y = (y * 65535) // display_height
return windows_x, windows_y
@staticmethod
def _interpolate_mouse_movement(start_windows_coordinates,
end_windows_coordinates,
steps=20):
x_coordinates = [
start_windows_coordinates[0], end_windows_coordinates[0]
]
y_coordinates = [
start_windows_coordinates[1], end_windows_coordinates[1]
]
if x_coordinates[0] == x_coordinates[1]:
x_coordinates[1] += 1
if y_coordinates[0] == y_coordinates[1]:
y_coordinates[1] += 1
interpolation_func = scipy.interpolate.interp1d(
x_coordinates, y_coordinates)
intermediate_x_coordinates = np.linspace(start_windows_coordinates[0],
end_windows_coordinates[0],
steps + 1)[1:]
coordinates = list(
map(lambda x: (int(round(x)), int(interpolation_func(x))),
intermediate_x_coordinates))
return coordinates
def find_sprite(self, sprite_name):
sprite_locator = SpriteLocator()
location = sprite_locator.locate(sprite=self.game.sprites[sprite_name],
game_frame=self.game_frame)
return location # None if not found
class PyAutoGUIInputController(InputController):
def __init__(self, game=None, **kwargs):
self.game = game
self.previous_key_collection_set = set()
self.sprite_locator = SpriteLocator()
# Keyboard Actions
def handle_keys(self, key_collection, **kwargs):
key_collection_set = set(key_collection)
keys_to_press = key_collection_set - self.previous_key_collection_set
keys_to_release = self.previous_key_collection_set - key_collection_set
for key in keys_to_press:
self.press_key(key, **kwargs)
for key in keys_to_release:
self.release_key(key, **kwargs)
self.previous_key_collection_set = key_collection_set
def tap_keys(self, keys, duration=0.05, **kwargs):
if ("force" in kwargs and kwargs["force"] is True) or self.game_is_focused:
for key in keys:
self.press_key(key, **kwargs)
time.sleep(duration)
for key in keys:
self.release_key(key, **kwargs)
def tap_key(self, key, duration=0.05, **kwargs):
if ("force" in kwargs and kwargs["force"] is True) or self.game_is_focused:
self.press_key(key, **kwargs)
time.sleep(duration)
self.release_key(key, **kwargs)
def press_keys(self, keys, **kwargs):
for key in keys:
self.press_key(key, **kwargs)
def press_key(self, key, **kwargs):
if ("force" in kwargs and kwargs["force"] is True) or self.game_is_focused:
pyautogui.keyDown(keyboard_key_mapping[key.name])
def release_keys(self, keys, **kwargs):
for key in keys:
self.release_key(key, **kwargs)
def release_key(self, key, **kwargs):
if ("force" in kwargs and kwargs["force"] is True) or self.game_is_focused:
pyautogui.keyUp(keyboard_key_mapping[key.name])
def type_string(self, string, duration=0.05, **kwargs):
if ("force" in kwargs and kwargs["force"] is True) or self.game_is_focused:
pyautogui.typewrite(message=string, interval=duration)
# Mouse Actions
def move(self, x=None, y=None, duration=0.25, absolute=True, **kwargs):
if ("force" in kwargs and kwargs["force"] is True) or self.game_is_focused:
x += self.game.window_geometry["x_offset"]
y += self.game.window_geometry["y_offset"]
if absolute:
pyautogui.moveTo(x=x, y=y, duration=duration)
else:
pyautogui.moveRel(xOffset=x, yOffset=y, duration=duration)
def click_down(self, button=MouseButton.LEFT, **kwargs):
if ("force" in kwargs and kwargs["force"] is True) or self.game_is_focused:
pyautogui.mouseDown(button=mouse_button_mapping[button.name])
def click_up(self, button=MouseButton.LEFT, **kwargs):
if ("force" in kwargs and kwargs["force"] is True) or self.game_is_focused:
pyautogui.mouseUp(button=mouse_button_mapping[button.name])
def click(self, button=MouseButton.LEFT, duration=0.25, **kwargs):
if ("force" in kwargs and kwargs["force"] is True) or self.game_is_focused:
self.click_down(button=button, **kwargs)
time.sleep(duration)
self.click_up(button=button, **kwargs)
def click_screen_region(self, button=MouseButton.LEFT, screen_region=None, **kwargs):
if ("force" in kwargs and kwargs["force"] is True) or self.game_is_focused:
screen_region_coordinates = self.game.screen_regions.get(screen_region)
x = (screen_region_coordinates[1] + screen_region_coordinates[3]) // 2
y = (screen_region_coordinates[0] + screen_region_coordinates[2]) // 2
self.move(x=x, y=y)
self.click(button=button, **kwargs)
def click_sprite(self, button=MouseButton.LEFT, sprite=None, game_frame=None, **kwargs):
if ("force" in kwargs and kwargs["force"] is True) or self.game_is_focused:
sprite_location = self.sprite_locator.locate(sprite=sprite, game_frame=game_frame)
if sprite_location is None:
return False
x = (sprite_location[1] + sprite_location[3]) // 2
y = (sprite_location[0] + sprite_location[2]) // 2
self.move(x=x, y=y)
self.click(button=button, **kwargs)
return True
# Requires the Serpent OCR module
def click_string(self, query_string, button=MouseButton.LEFT, game_frame=None, fuzziness=2, ocr_preset=None, **kwargs):
import serpent.ocr
if ("force" in kwargs and kwargs["force"] is True) or self.game_is_focused:
string_location = serpent.ocr.locate_string(
query_string,
game_frame.frame,
fuzziness=fuzziness,
ocr_preset=ocr_preset,
offset_x=game_frame.offset_x,
offset_y=game_frame.offset_y
)
if string_location is not None:
x = (string_location[1] + string_location[3]) // 2
y = (string_location[0] + string_location[2]) // 2
self.move(x=x, y=y)
self.click(button=button, **kwargs)
return True
return False
def drag(self, button=MouseButton.LEFT, x0=None, y0=None, x1=None, y1=None, duration=0.25, **kwargs):
if ("force" in kwargs and kwargs["force"] is True) or self.game_is_focused:
self.move(x=x0, y=y0)
self.click_down(button=button, **kwargs)
self.move(x=x1, y=y1, duration=duration)
self.click_up(button=button, **kwargs)
def drag_screen_region_to_screen_region(self, button=MouseButton.LEFT, start_screen_region=None, end_screen_region=None, duration=1, **kwargs):
if ("force" in kwargs and kwargs["force"] is True) or self.game_is_focused:
start_screen_region_coordinates = self._extract_screen_region_coordinates(start_screen_region)
end_screen_region_coordinates = self._extract_screen_region_coordinates(end_screen_region)
self.drag(
button=button,
x0=start_screen_region_coordinates[0],
y0=start_screen_region_coordinates[1],
x1=end_screen_region_coordinates[0],
y1=end_screen_region_coordinates[1],
duration=duration,
**kwargs
)
def scroll(self, clicks=1, direction="DOWN", **kwargs):
if ("force" in kwargs and kwargs["force"] is True) or self.game_is_focused:
clicks = clicks * (1 if direction == "DOWN" else -1)
pyautogui.scroll(clicks)
class DonkeyKongAPI(GameAPI):
def __init__(self, game=None):
super().__init__(game=game)
self.navigationGameFSM = NavigationGameFSM()
self._prepare_sprites()
self.WIDTH = 34
self.HEIGHT = 32
self.n_WIDTH = 2
self.n_HEIGHT = 2
self.MARIO = 1
self.MOVING_ENTITY = 2
self.LADDER = 3
self.LADDER_DELTA = 18
self.LEVEL_WIN_HEIGHT = 32
self.sprite_locator = SpriteLocator()
self.ladders_positions = dict()
self.ladders_positions[0] = [453, 10000]
self.ladders_positions[1] = [102, 249]
self.ladders_positions[2] = [285, 452]
self.ladders_positions[3] = [104, 194]
self.ladders_positions[4] = [452, 10000]
self.ladders_positions[5] = [324, 10000]
self.on_ladders = dict()
self.on_ladders[(0, 453)] = [374, 328]
self.on_ladders[(1, 102)] = [312, 268]
self.on_ladders[(1, 249)] = [318, 262]
self.on_ladders[(2, 285)] = [258, 202]
self.on_ladders[(2, 452)] = [252, 208]
self.on_ladders[(3, 104)] = [192, 148]
self.on_ladders[(3, 194)] = [196, 144]
self.on_ladders[(4, 452)] = [133, 88]
self.on_ladders[(5, 324)] = [81, 32]
self.last_stage = 0
self.max_stage = 0
self.last_change_stage = 0
self.useless_actions = 0
self.level_direction = dict()
orientation = 1
for i in range(6):
self.level_direction[i] = orientation
orientation = orientation * (-1)
self.ladders_thresholds = [384, 328, 268, 208, 148, 88]
def _prepare_sprites(self):
# Death Sprite
path = './plugins/SerpentDonkeyKongGamePlugin/files/data/sprites/death_{}.png'
image = io.imread(path.format(1))
self.death_sprite = Sprite("DEATH", image_data=image[..., np.newaxis])
for i in range(2, 11):
image = io.imread(path.format(i))
self.death_sprite.append_image_data(image[..., np.newaxis])
# Mario Sprite
path = './plugins/SerpentDonkeyKongGamePlugin/files/data/sprites/mario_{}.png'
image = io.imread(path.format(1))
self.mario_sprite = Sprite("MARIO", image_data=image[..., np.newaxis])
for i in range(2, 30):
image = io.imread(path.format(i))
self.mario_sprite.append_image_data(image[..., np.newaxis])
# Falling Barrels Sprite
path = './plugins/SerpentDonkeyKongGamePlugin/files/data/sprites/falling_barrel_{}.png'
image = io.imread(path.format(1))
self.falling_barrel_sprite = Sprite("FALLING",
image_data=image[..., np.newaxis])
for i in range(2, 3):
image = io.imread(path.format(i))
self.falling_barrel_sprite.append_image_data(image[...,
np.newaxis])
# Rolling Barrels Sprite
path = './plugins/SerpentDonkeyKongGamePlugin/files/data/sprites/rolling_barrel_{}.png'
image = io.imread(path.format(1))
self.rolling_barrel_sprite = Sprite("ROLLING",
image_data=image[..., np.newaxis])
for i in range(2, 5):
image = io.imread(path.format(i))
self.rolling_barrel_sprite.append_image_data(image[...,
np.newaxis])
# Fire Monster Sprite
path = './plugins/SerpentDonkeyKongGamePlugin/files/data/sprites/flammy_{}.png'
image = io.imread(path.format(1))
self.flammy_sprite = Sprite("FLAMMY",
image_data=image[..., np.newaxis])
for i in range(2, 5):
image = io.imread(path.format(i))
self.flammy_sprite.append_image_data(image[..., np.newaxis])
# Splash Screen Sprite
image = io.imread(
'./plugins/SerpentDonkeyKongGamePlugin/files/data/sprites/sprite_main_menu_splash_screen.png'
)
self.splash_screen = Sprite("SPLASH",
image_data=image[..., np.newaxis])
def analyze_frame(self, game_frame):
locations = [None, None]
if (self.navigationGameFSM.current == "black_screen"):
locations[0] = self.sprite_locator.locate(sprite=self.mario_sprite,
game_frame=game_frame)
if (locations[0] != None):
self.navigationGameFSM.play()
elif (self.navigationGameFSM.current == "playing"):
loc = self._multiple_locate(sprite=self.mario_sprite,
game_frame=game_frame,
forced=True)
if (len(loc) != 0):
locations[0] = loc[0]
locations[1] = None
else:
locations[0] = None
loc = self._multiple_locate(sprite=self.death_sprite,
game_frame=game_frame,
forced=True)
if (len(loc) != 0):
locations[1] = loc[0]
if (locations[0] != None):
has_mario = True
if (locations[1] != None and locations[0] == None):
self.navigationGameFSM.die()
has_mario = False
if (locations[0] != None and locations[1] == None
and locations[0][0] <= self.LEVEL_WIN_HEIGHT):
self.navigationGameFSM.win()
if (self.last_change_stage == 0):
self.last_change_stage = time.time()
return locations
def _get_ladders(self, mario_posY):
level = self._get_level(mario_posY)
return self.ladders_positions[level]
def _get_level(self, mario_posY):
stage = 0
for i in range(5, -1, -1):
if (mario_posY <= self.ladders_thresholds[i]):
stage = i
break
if (stage != self.last_stage):
self.last_stage = stage
if (stage > self.max_stage):
self.max_stage = stage
self.last_change_stage = time.time()
return stage
def _get_moving_entities(self, game_frame):
moving = []
inputs = [
self.rolling_barrel_sprite, self.falling_barrel_sprite,
self.flammy_sprite
]
with concurrent.futures.ThreadPoolExecutor(max_workers=3) as executor:
futures = {
executor.submit(self._multiple_locate, sprite, game_frame):
sprite
for sprite in inputs
}
for future in concurrent.futures.as_completed(futures):
moving = moving + future.result()
return moving
def get_projection_matrix(self, game_frame, location):
reduced_frame = None
units_array = None
if (location != None):
temp = game_frame.frame[location[0] -
self.HEIGHT * self.n_HEIGHT:location[2],
location[1] -
self.WIDTH * self.n_WIDTH:location[3] +
self.WIDTH * self.n_WIDTH]
reduced_frame = GameFrame(temp)
units_array = self._projection(reduced_frame, location)
nx, ny = units_array.shape
units_array = units_array.reshape(nx * ny)
orientation = self._get_orientation(location)
units_array = np.insert(units_array, len(units_array), orientation)
return (reduced_frame, units_array)
def _get_orientation(self, location):
level = self._get_level(location[0])
return self.level_direction[level]
def _projection(self, game_frame, global_mario_positions):
mario_location = self.sprite_locator.locate(sprite=self.mario_sprite,
game_frame=game_frame)
moving_entities = self._get_moving_entities(game_frame)
units_array = np.zeros((1 + self.n_HEIGHT, 1 + 2 * self.n_WIDTH))
units_array[self.n_HEIGHT][self.n_WIDTH] = self.MARIO
ladders = self._get_ladders(global_mario_positions[0])
for ladder in ladders:
if (abs(ladder - global_mario_positions[1]) <= self.LADDER_DELTA):
pos_ladder = self.n_WIDTH
elif (ladder < global_mario_positions[1]):
# ladder on left
pos_ladder = self.n_WIDTH - (int(
(global_mario_positions[1] - ladder) / self.WIDTH) + 1)
else:
# ladder on right
pos_ladder = int(
(ladder - global_mario_positions[1]) / self.WIDTH)
if (pos_ladder == 0):
pos_ladder = 1
pos_ladder = pos_ladder + self.n_WIDTH
if (pos_ladder >= 0 and pos_ladder <= (2 * self.n_WIDTH)):
units_array[self.n_HEIGHT][pos_ladder] = self.LADDER
for entity in moving_entities:
if (entity[3] <= mario_location[1]):
# left
posX = entity[3]
elif (entity[1] >= mario_location[3]):
# right
posX = entity[1]
else:
# centered on Mario
posX = self.n_WIDTH * self.WIDTH
if (entity[0] >= mario_location[2]):
# bottom
posY = entity[0]
elif (entity[2] <= mario_location[0]):
# up
posY = entity[2]
else:
# centered on Mario
posY = self.n_HEIGHT * self.HEIGHT
units_array[int(posY / self.HEIGHT)][int(
posX / self.WIDTH)] = self.MOVING_ENTITY
return units_array
def _multiple_locate(self,
sprite=None,
game_frame=None,
screen_region=None,
use_global_location=True,
forced=False):
constellation_of_pixel_images = sprite.generate_constellation_of_pixels_images(
)
locations = []
frame = game_frame.frame
if screen_region is not None:
frame = serpent.cv.extract_region_from_image(frame, screen_region)
for i in range(len(constellation_of_pixel_images)):
constellation_of_pixels_item = list(
sprite.constellation_of_pixels[i].items())[0]
query_coordinates = constellation_of_pixels_item[0]
query_rgb = constellation_of_pixels_item[1]
rgb_coordinates = Sprite.locate_color(query_rgb, image=frame)
rgb_coordinates = list(
map(
lambda yx: (yx[0] - query_coordinates[0], yx[1] -
query_coordinates[1]), rgb_coordinates))
maximum_y = frame.shape[0] - constellation_of_pixel_images[
i].shape[0]
maximum_x = frame.shape[1] - constellation_of_pixel_images[
i].shape[1]
for y, x in rgb_coordinates:
if y < 0 or x < 0 or y > maximum_y or x > maximum_x:
continue
for yx, rgb in sprite.constellation_of_pixels[i].items():
if tuple(frame[y + yx[0], x + yx[1], :]) != rgb:
break
else:
locations.append(
(y, x, y + constellation_of_pixel_images[i].shape[0],
x + constellation_of_pixel_images[i].shape[1]))
if (forced and len(locations) > 0):
break
if len(locations
) != 0 and screen_region is not None and use_global_location:
for i in range(0, len(locations)):
locations[i] = (locations[i][0] + screen_region[0],
locations[i][1] + screen_region[1],
locations[i][2] + screen_region[0],
locations[i][3] + screen_region[1])
if len(locations) > 0:
locations = self._epurate(locations)
return locations
def _epurate(self, locations):
temp = []
epurated = []
for i in range(0, len(locations)):
if (not locations[i][0] in temp and not locations[i][0] - 1 in temp
and not locations[i][0] + 1 in temp):
temp.append(locations[i][0])
epurated.append(locations[i])
return epurated
def get_score_value(self, location):
values = [-1000, -1000, -1000, -1000, -1000]
if (location != None):
values[0] = self._get_level(location[0])
ladders = self._get_ladders(location[0])
minimum_distance = int(abs(ladders[0] - location[3]))
for ladder in ladders:
if (abs(ladder - location[1]) <= self.LADDER_DELTA):
distance = 0
elif (ladder < location[1]):
distance = int(abs(ladder - location[1]))
else:
distance = int(abs(ladder - location[3]))
if (distance < minimum_distance):
minimum_distance = distance
values[1] = -1 * minimum_distance
values[2] = self.max_stage
values[3] = self.useless_actions
if (self._is_on_ladder(values[0], location)):
values[4] = 1
else:
values[4] = 0
self.last_change_stage = 0
self.last_stage = 0
self.max_stage = 0
self.useless_actions = 0
return values
def _is_on_ladder(self, level, location):
ladders = self.ladders_positions[level]
for ladder in ladders:
if (abs(ladder - location[1]) <= self.LADDER_DELTA):
couple = (level, ladder)
if (couple in self.on_ladders):
extremity = self.on_ladders[couple]
if (location[0] <= extremity[0]
and location[0] > extremity[1]):
return True
return False
def get_death_location(self, game_frame):
return self.sprite_locator.locate(sprite=self.death_sprite,
game_frame=game_frame)
def analyze_action(self, inputs, outputs):
### FILTRE LES JUMPS INUTILES
# N = self.n_WIDTH * 2 + 1
# row = self.n_HEIGHT * N
# has_obstacle = False
# for i in range(N):
# if(inputs[row+i] == self.MOVING_ENTITY):
# has_obstacle = True
# break;
# if (not has_obstacle and outputs[len(outputs)-1] == 1):
# self.useless_actions = self.useless_actions + 1
if (outputs[len(outputs) - 1] == 1):
self.useless_actions = self.useless_actions + 1
"""
Methods to use the NavigationGame FSM
"""
def not_running(self):
return self.navigationGameFSM.current == "not_running"
def is_in_menu(self):
return self.navigationGameFSM.current == "menu"
def is_playing(self):
return self.navigationGameFSM.current == "playing"
def is_dead(self):
return self.navigationGameFSM.current == "dead"
def has_won(self):
return self.navigationGameFSM.current == "has_won"
def run(self):
self.navigationGameFSM.run()
def next(self):
self.navigationGameFSM.next()
def win(self):
self.navigationGameFSM.win()
class MyAPINamespace:
@classmethod
def my_namespaced_api_function(cls):
api = DonkeyKongAPI.instance
def handle_play(self, game_frame):
#self.printer.add("")
#self.printer.add("BombermanAI")
#self.printer.add("Reinforcement Learning: Training a PPO Agent")
#self.printer.add("")
#self.printer.add(f"Stage Started At: {self.started_at}")
#self.printer.add(f"Current Run: #{self.current_attempts}")
#self.printer.add("")
#self.check_game_state(game_frame)
#####################CHECK STATE###########################
#game over?
locationGO = None
sprite_to_locate = Sprite("QUERY", image_data=self.spriteGO.image_data)
sprite_locator = SpriteLocator()
locationGO = sprite_locator.locate(sprite=sprite_to_locate,
game_frame=game_frame)
#print("Location Game over:",locationGO)
#won game?
locationWO = None
sprite_to_locate = Sprite("QUERY", image_data=self.spriteWO.image_data)
sprite_locator = SpriteLocator()
locationWO = sprite_locator.locate(sprite=sprite_to_locate,
game_frame=game_frame)
#print("Location Game won:",locationWO)
self.gamestate.victory = locationWO != None
self.gamestate.lose = locationGO != None
self.gamestate.girl_alive = (locationGO == None and locationWO == None)
self.gamestate.done = not self.gamestate.girl_alive
print(f"Is alive? {self.gamestate.girl_alive}")
print(f"Game over? {self.gamestate.lose}")
print(f"Won? {self.gamestate.victory}")
#####################VISUAL DEBUGGER###########################
for i, game_frame in enumerate(self.game_frame_buffer.frames):
self.visual_debugger.store_image_data(game_frame.frame,
game_frame.frame.shape,
str(i))
#####################MODEL###########################
#get buffer
frame_buffer = FrameGrabber.get_frames([0, 1, 2, 3],
frame_type="PIPELINE")
game_frame_buffer = self.extract_game_area(frame_buffer)
state = game_frame_buffer.reshape(4, 104, 136, 1)
if (self.gamestate.done):
print(f"Game over, attemp {self.epoch}")
if (self.epoch % 10) == 0:
print("saving model")
self.dqn_agent.save_model(
f"bombergirl_epoch_{self.epoch}.model")
self.printer.save_file()
self.printer.add(
f"{self.gamestate.victory},{self.gamestate.lose},{self.epoch},{self.gamestate.time},{self.total_reward}"
)
self.total_reward = 0
self.dqn_agent.remember(self.prev_state, self.prev_action,
self.prev_reward, state, True)
self.dqn_agent.replay()
self.input_controller.tap_key(KeyboardKey.KEY_ENTER)
self.epoch += 1
self.total_reward = 0
self.gamestate.restartState()
self.prev_state = None
self.prev_action = None
else:
#update time
self.gamestate.updateTime()
#print(np.stack(game_frame_buffer,axis=1).shape)
#print(game_frame_buffer.shape)
#print(state.shape)
if (not (self.prev_state is None)
and not (self.prev_action is None)):
self.dqn_agent.remember(self.prev_state, self.prev_action,
self.prev_reward, state, False)
#do something
action_index = self.dqn_agent.act(state)
#get key
action = self.game_actions[action_index]
#get random frame from buffer
game_frame_rand = random.choice(frame_buffer.frames).frame
#update enviroment accorind to frame
###################FUN UPDATE STATE#########################################
game_area = \
serpent.cv.extract_region_from_image(game_frame_rand,self.game.screen_regions['GAME_REGION'])
#game ...
# 0,0
# 32,32
game_squares = [[None for j in range(0, 11)] for i in range(0, 15)]
const_offset = 8
const = 32
#game variables
self.gamestate.bombs = [] #{x, y}
self.gamestate.enemies = [] #{x,y}
#force girl to die if not found
girl_found = False
for i in range(0, 15):
for j in range(0, 11):
izq = ((j + 1) * const - const_offset,
(i + 1) * const - const_offset)
der = ((j + 2) * const + const_offset,
(i + 2) * const + const_offset)
reg = (izq[0], izq[1], der[0], der[1])
square = serpent.cv.extract_region_from_image(
game_area, reg)
square = self.convert_to_rgba(square)
sprite_to_locate = Sprite("QUERY",
image_data=square[...,
np.newaxis])
sprite = self.sprite_identifier.identify(
sprite_to_locate, mode="SIGNATURE_COLORS")
game_squares[i][j] = sprite
if ("SPRITE_BETTY" in sprite):
self.girl = {"x": i, "y": j}
girl_found = True
elif ("SPRITE_GEORGE" in sprite):
self.gamestate.enemies.append({"x": i, "y": j})
elif ("SPRITE_BOMB" in sprite):
self.gamestate.bombs.append({"x": i, "y": j})
elif ("SPRITE_BONUSES" in sprite):
self.gamestate.bonus.append({"x": i, "y": j})
#####################CHECK STATE###########################
#game over?
locationGO = None
sprite_to_locate = Sprite("QUERY",
image_data=self.spriteGO.image_data)
sprite_locator = SpriteLocator()
locationGO = sprite_locator.locate(sprite=sprite_to_locate,
game_frame=game_frame)
#print("Location Game over:",locationGO)
#won game?
locationWO = None
sprite_to_locate = Sprite("QUERY",
image_data=self.spriteWO.image_data)
sprite_locator = SpriteLocator()
locationWO = sprite_locator.locate(sprite=sprite_to_locate,
game_frame=game_frame)
#print("Location Game won:",locationWO)
self.gamestate.lose = locationGO != None
self.gamestate.victory = locationWO != None
self.gamestate.girl_alive = (locationGO == None
and locationWO == None)
self.gamestate.done = not self.gamestate.girl_alive
print(f"Is alive? {self.gamestate.girl_alive}")
print(f"Game over? {self.gamestate.lose}")
print(f"Won? {self.gamestate.victory}")
###################REWARD#########################################
#get reward
reward = self.gamestate.getReward(action_index)
self.total_reward += reward
self.prev_state = state
self.prev_action = action_index
self.prev_reward = reward
if (action):
self.input_controller.tap_key(
action, 0.15 if action_index < 4 else 0.01)
print(
f"Action: {self.gamestate.game_inputs[action_index]}, reward: {reward}, total_reward: {self.total_reward}"
)
def enemy_action_capture(self, game_frame):
final_cultist_attack = []
attack_cultist_temp_list= []
# Unselect anything just incase
self.input_controller.click(button=MouseButton.RIGHT, duration=0.25)
time.sleep(.5)
# Home hover
self.input_controller.move(x=636, y=375, duration=0.25, absolute=True)
time.sleep(1)
# Enemy hover
self.input_controller.move(x=959, y=410, duration=0.25, absolute=True)
time.sleep(.75)
image_data = skimage.io.imread("plugins/SerpentSlayTheSpireGamePlugin/files/data/sprites/sprite_Attack_for_0.png")[..., np.newaxis]
attack_for_cultist = Sprite("attack_for_cultist", image_data=image_data)
# Full game frame capture
# print("-----------Full game frame capture-----------")
# full_game_frame = FrameGrabber.get_frames(
# [0],
# frame_shape=(self.game.frame_height, self.game.frame_width),
# frame_type="PIPELINE"
# ).frames[0]
# Allows for dynamic capture of enemy attack
sprite_locator = SpriteLocator()
attack_for_cultist_location = sprite_locator.locate(sprite=attack_for_cultist, game_frame=game_frame)
print("attack_for_cultist_location: ", attack_for_cultist_location)
# Tuples are immutable :(
if (attack_for_cultist_location != None):
attack_cultist_temp_list = list(attack_for_cultist_location)
attack_cultist_temp_list[1] = attack_cultist_temp_list[1] + 45
attack_cultist_temp_list[3] = attack_cultist_temp_list[3] + 15
attack_for_cultist_location = tuple(attack_cultist_temp_list)
print("Updated - attack_for_cultist_location: ", attack_for_cultist_location)
time.sleep(1)
cultist_attack = serpent.cv.extract_region_from_image(game_frame.frame, attack_for_cultist_location)
cultist_attack_grayscale = np.array(skimage.color.rgb2gray(cultist_attack) * 255, dtype="uint8")
cultist_attack = serpent.ocr.perform_ocr(image=cultist_attack_grayscale, scale=15, order=5, horizontal_closing=2, vertical_closing=1)
# This is actually an awkward work around for limitations in how tesseract works. By default it doesn't capture single char values so when dynamically
# searching and capturing the enemy attack the region it's looking for the region that includes the word "for " + attack value (i.e. "for 6"). There
# are ways of swapping the mode of tesseract to do a capture for single char values but because the attack values are dynamic it sometimes is
# less than 10 or much greater than 10 which is now multiple char's and messes with the capture. For the sake of just getting it working I did this
# TLDR: Awkward workaround for limitation in tesseract when capturing single char values. Likely easier way to capture then parse attack value
for elem in cultist_attack:
if (elem.isdigit() == True):
final_cultist_attack.append(elem)
print("final_cultist_attack", final_cultist_attack)
final_cultist_attack = ''.join(final_cultist_attack)
print("final_cultist_attack: ", final_cultist_attack)
print("------------------------------------")
self.game_state["final_cultist_attack"].insert(0, final_cultist_attack)
self.poison_check(game_frame)
else:
return print("Failed to capture enemy attack")
class InputController:
def __init__(self, game=None):
self.game = game
self.mouse_buttons = {
MouseButton.LEFT.name: "left",
MouseButton.MIDDLE.name: "middle",
MouseButton.RIGHT.name: "right"
}
self.previous_key_collection_set = set()
self.sprite_locator = SpriteLocator()
@property
def game_is_focused(self):
return self.game.is_focused
# Keyboard Actions
def handle_keys(self, key_collection):
key_collection_set = set(key_collection)
keys_to_press = key_collection_set - self.previous_key_collection_set
keys_to_release = self.previous_key_collection_set - key_collection_set
for key in keys_to_press:
self.press_key(key)
for key in keys_to_release:
self.release_key(key)
self.previous_key_collection_set = key_collection_set
def tap_keys(self, keys, duration=0.05):
if self.game_is_focused:
for key in keys:
pyautogui.keyDown(key)
time.sleep(duration)
for key in keys:
pyautogui.keyUp(key)
def tap_key(self, key, duration=0.05):
if self.game_is_focused:
pyautogui.keyDown(key)
time.sleep(duration)
pyautogui.keyUp(key)
def press_keys(self, keys):
for key in keys:
self.press_key(key)
def press_key(self, key):
if self.game_is_focused:
pyautogui.keyDown(key)
def release_keys(self):
for key in self.previous_key_collection_set:
self.release_key(key)
self.previous_key_collection_set = set()
def release_key(self, key):
if self.game_is_focused:
pyautogui.keyUp(key)
def type_string(self, string, duration=0.05):
if self.game_is_focused:
pyautogui.typewrite(message=string, interval=duration)
# Mouse Actions
def click(self, button=MouseButton.LEFT, y=None, x=None, duration=0.25):
if self.game_is_focused:
pyautogui.moveTo(x, y, duration=duration)
pyautogui.click(button=self.mouse_buttons.get(button.name, "left"))
def click_screen_region(self, button=MouseButton.LEFT, screen_region=None):
if self.game_is_focused:
screen_region_coordinates = self.game.screen_regions.get(
screen_region)
x = (screen_region_coordinates[1] +
screen_region_coordinates[3]) // 2
x += self.game.window_geometry["x_offset"]
y = (screen_region_coordinates[0] +
screen_region_coordinates[2]) // 2
y += self.game.window_geometry["y_offset"]
self.click(button=button, y=y, x=x)
def click_sprite(self,
button=MouseButton.LEFT,
sprite=None,
game_frame=None):
if self.game_is_focused:
sprite_location = self.sprite_locator.locate(sprite=sprite,
game_frame=game_frame)
if sprite_location is None:
return False
x = (sprite_location[1] + sprite_location[3]) // 2
x += self.game.window_geometry["x_offset"]
y = (sprite_location[0] + sprite_location[2]) // 2
y += self.game.window_geometry["y_offset"]
self.click(button=button, y=y, x=x)
return True
def click_string(self,
query_string,
button=MouseButton.LEFT,
game_frame=None,
fuzziness=2,
ocr_preset=None):
if self.game_is_focused:
string_location = serpent.ocr.locate_string(
query_string,
game_frame.frame,
fuzziness=fuzziness,
ocr_preset=ocr_preset,
offset_x=game_frame.offset_x,
offset_y=game_frame.offset_y)
if string_location is not None:
x = (string_location[1] + string_location[3]) // 2
x += self.game.window_geometry["x_offset"]
y = (string_location[0] + string_location[2]) // 2
y += self.game.window_geometry["y_offset"]
self.click(button=button, y=y, x=x)
return True
return False
def drag(self,
button=MouseButton.LEFT,
x0=None,
y0=None,
x1=None,
y1=None,
duration=1):
if self.game_is_focused:
pyautogui.moveTo(x0, y0, duration=0.2)
pyautogui.dragTo(x1, y1, button=button, duration=duration)
def drag_screen_region_to_screen_region(self,
button=MouseButton.LEFT,
start_screen_region=None,
end_screen_region=None,
duration=1):
if self.game_is_focused:
start_screen_region_coordinates = self._extract_screen_region_coordinates(
start_screen_region)
end_screen_region_coordinates = self._extract_screen_region_coordinates(
end_screen_region)
self.drag(button=button,
x0=start_screen_region_coordinates[0],
y0=start_screen_region_coordinates[1],
x1=end_screen_region_coordinates[0],
y1=end_screen_region_coordinates[1],
duration=duration)
def scroll(self, y=None, x=None, clicks=1, direction="DOWN"):
if self.game_is_focused:
clicks = clicks * (1 if direction == "DOWN" else -1)
pyautogui.scroll(clicks, x=x, y=y)
def _extract_screen_region_coordinates(self, screen_region):
screen_region_coordinates = self.game.screen_regions.get(screen_region)
x = (screen_region_coordinates[1] + screen_region_coordinates[3]) // 2
x += self.game.window_geometry["x_offset"]
y = (screen_region_coordinates[0] + screen_region_coordinates[2]) // 2
y += self.game.window_geometry["y_offset"]
return x, y