def poison_check(self, game_frame):
image_data = skimage.io.imread("plugins/SerpentSlayTheSpireGamePlugin/files/data/sprites/sprite_poison_check_0.png")[..., np.newaxis]
poison_check = Sprite("poison_check", image_data=image_data)
# # Full game frame capture
# print("-----------Full game frame capture-----------")
# full_game_frame = FrameGrabber.get_frames(
# [0],
# frame_shape=(self.game.frame_height, self.game.frame_width),
# frame_type="PIPELINE"
# ).frames[0]
sprite_locator = SpriteLocator()
poison_check_location = sprite_locator.locate(sprite=poison_check, game_frame=game_frame)
print("poison_check_location: ", poison_check_location)
if (poison_check_location != None):
self.game_state["poison_check"].insert(0, True)
print("POISON_CHECK == TRUE")
else:
self.game_state["poison_check"].insert(0, False)
print("POISON_CHECK == FALSE")
self.ddqn_setup(game_frame)
def _measure_actor_hp(self, game_frame):
hp_area_frame = serpent.cv.extract_region_from_image(
game_frame.frame, self.game.screen_regions["HP_AREA"])
hp_area_image = Image.fromarray(hp_area_frame)
actor_hp = 0
image_colors = hp_area_image.getcolors(
) # TODO: remove in favor of sprite detection and location
if image_colors:
actor_hp = len(image_colors) - 7
for name, sprite in self.game.sprites.items():
query_sprite = Sprite("QUERY", image_data=sprite.image_data)
sprite_name = self.sprite_identifier.identify(
query_sprite, mode="CONSTELLATION_OF_PIXELS"
) # Will be "UNKNOWN" if no match
print(sprite_name)
sprite_to_locate = Sprite("QUERY", image_data=sprite.image_data)
sprite_locator = SpriteLocator()
location = sprite_locator.locate(sprite=sprite_to_locate,
game_frame=game_frame)
print(location)
if location:
actor_hp = 1000000
return actor_hp
def identify_sprite(self, sprite, game_frame):
# sprite_identifier = SpriteIdentifier()
sprite_identify = self.sprite_identifier.identify(
sprite=sprite, mode="SIGNATURE_COLORS")
sprite_locator = SpriteLocator()
location = sprite_locator.locate(sprite=sprite, game_frame=game_frame)
output = {'sprite_name': sprite_identify, 'location': location}
return output
def __init__(self, game=None):
self.game = game
self.mouse_buttons = {
MouseButton.LEFT.name: "left",
MouseButton.MIDDLE.name: "middle",
MouseButton.RIGHT.name: "right"
}
self.previous_key_collection_set = set()
self.sprite_locator = SpriteLocator()
def __init__(self, game=None):
super().__init__(game=game)
self.navigationGameFSM = NavigationGameFSM()
self._prepare_sprites()
self.WIDTH = 34
self.HEIGHT = 32
self.n_WIDTH = 2
self.n_HEIGHT = 2
self.MARIO = 1
self.MOVING_ENTITY = 2
self.LADDER = 3
self.LADDER_DELTA = 18
self.LEVEL_WIN_HEIGHT = 32
self.sprite_locator = SpriteLocator()
self.ladders_positions = dict()
self.ladders_positions[0] = [453, 10000]
self.ladders_positions[1] = [102, 249]
self.ladders_positions[2] = [285, 452]
self.ladders_positions[3] = [104, 194]
self.ladders_positions[4] = [452, 10000]
self.ladders_positions[5] = [324, 10000]
self.on_ladders = dict()
self.on_ladders[(0, 453)] = [374, 328]
self.on_ladders[(1, 102)] = [312, 268]
self.on_ladders[(1, 249)] = [318, 262]
self.on_ladders[(2, 285)] = [258, 202]
self.on_ladders[(2, 452)] = [252, 208]
self.on_ladders[(3, 104)] = [192, 148]
self.on_ladders[(3, 194)] = [196, 144]
self.on_ladders[(4, 452)] = [133, 88]
self.on_ladders[(5, 324)] = [81, 32]
self.last_stage = 0
self.max_stage = 0
self.last_change_stage = 0
self.useless_actions = 0
self.level_direction = dict()
orientation = 1
for i in range(6):
self.level_direction[i] = orientation
orientation = orientation * (-1)
self.ladders_thresholds = [384, 328, 268, 208, 148, 88]
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.frame_handlers["PLAY"] = self.handle_play
self.frame_handler_setups["PLAY"] = self.setup_play
self.printer = TerminalPrinter()
self.sprite_locator = SpriteLocator()
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.frame_handlers["PLAY"] = self.handle_play
self.frame_handler_setups["PLAY"] = self.setup_play
self.sprite_locator = SpriteLocator()
self.game_state = None
self._reset_game_state()
def _measure_actor_hp(self, game_frame):
hp_area_frame = serpent.cv.extract_region_from_image(
game_frame.frame, self.game.screen_regions["HP_AREA"])
hp_area_image = Image.fromarray(hp_area_frame)
actor_hp = 0
image_colors = hp_area_image.getcolors(
) # TODO: remove in favor of sprite detection and location
if image_colors:
actor_hp = len(image_colors) - 7
for name, sprite in self.game.sprites.items():
sprite_to_locate = Sprite("QUERY", image_data=sprite.image_data)
sprite_locator = SpriteLocator()
location = sprite_locator.locate(sprite=sprite_to_locate,
game_frame=game_frame)
print(location)
if location:
actor_hp = 1000000
return actor_hp
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.frame_handlers["PLAY"] = self.handle_play
self.frame_handlers["PLAY_BOT"] = self.handle_play_bot
self.frame_handler_setups["PLAY"] = self.setup_play
self.frame_handler_setups["PLAY_BOT"] = self.setup_play
self.analytics_client = None
self.sprite_locator = SpriteLocator()
self.username_entered = False
def check_game_state(self, game_frame):
#game over?
locationGO = None
sprite_to_locate = Sprite("QUERY", image_data=self.spriteGO.image_data)
sprite_locator = SpriteLocator()
locationGO = sprite_locator.locate(sprite=sprite_to_locate,
game_frame=game_frame)
print("Location Game over:", locationGO)
#won game?
locationWO = None
sprite_to_locate = Sprite("QUERY", image_data=self.spriteWO.image_data)
sprite_locator = SpriteLocator()
locationWO = sprite_locator.locate(sprite=sprite_to_locate,
game_frame=game_frame.frames)
print("Location Game won:", locationWO)
self.gamestate.girl_alive = (locationGO == None and locationWO == None)
self.gamestate.done = not self.gamestate.girl_alive
self.gamestate.victory = locationWO != None
print(f"Is alive? {self.gamestate.girl_alive}")
print(f"Game over? {self.gamestate.lose}")
print(f"Won? {self.gamestate.victory}")
def find_sprite(self, sprite_name):
sprite_locator = SpriteLocator()
location = sprite_locator.locate(sprite=self.game.sprites[sprite_name],
game_frame=self.game_frame)
return location # None if not found
class NativeWin32InputController(InputController):
def __init__(self, game=None, **kwargs):
self.game = game
self.previous_key_collection_set = set()
self.sprite_locator = SpriteLocator()
# Keyboard Actions
def handle_keys(self, key_collection, **kwargs):
key_collection_set = set(key_collection)
keys_to_press = key_collection_set - self.previous_key_collection_set
keys_to_release = self.previous_key_collection_set - key_collection_set
for key in keys_to_press:
self.press_key(key, **kwargs)
for key in keys_to_release:
self.release_key(key, **kwargs)
self.previous_key_collection_set = key_collection_set
def tap_keys(self, keys, duration=0.05, **kwargs):
if ("force" in kwargs
and kwargs["force"] is True) or self.game_is_focused:
for key in keys:
self.press_key(key, **kwargs)
time.sleep(duration)
for key in keys:
self.release_key(key, **kwargs)
def tap_key(self, key, duration=0.05, **kwargs):
if ("force" in kwargs
and kwargs["force"] is True) or self.game_is_focused:
self.press_key(key, **kwargs)
time.sleep(duration)
self.release_key(key, **kwargs)
def press_keys(self, keys, **kwargs):
for key in keys:
self.press_key(key, **kwargs)
def press_key(self, key, **kwargs):
if ("force" in kwargs
and kwargs["force"] is True) or self.game_is_focused:
extra = ctypes.c_ulong(0)
ii_ = Input_I()
if keyboard_key_mapping[key.name] >= 1024:
key = keyboard_key_mapping[key.name] - 1024
flags = 0x0008 | 0x0001
else:
key = keyboard_key_mapping[key.name]
flags = 0x0008
ii_.ki = KeyBdInput(0, key, flags, 0, ctypes.pointer(extra))
x = Input(ctypes.c_ulong(1), ii_)
ctypes.windll.user32.SendInput(1, ctypes.pointer(x),
ctypes.sizeof(x))
def release_keys(self, keys, **kwargs):
for key in keys:
self.release_key(key, **kwargs)
def release_key(self, key, **kwargs):
if ("force" in kwargs
and kwargs["force"] is True) or self.game_is_focused:
extra = ctypes.c_ulong(0)
ii_ = Input_I()
if keyboard_key_mapping[key.name] >= 1024:
key = keyboard_key_mapping[key.name] - 1024
flags = 0x0008 | 0x0001 | 0x0002
else:
key = keyboard_key_mapping[key.name]
flags = 0x0008 | 0x0002
ii_.ki = KeyBdInput(0, key, flags, 0, ctypes.pointer(extra))
x = Input(ctypes.c_ulong(1), ii_)
ctypes.windll.user32.SendInput(1, ctypes.pointer(x),
ctypes.sizeof(x))
def type_string(self, string, duration=0.05, **kwargs):
if ("force" in kwargs
and kwargs["force"] is True) or self.game_is_focused:
for character in string:
keys = character_keyboard_key_mapping.get(character)
if keys is not None:
self.tap_keys(keys, duration=duration, **kwargs)
# Mouse Actions
def move(self, x=None, y=None, duration=0.25, absolute=True, **kwargs):
if ("force" in kwargs
and kwargs["force"] is True) or self.game_is_focused:
x += self.game.window_geometry["x_offset"]
y += self.game.window_geometry["y_offset"]
current_pixel_coordinates = win32api.GetCursorPos()
start_coordinates = self._to_windows_coordinates(
*current_pixel_coordinates)
if absolute:
end_coordinates = self._to_windows_coordinates(x, y)
else:
end_coordinates = self._to_windows_coordinates(
current_pixel_coordinates[0] + x,
current_pixel_coordinates[1] + y)
coordinates = self._interpolate_mouse_movement(
start_windows_coordinates=start_coordinates,
end_windows_coordinates=end_coordinates)
for x, y in coordinates:
extra = ctypes.c_ulong(0)
ii_ = Input_I()
ii_.mi = MouseInput(x, y, 0, (0x0001 | 0x8000), 0,
ctypes.pointer(extra))
x = Input(ctypes.c_ulong(0), ii_)
ctypes.windll.user32.SendInput(1, ctypes.pointer(x),
ctypes.sizeof(x))
time.sleep(duration / 20)
def click_down(self, button=MouseButton.LEFT, **kwargs):
if ("force" in kwargs
and kwargs["force"] is True) or self.game_is_focused:
extra = ctypes.c_ulong(0)
ii_ = Input_I()
ii_.mi = MouseInput(0, 0, 0,
mouse_button_down_mapping[button.name], 0,
ctypes.pointer(extra))
x = Input(ctypes.c_ulong(0), ii_)
ctypes.windll.user32.SendInput(1, ctypes.pointer(x),
ctypes.sizeof(x))
def click_up(self, button=MouseButton.LEFT, **kwargs):
if ("force" in kwargs
and kwargs["force"] is True) or self.game_is_focused:
extra = ctypes.c_ulong(0)
ii_ = Input_I()
ii_.mi = MouseInput(0, 0, 0, mouse_button_up_mapping[button.name],
0, ctypes.pointer(extra))
x = Input(ctypes.c_ulong(0), ii_)
ctypes.windll.user32.SendInput(1, ctypes.pointer(x),
ctypes.sizeof(x))
def click(self, button=MouseButton.LEFT, duration=0.05, **kwargs):
if ("force" in kwargs
and kwargs["force"] is True) or self.game_is_focused:
self.click_down(button=button, **kwargs)
time.sleep(duration)
self.click_up(button=button, **kwargs)
def click_screen_region(self,
button=MouseButton.LEFT,
screen_region=None,
**kwargs):
if ("force" in kwargs
and kwargs["force"] is True) or self.game_is_focused:
screen_region_coordinates = self.game.screen_regions.get(
screen_region)
x = (screen_region_coordinates[1] +
screen_region_coordinates[3]) // 2
y = (screen_region_coordinates[0] +
screen_region_coordinates[2]) // 2
self.move(x, y)
self.click(button=button, **kwargs)
def click_sprite(self,
button=MouseButton.LEFT,
sprite=None,
game_frame=None,
**kwargs):
if ("force" in kwargs
and kwargs["force"] is True) or self.game_is_focused:
sprite_location = self.sprite_locator.locate(sprite=sprite,
game_frame=game_frame)
if sprite_location is None:
return False
x = (sprite_location[1] + sprite_location[3]) // 2
y = (sprite_location[0] + sprite_location[2]) // 2
self.move(x, y)
self.click(button=button, **kwargs)
return True
# Requires the Serpent OCR module
def click_string(self,
query_string,
button=MouseButton.LEFT,
game_frame=None,
fuzziness=2,
ocr_preset=None,
**kwargs):
import serpent.ocr
if ("force" in kwargs
and kwargs["force"] is True) or self.game_is_focused:
string_location = serpent.ocr.locate_string(
query_string,
game_frame.frame,
fuzziness=fuzziness,
ocr_preset=ocr_preset,
offset_x=game_frame.offset_x,
offset_y=game_frame.offset_y)
if string_location is not None:
x = (string_location[1] + string_location[3]) // 2
y = (string_location[0] + string_location[2]) // 2
self.move(x, y)
self.click(button=button, **kwargs)
return True
return False
def drag(self,
button=MouseButton.LEFT,
x0=None,
y0=None,
x1=None,
y1=None,
duration=0.25,
**kwargs):
if ("force" in kwargs
and kwargs["force"] is True) or self.game_is_focused:
self.move(x=x0, y=y0)
self.click_down(button=button)
self.move(x=x1, y=y1, duration=duration)
self.click_up(button=button)
def drag_screen_region_to_screen_region(self,
button=MouseButton.LEFT,
start_screen_region=None,
end_screen_region=None,
duration=1,
**kwargs):
if ("force" in kwargs
and kwargs["force"] is True) or self.game_is_focused:
start_screen_region_coordinates = self._extract_screen_region_coordinates(
start_screen_region)
end_screen_region_coordinates = self._extract_screen_region_coordinates(
end_screen_region)
self.drag(button=button,
x0=start_screen_region_coordinates[0],
y0=start_screen_region_coordinates[1],
x1=end_screen_region_coordinates[0],
y1=end_screen_region_coordinates[1],
duration=duration,
**kwargs)
def scroll(self, clicks=1, direction="DOWN", **kwargs):
if ("force" in kwargs
and kwargs["force"] is True) or self.game_is_focused:
clicks = clicks * (1 if direction == "UP" else -1) * 120
extra = ctypes.c_ulong(0)
ii_ = Input_I()
ii_.mi = MouseInput(0, 0, clicks, 0x0800, 0, ctypes.pointer(extra))
x = Input(ctypes.c_ulong(0), ii_)
ctypes.windll.user32.SendInput(1, ctypes.pointer(x),
ctypes.sizeof(x))
@staticmethod
def _to_windows_coordinates(x=0, y=0):
display_width = win32api.GetSystemMetrics(0)
display_height = win32api.GetSystemMetrics(1)
windows_x = (x * 65535) // display_width
windows_y = (y * 65535) // display_height
return windows_x, windows_y
@staticmethod
def _interpolate_mouse_movement(start_windows_coordinates,
end_windows_coordinates,
steps=20):
x_coordinates = [
start_windows_coordinates[0], end_windows_coordinates[0]
]
y_coordinates = [
start_windows_coordinates[1], end_windows_coordinates[1]
]
if x_coordinates[0] == x_coordinates[1]:
x_coordinates[1] += 1
if y_coordinates[0] == y_coordinates[1]:
y_coordinates[1] += 1
interpolation_func = scipy.interpolate.interp1d(
x_coordinates, y_coordinates)
intermediate_x_coordinates = np.linspace(start_windows_coordinates[0],
end_windows_coordinates[0],
steps + 1)[1:]
coordinates = list(
map(lambda x: (int(round(x)), int(interpolation_func(x))),
intermediate_x_coordinates))
return coordinates
class PyAutoGUIInputController(InputController):
def __init__(self, game=None, **kwargs):
self.game = game
self.previous_key_collection_set = set()
self.sprite_locator = SpriteLocator()
# Keyboard Actions
def handle_keys(self, key_collection, **kwargs):
key_collection_set = set(key_collection)
keys_to_press = key_collection_set - self.previous_key_collection_set
keys_to_release = self.previous_key_collection_set - key_collection_set
for key in keys_to_press:
self.press_key(key, **kwargs)
for key in keys_to_release:
self.release_key(key, **kwargs)
self.previous_key_collection_set = key_collection_set
def tap_keys(self, keys, duration=0.05, **kwargs):
if ("force" in kwargs and kwargs["force"] is True) or self.game_is_focused:
for key in keys:
self.press_key(key, **kwargs)
time.sleep(duration)
for key in keys:
self.release_key(key, **kwargs)
def tap_key(self, key, duration=0.05, **kwargs):
if ("force" in kwargs and kwargs["force"] is True) or self.game_is_focused:
self.press_key(key, **kwargs)
time.sleep(duration)
self.release_key(key, **kwargs)
def press_keys(self, keys, **kwargs):
for key in keys:
self.press_key(key, **kwargs)
def press_key(self, key, **kwargs):
if ("force" in kwargs and kwargs["force"] is True) or self.game_is_focused:
pyautogui.keyDown(keyboard_key_mapping[key.name])
def release_keys(self, keys, **kwargs):
for key in keys:
self.release_key(key, **kwargs)
def release_key(self, key, **kwargs):
if ("force" in kwargs and kwargs["force"] is True) or self.game_is_focused:
pyautogui.keyUp(keyboard_key_mapping[key.name])
def type_string(self, string, duration=0.05, **kwargs):
if ("force" in kwargs and kwargs["force"] is True) or self.game_is_focused:
pyautogui.typewrite(message=string, interval=duration)
# Mouse Actions
def move(self, x=None, y=None, duration=0.25, absolute=True, **kwargs):
if ("force" in kwargs and kwargs["force"] is True) or self.game_is_focused:
x += self.game.window_geometry["x_offset"]
y += self.game.window_geometry["y_offset"]
if absolute:
pyautogui.moveTo(x=x, y=y, duration=duration)
else:
pyautogui.moveRel(xOffset=x, yOffset=y, duration=duration)
def click_down(self, button=MouseButton.LEFT, **kwargs):
if ("force" in kwargs and kwargs["force"] is True) or self.game_is_focused:
pyautogui.mouseDown(button=mouse_button_mapping[button.name])
def click_up(self, button=MouseButton.LEFT, **kwargs):
if ("force" in kwargs and kwargs["force"] is True) or self.game_is_focused:
pyautogui.mouseUp(button=mouse_button_mapping[button.name])
def click(self, button=MouseButton.LEFT, duration=0.25, **kwargs):
if ("force" in kwargs and kwargs["force"] is True) or self.game_is_focused:
self.click_down(button=button, **kwargs)
time.sleep(duration)
self.click_up(button=button, **kwargs)
def click_screen_region(self, button=MouseButton.LEFT, screen_region=None, **kwargs):
if ("force" in kwargs and kwargs["force"] is True) or self.game_is_focused:
screen_region_coordinates = self.game.screen_regions.get(screen_region)
x = (screen_region_coordinates[1] + screen_region_coordinates[3]) // 2
y = (screen_region_coordinates[0] + screen_region_coordinates[2]) // 2
self.move(x=x, y=y)
self.click(button=button, **kwargs)
def click_sprite(self, button=MouseButton.LEFT, sprite=None, game_frame=None, **kwargs):
if ("force" in kwargs and kwargs["force"] is True) or self.game_is_focused:
sprite_location = self.sprite_locator.locate(sprite=sprite, game_frame=game_frame)
if sprite_location is None:
return False
x = (sprite_location[1] + sprite_location[3]) // 2
y = (sprite_location[0] + sprite_location[2]) // 2
self.move(x=x, y=y)
self.click(button=button, **kwargs)
return True
# Requires the Serpent OCR module
def click_string(self, query_string, button=MouseButton.LEFT, game_frame=None, fuzziness=2, ocr_preset=None, **kwargs):
import serpent.ocr
if ("force" in kwargs and kwargs["force"] is True) or self.game_is_focused:
string_location = serpent.ocr.locate_string(
query_string,
game_frame.frame,
fuzziness=fuzziness,
ocr_preset=ocr_preset,
offset_x=game_frame.offset_x,
offset_y=game_frame.offset_y
)
if string_location is not None:
x = (string_location[1] + string_location[3]) // 2
y = (string_location[0] + string_location[2]) // 2
self.move(x=x, y=y)
self.click(button=button, **kwargs)
return True
return False
def drag(self, button=MouseButton.LEFT, x0=None, y0=None, x1=None, y1=None, duration=0.25, **kwargs):
if ("force" in kwargs and kwargs["force"] is True) or self.game_is_focused:
self.move(x=x0, y=y0)
self.click_down(button=button, **kwargs)
self.move(x=x1, y=y1, duration=duration)
self.click_up(button=button, **kwargs)
def drag_screen_region_to_screen_region(self, button=MouseButton.LEFT, start_screen_region=None, end_screen_region=None, duration=1, **kwargs):
if ("force" in kwargs and kwargs["force"] is True) or self.game_is_focused:
start_screen_region_coordinates = self._extract_screen_region_coordinates(start_screen_region)
end_screen_region_coordinates = self._extract_screen_region_coordinates(end_screen_region)
self.drag(
button=button,
x0=start_screen_region_coordinates[0],
y0=start_screen_region_coordinates[1],
x1=end_screen_region_coordinates[0],
y1=end_screen_region_coordinates[1],
duration=duration,
**kwargs
)
def scroll(self, clicks=1, direction="DOWN", **kwargs):
if ("force" in kwargs and kwargs["force"] is True) or self.game_is_focused:
clicks = clicks * (1 if direction == "DOWN" else -1)
pyautogui.scroll(clicks)
def enemy_action_capture(self, game_frame):
final_cultist_attack = []
attack_cultist_temp_list= []
# Unselect anything just incase
self.input_controller.click(button=MouseButton.RIGHT, duration=0.25)
time.sleep(.5)
# Home hover
self.input_controller.move(x=636, y=375, duration=0.25, absolute=True)
time.sleep(1)
# Enemy hover
self.input_controller.move(x=959, y=410, duration=0.25, absolute=True)
time.sleep(.75)
image_data = skimage.io.imread("plugins/SerpentSlayTheSpireGamePlugin/files/data/sprites/sprite_Attack_for_0.png")[..., np.newaxis]
attack_for_cultist = Sprite("attack_for_cultist", image_data=image_data)
# Full game frame capture
# print("-----------Full game frame capture-----------")
# full_game_frame = FrameGrabber.get_frames(
# [0],
# frame_shape=(self.game.frame_height, self.game.frame_width),
# frame_type="PIPELINE"
# ).frames[0]
# Allows for dynamic capture of enemy attack
sprite_locator = SpriteLocator()
attack_for_cultist_location = sprite_locator.locate(sprite=attack_for_cultist, game_frame=game_frame)
print("attack_for_cultist_location: ", attack_for_cultist_location)
# Tuples are immutable :(
if (attack_for_cultist_location != None):
attack_cultist_temp_list = list(attack_for_cultist_location)
attack_cultist_temp_list[1] = attack_cultist_temp_list[1] + 45
attack_cultist_temp_list[3] = attack_cultist_temp_list[3] + 15
attack_for_cultist_location = tuple(attack_cultist_temp_list)
print("Updated - attack_for_cultist_location: ", attack_for_cultist_location)
time.sleep(1)
cultist_attack = serpent.cv.extract_region_from_image(game_frame.frame, attack_for_cultist_location)
cultist_attack_grayscale = np.array(skimage.color.rgb2gray(cultist_attack) * 255, dtype="uint8")
cultist_attack = serpent.ocr.perform_ocr(image=cultist_attack_grayscale, scale=15, order=5, horizontal_closing=2, vertical_closing=1)
# This is actually an awkward work around for limitations in how tesseract works. By default it doesn't capture single char values so when dynamically
# searching and capturing the enemy attack the region it's looking for the region that includes the word "for " + attack value (i.e. "for 6"). There
# are ways of swapping the mode of tesseract to do a capture for single char values but because the attack values are dynamic it sometimes is
# less than 10 or much greater than 10 which is now multiple char's and messes with the capture. For the sake of just getting it working I did this
# TLDR: Awkward workaround for limitation in tesseract when capturing single char values. Likely easier way to capture then parse attack value
for elem in cultist_attack:
if (elem.isdigit() == True):
final_cultist_attack.append(elem)
print("final_cultist_attack", final_cultist_attack)
final_cultist_attack = ''.join(final_cultist_attack)
print("final_cultist_attack: ", final_cultist_attack)
print("------------------------------------")
self.game_state["final_cultist_attack"].insert(0, final_cultist_attack)
self.poison_check(game_frame)
else:
return print("Failed to capture enemy attack")
def handle_play(self, game_frame):
#self.printer.add("")
#self.printer.add("BombermanAI")
#self.printer.add("Reinforcement Learning: Training a PPO Agent")
#self.printer.add("")
#self.printer.add(f"Stage Started At: {self.started_at}")
#self.printer.add(f"Current Run: #{self.current_attempts}")
#self.printer.add("")
#self.check_game_state(game_frame)
#####################CHECK STATE###########################
#game over?
locationGO = None
sprite_to_locate = Sprite("QUERY", image_data=self.spriteGO.image_data)
sprite_locator = SpriteLocator()
locationGO = sprite_locator.locate(sprite=sprite_to_locate,
game_frame=game_frame)
#print("Location Game over:",locationGO)
#won game?
locationWO = None
sprite_to_locate = Sprite("QUERY", image_data=self.spriteWO.image_data)
sprite_locator = SpriteLocator()
locationWO = sprite_locator.locate(sprite=sprite_to_locate,
game_frame=game_frame)
#print("Location Game won:",locationWO)
self.gamestate.victory = locationWO != None
self.gamestate.lose = locationGO != None
self.gamestate.girl_alive = (locationGO == None and locationWO == None)
self.gamestate.done = not self.gamestate.girl_alive
print(f"Is alive? {self.gamestate.girl_alive}")
print(f"Game over? {self.gamestate.lose}")
print(f"Won? {self.gamestate.victory}")
#####################VISUAL DEBUGGER###########################
for i, game_frame in enumerate(self.game_frame_buffer.frames):
self.visual_debugger.store_image_data(game_frame.frame,
game_frame.frame.shape,
str(i))
#####################MODEL###########################
#get buffer
frame_buffer = FrameGrabber.get_frames([0, 1, 2, 3],
frame_type="PIPELINE")
game_frame_buffer = self.extract_game_area(frame_buffer)
state = game_frame_buffer.reshape(4, 104, 136, 1)
if (self.gamestate.done):
print(f"Game over, attemp {self.epoch}")
if (self.epoch % 10) == 0:
print("saving model")
self.dqn_agent.save_model(
f"bombergirl_epoch_{self.epoch}.model")
self.printer.save_file()
self.printer.add(
f"{self.gamestate.victory},{self.gamestate.lose},{self.epoch},{self.gamestate.time},{self.total_reward}"
)
self.total_reward = 0
self.dqn_agent.remember(self.prev_state, self.prev_action,
self.prev_reward, state, True)
self.dqn_agent.replay()
self.input_controller.tap_key(KeyboardKey.KEY_ENTER)
self.epoch += 1
self.total_reward = 0
self.gamestate.restartState()
self.prev_state = None
self.prev_action = None
else:
#update time
self.gamestate.updateTime()
#print(np.stack(game_frame_buffer,axis=1).shape)
#print(game_frame_buffer.shape)
#print(state.shape)
if (not (self.prev_state is None)
and not (self.prev_action is None)):
self.dqn_agent.remember(self.prev_state, self.prev_action,
self.prev_reward, state, False)
#do something
action_index = self.dqn_agent.act(state)
#get key
action = self.game_actions[action_index]
#get random frame from buffer
game_frame_rand = random.choice(frame_buffer.frames).frame
#update enviroment accorind to frame
###################FUN UPDATE STATE#########################################
game_area = \
serpent.cv.extract_region_from_image(game_frame_rand,self.game.screen_regions['GAME_REGION'])
#game ...
# 0,0
# 32,32
game_squares = [[None for j in range(0, 11)] for i in range(0, 15)]
const_offset = 8
const = 32
#game variables
self.gamestate.bombs = [] #{x, y}
self.gamestate.enemies = [] #{x,y}
#force girl to die if not found
girl_found = False
for i in range(0, 15):
for j in range(0, 11):
izq = ((j + 1) * const - const_offset,
(i + 1) * const - const_offset)
der = ((j + 2) * const + const_offset,
(i + 2) * const + const_offset)
reg = (izq[0], izq[1], der[0], der[1])
square = serpent.cv.extract_region_from_image(
game_area, reg)
square = self.convert_to_rgba(square)
sprite_to_locate = Sprite("QUERY",
image_data=square[...,
np.newaxis])
sprite = self.sprite_identifier.identify(
sprite_to_locate, mode="SIGNATURE_COLORS")
game_squares[i][j] = sprite
if ("SPRITE_BETTY" in sprite):
self.girl = {"x": i, "y": j}
girl_found = True
elif ("SPRITE_GEORGE" in sprite):
self.gamestate.enemies.append({"x": i, "y": j})
elif ("SPRITE_BOMB" in sprite):
self.gamestate.bombs.append({"x": i, "y": j})
elif ("SPRITE_BONUSES" in sprite):
self.gamestate.bonus.append({"x": i, "y": j})
#####################CHECK STATE###########################
#game over?
locationGO = None
sprite_to_locate = Sprite("QUERY",
image_data=self.spriteGO.image_data)
sprite_locator = SpriteLocator()
locationGO = sprite_locator.locate(sprite=sprite_to_locate,
game_frame=game_frame)
#print("Location Game over:",locationGO)
#won game?
locationWO = None
sprite_to_locate = Sprite("QUERY",
image_data=self.spriteWO.image_data)
sprite_locator = SpriteLocator()
locationWO = sprite_locator.locate(sprite=sprite_to_locate,
game_frame=game_frame)
#print("Location Game won:",locationWO)
self.gamestate.lose = locationGO != None
self.gamestate.victory = locationWO != None
self.gamestate.girl_alive = (locationGO == None
and locationWO == None)
self.gamestate.done = not self.gamestate.girl_alive
print(f"Is alive? {self.gamestate.girl_alive}")
print(f"Game over? {self.gamestate.lose}")
print(f"Won? {self.gamestate.victory}")
###################REWARD#########################################
#get reward
reward = self.gamestate.getReward(action_index)
self.total_reward += reward
self.prev_state = state
self.prev_action = action_index
self.prev_reward = reward
if (action):
self.input_controller.tap_key(
action, 0.15 if action_index < 4 else 0.01)
print(
f"Action: {self.gamestate.game_inputs[action_index]}, reward: {reward}, total_reward: {self.total_reward}"
)
class DonkeyKongAPI(GameAPI):
def __init__(self, game=None):
super().__init__(game=game)
self.navigationGameFSM = NavigationGameFSM()
self._prepare_sprites()
self.WIDTH = 34
self.HEIGHT = 32
self.n_WIDTH = 2
self.n_HEIGHT = 2
self.MARIO = 1
self.MOVING_ENTITY = 2
self.LADDER = 3
self.LADDER_DELTA = 18
self.LEVEL_WIN_HEIGHT = 32
self.sprite_locator = SpriteLocator()
self.ladders_positions = dict()
self.ladders_positions[0] = [453, 10000]
self.ladders_positions[1] = [102, 249]
self.ladders_positions[2] = [285, 452]
self.ladders_positions[3] = [104, 194]
self.ladders_positions[4] = [452, 10000]
self.ladders_positions[5] = [324, 10000]
self.on_ladders = dict()
self.on_ladders[(0, 453)] = [374, 328]
self.on_ladders[(1, 102)] = [312, 268]
self.on_ladders[(1, 249)] = [318, 262]
self.on_ladders[(2, 285)] = [258, 202]
self.on_ladders[(2, 452)] = [252, 208]
self.on_ladders[(3, 104)] = [192, 148]
self.on_ladders[(3, 194)] = [196, 144]
self.on_ladders[(4, 452)] = [133, 88]
self.on_ladders[(5, 324)] = [81, 32]
self.last_stage = 0
self.max_stage = 0
self.last_change_stage = 0
self.useless_actions = 0
self.level_direction = dict()
orientation = 1
for i in range(6):
self.level_direction[i] = orientation
orientation = orientation * (-1)
self.ladders_thresholds = [384, 328, 268, 208, 148, 88]
def _prepare_sprites(self):
# Death Sprite
path = './plugins/SerpentDonkeyKongGamePlugin/files/data/sprites/death_{}.png'
image = io.imread(path.format(1))
self.death_sprite = Sprite("DEATH", image_data=image[..., np.newaxis])
for i in range(2, 11):
image = io.imread(path.format(i))
self.death_sprite.append_image_data(image[..., np.newaxis])
# Mario Sprite
path = './plugins/SerpentDonkeyKongGamePlugin/files/data/sprites/mario_{}.png'
image = io.imread(path.format(1))
self.mario_sprite = Sprite("MARIO", image_data=image[..., np.newaxis])
for i in range(2, 30):
image = io.imread(path.format(i))
self.mario_sprite.append_image_data(image[..., np.newaxis])
# Falling Barrels Sprite
path = './plugins/SerpentDonkeyKongGamePlugin/files/data/sprites/falling_barrel_{}.png'
image = io.imread(path.format(1))
self.falling_barrel_sprite = Sprite("FALLING",
image_data=image[..., np.newaxis])
for i in range(2, 3):
image = io.imread(path.format(i))
self.falling_barrel_sprite.append_image_data(image[...,
np.newaxis])
# Rolling Barrels Sprite
path = './plugins/SerpentDonkeyKongGamePlugin/files/data/sprites/rolling_barrel_{}.png'
image = io.imread(path.format(1))
self.rolling_barrel_sprite = Sprite("ROLLING",
image_data=image[..., np.newaxis])
for i in range(2, 5):
image = io.imread(path.format(i))
self.rolling_barrel_sprite.append_image_data(image[...,
np.newaxis])
# Fire Monster Sprite
path = './plugins/SerpentDonkeyKongGamePlugin/files/data/sprites/flammy_{}.png'
image = io.imread(path.format(1))
self.flammy_sprite = Sprite("FLAMMY",
image_data=image[..., np.newaxis])
for i in range(2, 5):
image = io.imread(path.format(i))
self.flammy_sprite.append_image_data(image[..., np.newaxis])
# Splash Screen Sprite
image = io.imread(
'./plugins/SerpentDonkeyKongGamePlugin/files/data/sprites/sprite_main_menu_splash_screen.png'
)
self.splash_screen = Sprite("SPLASH",
image_data=image[..., np.newaxis])
def analyze_frame(self, game_frame):
locations = [None, None]
if (self.navigationGameFSM.current == "black_screen"):
locations[0] = self.sprite_locator.locate(sprite=self.mario_sprite,
game_frame=game_frame)
if (locations[0] != None):
self.navigationGameFSM.play()
elif (self.navigationGameFSM.current == "playing"):
loc = self._multiple_locate(sprite=self.mario_sprite,
game_frame=game_frame,
forced=True)
if (len(loc) != 0):
locations[0] = loc[0]
locations[1] = None
else:
locations[0] = None
loc = self._multiple_locate(sprite=self.death_sprite,
game_frame=game_frame,
forced=True)
if (len(loc) != 0):
locations[1] = loc[0]
if (locations[0] != None):
has_mario = True
if (locations[1] != None and locations[0] == None):
self.navigationGameFSM.die()
has_mario = False
if (locations[0] != None and locations[1] == None
and locations[0][0] <= self.LEVEL_WIN_HEIGHT):
self.navigationGameFSM.win()
if (self.last_change_stage == 0):
self.last_change_stage = time.time()
return locations
def _get_ladders(self, mario_posY):
level = self._get_level(mario_posY)
return self.ladders_positions[level]
def _get_level(self, mario_posY):
stage = 0
for i in range(5, -1, -1):
if (mario_posY <= self.ladders_thresholds[i]):
stage = i
break
if (stage != self.last_stage):
self.last_stage = stage
if (stage > self.max_stage):
self.max_stage = stage
self.last_change_stage = time.time()
return stage
def _get_moving_entities(self, game_frame):
moving = []
inputs = [
self.rolling_barrel_sprite, self.falling_barrel_sprite,
self.flammy_sprite
]
with concurrent.futures.ThreadPoolExecutor(max_workers=3) as executor:
futures = {
executor.submit(self._multiple_locate, sprite, game_frame):
sprite
for sprite in inputs
}
for future in concurrent.futures.as_completed(futures):
moving = moving + future.result()
return moving
def get_projection_matrix(self, game_frame, location):
reduced_frame = None
units_array = None
if (location != None):
temp = game_frame.frame[location[0] -
self.HEIGHT * self.n_HEIGHT:location[2],
location[1] -
self.WIDTH * self.n_WIDTH:location[3] +
self.WIDTH * self.n_WIDTH]
reduced_frame = GameFrame(temp)
units_array = self._projection(reduced_frame, location)
nx, ny = units_array.shape
units_array = units_array.reshape(nx * ny)
orientation = self._get_orientation(location)
units_array = np.insert(units_array, len(units_array), orientation)
return (reduced_frame, units_array)
def _get_orientation(self, location):
level = self._get_level(location[0])
return self.level_direction[level]
def _projection(self, game_frame, global_mario_positions):
mario_location = self.sprite_locator.locate(sprite=self.mario_sprite,
game_frame=game_frame)
moving_entities = self._get_moving_entities(game_frame)
units_array = np.zeros((1 + self.n_HEIGHT, 1 + 2 * self.n_WIDTH))
units_array[self.n_HEIGHT][self.n_WIDTH] = self.MARIO
ladders = self._get_ladders(global_mario_positions[0])
for ladder in ladders:
if (abs(ladder - global_mario_positions[1]) <= self.LADDER_DELTA):
pos_ladder = self.n_WIDTH
elif (ladder < global_mario_positions[1]):
# ladder on left
pos_ladder = self.n_WIDTH - (int(
(global_mario_positions[1] - ladder) / self.WIDTH) + 1)
else:
# ladder on right
pos_ladder = int(
(ladder - global_mario_positions[1]) / self.WIDTH)
if (pos_ladder == 0):
pos_ladder = 1
pos_ladder = pos_ladder + self.n_WIDTH
if (pos_ladder >= 0 and pos_ladder <= (2 * self.n_WIDTH)):
units_array[self.n_HEIGHT][pos_ladder] = self.LADDER
for entity in moving_entities:
if (entity[3] <= mario_location[1]):
# left
posX = entity[3]
elif (entity[1] >= mario_location[3]):
# right
posX = entity[1]
else:
# centered on Mario
posX = self.n_WIDTH * self.WIDTH
if (entity[0] >= mario_location[2]):
# bottom
posY = entity[0]
elif (entity[2] <= mario_location[0]):
# up
posY = entity[2]
else:
# centered on Mario
posY = self.n_HEIGHT * self.HEIGHT
units_array[int(posY / self.HEIGHT)][int(
posX / self.WIDTH)] = self.MOVING_ENTITY
return units_array
def _multiple_locate(self,
sprite=None,
game_frame=None,
screen_region=None,
use_global_location=True,
forced=False):
constellation_of_pixel_images = sprite.generate_constellation_of_pixels_images(
)
locations = []
frame = game_frame.frame
if screen_region is not None:
frame = serpent.cv.extract_region_from_image(frame, screen_region)
for i in range(len(constellation_of_pixel_images)):
constellation_of_pixels_item = list(
sprite.constellation_of_pixels[i].items())[0]
query_coordinates = constellation_of_pixels_item[0]
query_rgb = constellation_of_pixels_item[1]
rgb_coordinates = Sprite.locate_color(query_rgb, image=frame)
rgb_coordinates = list(
map(
lambda yx: (yx[0] - query_coordinates[0], yx[1] -
query_coordinates[1]), rgb_coordinates))
maximum_y = frame.shape[0] - constellation_of_pixel_images[
i].shape[0]
maximum_x = frame.shape[1] - constellation_of_pixel_images[
i].shape[1]
for y, x in rgb_coordinates:
if y < 0 or x < 0 or y > maximum_y or x > maximum_x:
continue
for yx, rgb in sprite.constellation_of_pixels[i].items():
if tuple(frame[y + yx[0], x + yx[1], :]) != rgb:
break
else:
locations.append(
(y, x, y + constellation_of_pixel_images[i].shape[0],
x + constellation_of_pixel_images[i].shape[1]))
if (forced and len(locations) > 0):
break
if len(locations
) != 0 and screen_region is not None and use_global_location:
for i in range(0, len(locations)):
locations[i] = (locations[i][0] + screen_region[0],
locations[i][1] + screen_region[1],
locations[i][2] + screen_region[0],
locations[i][3] + screen_region[1])
if len(locations) > 0:
locations = self._epurate(locations)
return locations
def _epurate(self, locations):
temp = []
epurated = []
for i in range(0, len(locations)):
if (not locations[i][0] in temp and not locations[i][0] - 1 in temp
and not locations[i][0] + 1 in temp):
temp.append(locations[i][0])
epurated.append(locations[i])
return epurated
def get_score_value(self, location):
values = [-1000, -1000, -1000, -1000, -1000]
if (location != None):
values[0] = self._get_level(location[0])
ladders = self._get_ladders(location[0])
minimum_distance = int(abs(ladders[0] - location[3]))
for ladder in ladders:
if (abs(ladder - location[1]) <= self.LADDER_DELTA):
distance = 0
elif (ladder < location[1]):
distance = int(abs(ladder - location[1]))
else:
distance = int(abs(ladder - location[3]))
if (distance < minimum_distance):
minimum_distance = distance
values[1] = -1 * minimum_distance
values[2] = self.max_stage
values[3] = self.useless_actions
if (self._is_on_ladder(values[0], location)):
values[4] = 1
else:
values[4] = 0
self.last_change_stage = 0
self.last_stage = 0
self.max_stage = 0
self.useless_actions = 0
return values
def _is_on_ladder(self, level, location):
ladders = self.ladders_positions[level]
for ladder in ladders:
if (abs(ladder - location[1]) <= self.LADDER_DELTA):
couple = (level, ladder)
if (couple in self.on_ladders):
extremity = self.on_ladders[couple]
if (location[0] <= extremity[0]
and location[0] > extremity[1]):
return True
return False
def get_death_location(self, game_frame):
return self.sprite_locator.locate(sprite=self.death_sprite,
game_frame=game_frame)
def analyze_action(self, inputs, outputs):
### FILTRE LES JUMPS INUTILES
# N = self.n_WIDTH * 2 + 1
# row = self.n_HEIGHT * N
# has_obstacle = False
# for i in range(N):
# if(inputs[row+i] == self.MOVING_ENTITY):
# has_obstacle = True
# break;
# if (not has_obstacle and outputs[len(outputs)-1] == 1):
# self.useless_actions = self.useless_actions + 1
if (outputs[len(outputs) - 1] == 1):
self.useless_actions = self.useless_actions + 1
"""
Methods to use the NavigationGame FSM
"""
def not_running(self):
return self.navigationGameFSM.current == "not_running"
def is_in_menu(self):
return self.navigationGameFSM.current == "menu"
def is_playing(self):
return self.navigationGameFSM.current == "playing"
def is_dead(self):
return self.navigationGameFSM.current == "dead"
def has_won(self):
return self.navigationGameFSM.current == "has_won"
def run(self):
self.navigationGameFSM.run()
def next(self):
self.navigationGameFSM.next()
def win(self):
self.navigationGameFSM.win()
class MyAPINamespace:
@classmethod
def my_namespaced_api_function(cls):
api = DonkeyKongAPI.instance
def __init__(self, game=None, **kwargs):
self.game = game
self.previous_key_collection_set = set()
self.sprite_locator = SpriteLocator()
from serpent.game_agent import GameAgent from serpent.input_controller import KeyboardKey from serpent.sprite_locator import SpriteLocator from serpent.frame_grabber import FrameGrabber from serpent.config import config import serpent.utilities from serpent.machine_learning.reinforcement_learning.ddqn import DDQN from serpent.machine_learning.reinforcement_learning.keyboard_mouse_action_space import KeyboardMouseActionSpace sprite_locator = SpriteLocator() import serpent.cv import time import cv2 import skimage.io import skimage.filters import skimage.morphology import skimage.measure import skimage.draw import skimage.segmentation import skimage.color import os import gc from datetime import datetime import collections import numpy as np from .helpers.memory import readhp from colorama import init init() from colorama import Fore, Style class SerpentRoboGameAgent(GameAgent):
class InputController:
def __init__(self, game=None):
self.game = game
self.mouse_buttons = {
MouseButton.LEFT.name: "left",
MouseButton.MIDDLE.name: "middle",
MouseButton.RIGHT.name: "right"
}
self.previous_key_collection_set = set()
self.sprite_locator = SpriteLocator()
@property
def game_is_focused(self):
return self.game.is_focused
# Keyboard Actions
def handle_keys(self, key_collection):
key_collection_set = set(key_collection)
keys_to_press = key_collection_set - self.previous_key_collection_set
keys_to_release = self.previous_key_collection_set - key_collection_set
for key in keys_to_press:
self.press_key(key)
for key in keys_to_release:
self.release_key(key)
self.previous_key_collection_set = key_collection_set
def tap_keys(self, keys, duration=0.05):
if self.game_is_focused:
for key in keys:
pyautogui.keyDown(key)
time.sleep(duration)
for key in keys:
pyautogui.keyUp(key)
def tap_key(self, key, duration=0.05):
if self.game_is_focused:
pyautogui.keyDown(key)
time.sleep(duration)
pyautogui.keyUp(key)
def press_keys(self, keys):
for key in keys:
self.press_key(key)
def press_key(self, key):
if self.game_is_focused:
pyautogui.keyDown(key)
def release_keys(self):
for key in self.previous_key_collection_set:
self.release_key(key)
self.previous_key_collection_set = set()
def release_key(self, key):
if self.game_is_focused:
pyautogui.keyUp(key)
def type_string(self, string, duration=0.05):
if self.game_is_focused:
pyautogui.typewrite(message=string, interval=duration)
# Mouse Actions
def click(self, button=MouseButton.LEFT, y=None, x=None, duration=0.25):
if self.game_is_focused:
pyautogui.moveTo(x, y, duration=duration)
pyautogui.click(button=self.mouse_buttons.get(button.name, "left"))
def click_screen_region(self, button=MouseButton.LEFT, screen_region=None):
if self.game_is_focused:
screen_region_coordinates = self.game.screen_regions.get(
screen_region)
x = (screen_region_coordinates[1] +
screen_region_coordinates[3]) // 2
x += self.game.window_geometry["x_offset"]
y = (screen_region_coordinates[0] +
screen_region_coordinates[2]) // 2
y += self.game.window_geometry["y_offset"]
self.click(button=button, y=y, x=x)
def click_sprite(self,
button=MouseButton.LEFT,
sprite=None,
game_frame=None):
if self.game_is_focused:
sprite_location = self.sprite_locator.locate(sprite=sprite,
game_frame=game_frame)
if sprite_location is None:
return False
x = (sprite_location[1] + sprite_location[3]) // 2
x += self.game.window_geometry["x_offset"]
y = (sprite_location[0] + sprite_location[2]) // 2
y += self.game.window_geometry["y_offset"]
self.click(button=button, y=y, x=x)
return True
def click_string(self,
query_string,
button=MouseButton.LEFT,
game_frame=None,
fuzziness=2,
ocr_preset=None):
if self.game_is_focused:
string_location = serpent.ocr.locate_string(
query_string,
game_frame.frame,
fuzziness=fuzziness,
ocr_preset=ocr_preset,
offset_x=game_frame.offset_x,
offset_y=game_frame.offset_y)
if string_location is not None:
x = (string_location[1] + string_location[3]) // 2
x += self.game.window_geometry["x_offset"]
y = (string_location[0] + string_location[2]) // 2
y += self.game.window_geometry["y_offset"]
self.click(button=button, y=y, x=x)
return True
return False
def drag(self,
button=MouseButton.LEFT,
x0=None,
y0=None,
x1=None,
y1=None,
duration=1):
if self.game_is_focused:
pyautogui.moveTo(x0, y0, duration=0.2)
pyautogui.dragTo(x1, y1, button=button, duration=duration)
def drag_screen_region_to_screen_region(self,
button=MouseButton.LEFT,
start_screen_region=None,
end_screen_region=None,
duration=1):
if self.game_is_focused:
start_screen_region_coordinates = self._extract_screen_region_coordinates(
start_screen_region)
end_screen_region_coordinates = self._extract_screen_region_coordinates(
end_screen_region)
self.drag(button=button,
x0=start_screen_region_coordinates[0],
y0=start_screen_region_coordinates[1],
x1=end_screen_region_coordinates[0],
y1=end_screen_region_coordinates[1],
duration=duration)
def scroll(self, y=None, x=None, clicks=1, direction="DOWN"):
if self.game_is_focused:
clicks = clicks * (1 if direction == "DOWN" else -1)
pyautogui.scroll(clicks, x=x, y=y)
def _extract_screen_region_coordinates(self, screen_region):
screen_region_coordinates = self.game.screen_regions.get(screen_region)
x = (screen_region_coordinates[1] + screen_region_coordinates[3]) // 2
x += self.game.window_geometry["x_offset"]
y = (screen_region_coordinates[0] + screen_region_coordinates[2]) // 2
y += self.game.window_geometry["y_offset"]
return x, y