def _discrete_to_action(self, i):
if self.noAgent:
return []
else:
if type(self.merged_actions[i]) == tuple:
xc, yc = self.merged_actions[i]
# Click in text field or on submit button
return [
vnc_spaces.PointerEvent(xc, yc, buttonmask=0),
vnc_spaces.PointerEvent(xc, yc, buttonmask=1),
vnc_spaces.PointerEvent(xc, yc, buttonmask=0)
]
elif type(self.merged_actions[i]) == int:
num = self.merged_actions[i]
# Enter number
return [
vnc_spaces.KeyEvent.by_name(str(num), down=True),
vnc_spaces.KeyEvent.by_name(str(num), down=False)
]
else:
letter = self.merged_actions[i]
# Enter letter
return [
vnc_spaces.KeyEvent.by_name(str(letter), down=True),
vnc_spaces.KeyEvent.by_name(str(letter), down=False)
]
def _discrete_to_action(self, i):
if self.noAgent:
return []
if type(self.merged_actions[i]) == tuple:
xc, yc = self.merged_actions[i]
# click in text field, in empty field or on submit button
return [
vnc_spaces.PointerEvent(xc, yc, buttonmask=0), # release
vnc_spaces.PointerEvent(xc, yc, buttonmask=1), # click
vnc_spaces.PointerEvent(xc, yc, buttonmask=0) # release
]
else:
key = self.merged_actions[i]
split1 = 'ctrl'
if (key == 'ctrl-a'):
split2 = 'a'
elif (key == 'ctrl-c'):
split2 = 'c'
else:
split2 = 'v'
return [
vnc_spaces.KeyEvent.by_name(split1, down=True), # press
vnc_spaces.KeyEvent.by_name(split2, down=True), # press
vnc_spaces.KeyEvent.by_name(split1, down=False), # release
vnc_spaces.KeyEvent.by_name(split2, down=False) # release
]
def _discrete_to_action(self, i):
xc, yc = self._points[i]
return [
spaces.PointerEvent(xc, yc, buttonmask=0), # release
spaces.PointerEvent(xc, yc, buttonmask=1), # click
spaces.PointerEvent(xc, yc, buttonmask=0), # release
]
def __init__(self,
env,
active_region=(10, 75 + 50, 10 + 160, 75 + 210),
discrete_mouse_step=10,
noclick_regions=[]):
super(SoftmaxClickMouse, self).__init__(env)
xlow, ylow, xhigh, yhigh = active_region
xs = range(xlow, xhigh, discrete_mouse_step)
ys = range(ylow, yhigh, discrete_mouse_step)
self._actions = []
removed = 0
for x in xs:
for y in ys:
xc = min(x + int(discrete_mouse_step / 2),
xhigh - 1) # click to center of a cell
yc = min(y + int(discrete_mouse_step / 2), yhigh - 1)
if any(
self.is_contained((xc, yc), r)
for r in noclick_regions):
removed += 1
continue
e1 = spaces.PointerEvent(xc, yc, buttonmask=0) # release
e2 = spaces.PointerEvent(xc, yc, buttonmask=1) # click
e3 = spaces.PointerEvent(xc, yc, buttonmask=0) # release
self._actions.append([e1, e2, e3])
logger.info('noclick regions removed {} of {} actions'.format(
removed, removed + len(self._actions)))
self.action_space = gym.spaces.Discrete(len(self._actions))
def _discrete_to_action(self, i):
if self.noAgent:
return []
else:
xc, yc = self._points[i]
return [
vnc_spaces.PointerEvent(xc, yc, buttonmask=0), # release
vnc_spaces.PointerEvent(xc, yc, buttonmask=1), # click
vnc_spaces.PointerEvent(xc, yc, buttonmask=0) # release
]
def _discrete_to_action(self, i):
if self.noAgent:
return []
else:
if type(self.merged_actions[i]) == tuple:
xc, yc = self.merged_actions[i]
return [
# Click in text field or on submit button
vnc_spaces.PointerEvent(xc, yc, buttonmask=0),
vnc_spaces.PointerEvent(xc, yc, buttonmask=1),
vnc_spaces.PointerEvent(xc, yc, buttonmask=0)
]
else:
key = self.merged_actions[i]
# Enter number
if key >= 0:
if key < 10:
return [
vnc_spaces.KeyEvent.by_name(str(key), down=True),
vnc_spaces.KeyEvent.by_name(str(key), down=False)
]
else:
return [
vnc_spaces.KeyEvent.by_name(str(key)[:1],
down=True),
vnc_spaces.KeyEvent.by_name(str(key)[:1],
down=False),
vnc_spaces.KeyEvent.by_name(str(key)[1:],
down=True),
vnc_spaces.KeyEvent.by_name(str(key)[1:],
down=False)
]
else:
if key > -10:
return [
vnc_spaces.KeyEvent.by_name('-', down=True),
vnc_spaces.KeyEvent.by_name('-', down=False),
vnc_spaces.KeyEvent.by_name(str(key)[1:],
down=True),
vnc_spaces.KeyEvent.by_name(str(key)[1:],
down=False)
]
else:
return [
vnc_spaces.KeyEvent.by_name('-', down=True),
vnc_spaces.KeyEvent.by_name('-', down=False),
vnc_spaces.KeyEvent.by_name(str(key)[1:2],
down=True),
vnc_spaces.KeyEvent.by_name(str(key)[1:2],
down=False),
vnc_spaces.KeyEvent.by_name(str(key)[2:],
down=True),
vnc_spaces.KeyEvent.by_name(str(key)[2:],
down=False)
]
def prepare(self, env):
if self._drag_start_time is None:
self._drag_start_time = time.time()
delta = time.time() - self._drag_start_time
if delta < self._wait_time:
return
delta -= self._wait_time
weight = delta / self._drag_time
if weight > 1:
self.i += 1
# Give up if we've exceeded our number of tries
give_up = self.i >= self._drag_tries
if give_up:
self.reset()
else:
self._drag_start_time = None
return give_up
x = int(weight * self.x + (1 - weight) * self.drag_from_x)
y = int(weight * self.y + (1 - weight) * self.drag_from_y)
buttonmask = self.drag_from_buttonmask
action = [spaces.PointerEvent(x, y, buttonmask)]
env.step(action)
def on_mouse_press(x, y, button, modifiers):
if self.mouse_grid_stride != None:
x_on_grid, y_on_grid = get_xy_on_grid(x, y)
next_openai_input = [
self.action_indices[str(
[[spaces.PointerEvent(x_on_grid, y_on_grid, 1)]])]
]
self.push_action_to_queue(next_openai_input)
self.last_input_was_mouse_motion = False
def compile_action(event):
if isinstance(event, tuple):
if event[0] == 'KeyEvent':
name, down = event[1:]
return spaces.KeyEvent.by_name(name, down=down).compile()
elif event[0] == 'PointerEvent':
x, y, buttonmask = event[1:]
return spaces.PointerEvent(x, y, buttonmask).compile()
else:
return event.compile()
def __init__(self,
src,
x,
y,
dsts=None,
buttonmask=None,
hold_after=None,
hold_between=None,
is_drag=False):
if dsts is None: dsts = []
if buttonmask is None: buttonmask = 0
if hold_after is None: hold_after = 0
self.src = src
self.dsts = dsts
self.x = x
self.y = y
self.buttonmask = buttonmask
self.is_drag = False
self.transition_name = '{}->{}'.format(self.src, self.dsts)
# Move mouse back and forth every second in order to trigger
# hover state.
self._prepares = [
[spaces.PointerEvent(x + 1, y)],
[spaces.PointerEvent(x, y)],
# Probably not completely off-screen but outside of
# whatever button is active
[spaces.PointerEvent(25, 100)],
]
self._apply_down = [spaces.PointerEvent(x, y, buttonmask)]
if self.is_drag:
self._apply_up = []
else:
self._apply_up = [spaces.PointerEvent(x, y)]
self.hold_after = hold_after
self.hold_between = hold_between
self.reset()
def on_mouse_motion(x, y, dx, dy):
if self.mouse_grid_stride != None:
x_on_grid, y_on_grid = get_xy_on_grid(x, y)
next_openai_input = [
self.action_indices[str(
[[spaces.PointerEvent(x_on_grid, y_on_grid, 0)]])]
]
self.replace_last_in_queue = self.last_input_was_mouse_motion
self.push_action_to_queue(next_openai_input)
self.replace_last_in_queue = False
self.last_input_was_mouse_motion = True
def __init__(self,
env=None,
mouse_area=(30, 95, 666, 575),
mouse_grid_stride=10):
""" Wraps an OpenAi environment so that it can be used
in combination with the baseline DQNs."""
super(OpenAiEnv, self).__init__(env)
self.pixel_skip = 3
self.height, self.width = 768, 1024
self.env.observation_space.shape = (
int(np.ceil(self.height / self.pixel_skip)),
int(np.ceil(self.width / self.pixel_skip)), 1
) # Note that every other pixel row/column is dropped in the preprocessing step.
self.keys = ['w', 'a', 's', 'd', 'r']
self.actions = [[[spaces.KeyEvent.by_name(key, down=down_or_up)]]
for key in self.keys for down_or_up in [True, False]]
# If I see it correctly the game sreen is rendered in a rectangle with coordinates ul_x=30, ul_y=95, lr_x=666, lr_y=575.
# In order to have the AI play BulletFury (which I think would be cool) I'll give it three possible positions to place the mouse at, so that it can turn left and right and aim at the center. Also, I'll give it an action for clicking (=firing) at the center.
self.game_ulx, self.game_uly, self.game_lrx, self.game_lry = mouse_area if mouse_area != None else (
0, 0, self.height, self.width)
self.mouse_grid_stride = mouse_grid_stride
if self.mouse_grid_stride != None:
for x in range(self.game_ulx,
self.game_lrx + self.mouse_grid_stride,
self.mouse_grid_stride):
for y in range(self.game_uly,
self.game_lry + self.mouse_grid_stride,
self.mouse_grid_stride):
self.actions.append([[spaces.PointerEvent(
x, y, 0)]]) # moves mouse to location on the grid.
self.actions.append([[spaces.PointerEvent(x, y, 1)]
]) # clicks at location on the grid.
self.env.action_space.n = len(self.actions)
self.black_screen = np.zeros(self.env.observation_space.shape)
self.manual_reward = 0
def _discrete_to_action(self, i):
if self.noAgent:
return []
xc, yc = self._points[i]
if self._is_clicked:
self._is_clicked = False
if self.env.spec.id == 'wob.mini.DragBox-v0':
return [
vnc_spaces.PointerEvent(xc, yc, buttonmask=0), # release
# Click Submit button
vnc_spaces.PointerEvent(10 + 47,
75 + 50 + 160 - 38,
buttonmask=0), # release
vnc_spaces.PointerEvent(10 + 47,
75 + 50 + 160 - 38,
buttonmask=1), # click
vnc_spaces.PointerEvent(10 + 47,
75 + 50 + 160 - 38,
buttonmask=0) # release
]
else:
return [
vnc_spaces.PointerEvent(xc, yc, buttonmask=0), # release
# Click Submit button
vnc_spaces.PointerEvent(10 + 55,
75 + 50 + 67,
buttonmask=0), # release
vnc_spaces.PointerEvent(10 + 55,
75 + 50 + 67,
buttonmask=1), # click
vnc_spaces.PointerEvent(10 + 55,
75 + 50 + 67,
buttonmask=0) # release
]
else:
self._is_clicked = True
return [
vnc_spaces.PointerEvent(xc, yc, buttonmask=0), # release
vnc_spaces.PointerEvent(xc, yc, buttonmask=1) # click
]
def _discrete_to_action(self, i):
if self.noAgent:
return []
else:
xc, yc = self._points[i]
if self._action_code == 0:
self._action_code = 1
return [
# click in text field
vnc_spaces.PointerEvent(xc, yc, buttonmask=0), # release
vnc_spaces.PointerEvent(xc, yc, buttonmask=1), # click
vnc_spaces.PointerEvent(xc, yc, buttonmask=0), # release
vnc_spaces.KeyEvent.by_name('ctrl', down=True), # press
vnc_spaces.KeyEvent.by_name('a', down=True), # press
vnc_spaces.KeyEvent.by_name('ctrl', down=False), # release
vnc_spaces.KeyEvent.by_name('a', down=False), # release
vnc_spaces.KeyEvent.by_name('ctrl', down=True), # press
vnc_spaces.KeyEvent.by_name('c', down=True), # press
vnc_spaces.KeyEvent.by_name('ctrl', down=False), # release
vnc_spaces.KeyEvent.by_name('c', down=False) # release
]
elif self._action_code == 1:
self._action_code = 2
return [
# click in empty field
vnc_spaces.PointerEvent(xc, yc, buttonmask=0), # release
vnc_spaces.PointerEvent(xc, yc, buttonmask=1), # click
vnc_spaces.PointerEvent(xc, yc, buttonmask=0), # release
vnc_spaces.KeyEvent.by_name('ctrl', down=True), # press
vnc_spaces.KeyEvent.by_name('v', down=True), # press
vnc_spaces.KeyEvent.by_name('ctrl', down=False), # release
vnc_spaces.KeyEvent.by_name('v', down=False) # release
]
else:
self._action_code = 0
return [
# click submit button
vnc_spaces.PointerEvent(xc, yc, buttonmask=0), # release
vnc_spaces.PointerEvent(xc, yc, buttonmask=1), # click
vnc_spaces.PointerEvent(xc, yc, buttonmask=0), # release
]
def _discrete_to_action(self, i):
if self.noAgent:
return []
xc, yc = self._points[i]
return [
vnc_spaces.PointerEvent(xc, yc, buttonmask=0), # release
vnc_spaces.PointerEvent(xc, yc, buttonmask=1), # click
vnc_spaces.PointerEvent(xc, yc, buttonmask=0), # release
# Click on Submit Button
vnc_spaces.PointerEvent(10 + 160 / 2,
75 + 50 + 160 - 30 / 2,
buttonmask=0), # release
vnc_spaces.PointerEvent(10 + 160 / 2,
75 + 50 + 160 - 30 / 2,
buttonmask=1), # click
vnc_spaces.PointerEvent(10 + 160 / 2,
75 + 50 + 160 - 30 / 2,
buttonmask=0), # release
]
def _discrete_to_action(self, i):
if self.noAgent:
return []
xc, yc = self._points[i % len(self._points)]
if i < len(self._points):
# Click on Object
return [
vnc_spaces.PointerEvent(xc, yc, buttonmask=0),
vnc_spaces.PointerEvent(xc, yc, buttonmask=1)
]
elif i < len(self._points * 2):
# Drag Object to a place and release it
return [vnc_spaces.PointerEvent(xc, yc, buttonmask=0)]
else:
# Click Submit Button
return [
vnc_spaces.PointerEvent(xc, yc, buttonmask=0),
vnc_spaces.PointerEvent(xc, yc, buttonmask=1),
vnc_spaces.PointerEvent(xc, yc, buttonmask=0)
]
def apply(self, env):
apply = [
spaces.PointerEvent(self.x, self.y, self.drag_from_buttonmask),
spaces.PointerEvent(self.x, self.y, self.buttonmask)
]
env.step(apply)
def _discrete_to_action(self, i):
if self.noAgent:
return []
else:
key = self._keys[i]
result = []
# Click in text field
if self.env.spec.id == 'wob.mini.SimpleAlgebra-v0':
result.append(
vnc_spaces.PointerEvent(10 + 110, 75 + 50 + 65, buttonmask=0))
result.append(
vnc_spaces.PointerEvent(10 + 110, 75 + 50 + 65, buttonmask=1))
result.append(
vnc_spaces.PointerEvent(10 + 110, 75 + 50 + 65, buttonmask=0))
elif self.env.spec.id == 'wob.mini.SimpleArithmetic-v0':
result.append(
vnc_spaces.PointerEvent(10 + 140, 75 + 50 + 35, buttonmask=0))
result.append(
vnc_spaces.PointerEvent(10 + 140, 75 + 50 + 35, buttonmask=1))
result.append(
vnc_spaces.PointerEvent(10 + 140, 75 + 50 + 35, buttonmask=0))
elif self.env.spec.id == 'wob.mini.VisualAddition-v0':
result.append(
vnc_spaces.PointerEvent(10 + 35, 75 + 50 + 125, buttonmask=0))
result.append(
vnc_spaces.PointerEvent(10 + 35, 75 + 50 + 125, buttonmask=1))
result.append(
vnc_spaces.PointerEvent(10 + 35, 75 + 50 + 125, buttonmask=0))
# Enter number
if key >= 0:
if key < 10:
result.append(vnc_spaces.KeyEvent.by_name(str(key), down=True))
result.append(vnc_spaces.KeyEvent.by_name(str(key),
down=False))
else:
result.append(
vnc_spaces.KeyEvent.by_name(str(key)[:1], down=True))
result.append(
vnc_spaces.KeyEvent.by_name(str(key)[:1], down=False))
result.append(
vnc_spaces.KeyEvent.by_name(str(key)[1:], down=True))
result.append(
vnc_spaces.KeyEvent.by_name(str(key)[1:], down=False))
else:
if key > -10:
result.append(vnc_spaces.KeyEvent.by_name('-', down=True))
result.append(vnc_spaces.KeyEvent.by_name('-', down=False))
result.append(
vnc_spaces.KeyEvent.by_name(str(key)[1:], down=True))
result.append(
vnc_spaces.KeyEvent.by_name(str(key)[1:], down=False))
else:
result.append(vnc_spaces.KeyEvent.by_name('-', down=True))
result.append(vnc_spaces.KeyEvent.by_name('-', down=False))
result.append(
vnc_spaces.KeyEvent.by_name(str(key)[1:2], down=True))
result.append(
vnc_spaces.KeyEvent.by_name(str(key)[1:2], down=False))
result.append(
vnc_spaces.KeyEvent.by_name(str(key)[2:], down=True))
result.append(
vnc_spaces.KeyEvent.by_name(str(key)[2:], down=False))
# Click on submit button
if self.env.spec.id == 'wob.mini.SimpleAlgebra-v0':
result.append(
vnc_spaces.PointerEvent(10 + 160 / 2,
75 + 50 + 115,
buttonmask=0))
result.append(
vnc_spaces.PointerEvent(10 + 160 / 2,
75 + 50 + 115,
buttonmask=1))
result.append(
vnc_spaces.PointerEvent(10 + 160 / 2,
75 + 50 + 115,
buttonmask=0))
elif self.env.spec.id == 'wob.mini.SimpleArithmetic-v0':
result.append(
vnc_spaces.PointerEvent(10 + 160 / 2,
75 + 50 + 83,
buttonmask=0))
result.append(
vnc_spaces.PointerEvent(10 + 160 / 2,
75 + 50 + 83,
buttonmask=1))
result.append(
vnc_spaces.PointerEvent(10 + 160 / 2,
75 + 50 + 83,
buttonmask=0))
elif self.env.spec.id == 'wob.mini.VisualAddition-v0':
result.append(
vnc_spaces.PointerEvent(10 + 110, 75 + 50 + 125, buttonmask=0))
result.append(
vnc_spaces.PointerEvent(10 + 110, 75 + 50 + 125, buttonmask=1))
result.append(
vnc_spaces.PointerEvent(10 + 110, 75 + 50 + 125, buttonmask=0))
return result
