def _discrete_to_action(self, i): if self.noAgent: return [] else: if type(self.merged_actions[i]) == tuple: xc, yc = self.merged_actions[i] # Click in text field or on submit button return [ vnc_spaces.PointerEvent(xc, yc, buttonmask=0), vnc_spaces.PointerEvent(xc, yc, buttonmask=1), vnc_spaces.PointerEvent(xc, yc, buttonmask=0) ] elif type(self.merged_actions[i]) == int: num = self.merged_actions[i] # Enter number return [ vnc_spaces.KeyEvent.by_name(str(num), down=True), vnc_spaces.KeyEvent.by_name(str(num), down=False) ] else: letter = self.merged_actions[i] # Enter letter return [ vnc_spaces.KeyEvent.by_name(str(letter), down=True), vnc_spaces.KeyEvent.by_name(str(letter), down=False) ]
def _discrete_to_action(self, i): if self.noAgent: return [] if type(self.merged_actions[i]) == tuple: xc, yc = self.merged_actions[i] # click in text field, in empty field or on submit button return [ vnc_spaces.PointerEvent(xc, yc, buttonmask=0), # release vnc_spaces.PointerEvent(xc, yc, buttonmask=1), # click vnc_spaces.PointerEvent(xc, yc, buttonmask=0) # release ] else: key = self.merged_actions[i] split1 = 'ctrl' if (key == 'ctrl-a'): split2 = 'a' elif (key == 'ctrl-c'): split2 = 'c' else: split2 = 'v' return [ vnc_spaces.KeyEvent.by_name(split1, down=True), # press vnc_spaces.KeyEvent.by_name(split2, down=True), # press vnc_spaces.KeyEvent.by_name(split1, down=False), # release vnc_spaces.KeyEvent.by_name(split2, down=False) # release ]
def _discrete_to_action(self, i): xc, yc = self._points[i] return [ spaces.PointerEvent(xc, yc, buttonmask=0), # release spaces.PointerEvent(xc, yc, buttonmask=1), # click spaces.PointerEvent(xc, yc, buttonmask=0), # release ]
def __init__(self, env, active_region=(10, 75 + 50, 10 + 160, 75 + 210), discrete_mouse_step=10, noclick_regions=[]): super(SoftmaxClickMouse, self).__init__(env) xlow, ylow, xhigh, yhigh = active_region xs = range(xlow, xhigh, discrete_mouse_step) ys = range(ylow, yhigh, discrete_mouse_step) self._actions = [] removed = 0 for x in xs: for y in ys: xc = min(x + int(discrete_mouse_step / 2), xhigh - 1) # click to center of a cell yc = min(y + int(discrete_mouse_step / 2), yhigh - 1) if any( self.is_contained((xc, yc), r) for r in noclick_regions): removed += 1 continue e1 = spaces.PointerEvent(xc, yc, buttonmask=0) # release e2 = spaces.PointerEvent(xc, yc, buttonmask=1) # click e3 = spaces.PointerEvent(xc, yc, buttonmask=0) # release self._actions.append([e1, e2, e3]) logger.info('noclick regions removed {} of {} actions'.format( removed, removed + len(self._actions))) self.action_space = gym.spaces.Discrete(len(self._actions))
def _discrete_to_action(self, i): if self.noAgent: return [] else: xc, yc = self._points[i] return [ vnc_spaces.PointerEvent(xc, yc, buttonmask=0), # release vnc_spaces.PointerEvent(xc, yc, buttonmask=1), # click vnc_spaces.PointerEvent(xc, yc, buttonmask=0) # release ]
def _discrete_to_action(self, i): if self.noAgent: return [] else: if type(self.merged_actions[i]) == tuple: xc, yc = self.merged_actions[i] return [ # Click in text field or on submit button vnc_spaces.PointerEvent(xc, yc, buttonmask=0), vnc_spaces.PointerEvent(xc, yc, buttonmask=1), vnc_spaces.PointerEvent(xc, yc, buttonmask=0) ] else: key = self.merged_actions[i] # Enter number if key >= 0: if key < 10: return [ vnc_spaces.KeyEvent.by_name(str(key), down=True), vnc_spaces.KeyEvent.by_name(str(key), down=False) ] else: return [ vnc_spaces.KeyEvent.by_name(str(key)[:1], down=True), vnc_spaces.KeyEvent.by_name(str(key)[:1], down=False), vnc_spaces.KeyEvent.by_name(str(key)[1:], down=True), vnc_spaces.KeyEvent.by_name(str(key)[1:], down=False) ] else: if key > -10: return [ vnc_spaces.KeyEvent.by_name('-', down=True), vnc_spaces.KeyEvent.by_name('-', down=False), vnc_spaces.KeyEvent.by_name(str(key)[1:], down=True), vnc_spaces.KeyEvent.by_name(str(key)[1:], down=False) ] else: return [ vnc_spaces.KeyEvent.by_name('-', down=True), vnc_spaces.KeyEvent.by_name('-', down=False), vnc_spaces.KeyEvent.by_name(str(key)[1:2], down=True), vnc_spaces.KeyEvent.by_name(str(key)[1:2], down=False), vnc_spaces.KeyEvent.by_name(str(key)[2:], down=True), vnc_spaces.KeyEvent.by_name(str(key)[2:], down=False) ]
def prepare(self, env): if self._drag_start_time is None: self._drag_start_time = time.time() delta = time.time() - self._drag_start_time if delta < self._wait_time: return delta -= self._wait_time weight = delta / self._drag_time if weight > 1: self.i += 1 # Give up if we've exceeded our number of tries give_up = self.i >= self._drag_tries if give_up: self.reset() else: self._drag_start_time = None return give_up x = int(weight * self.x + (1 - weight) * self.drag_from_x) y = int(weight * self.y + (1 - weight) * self.drag_from_y) buttonmask = self.drag_from_buttonmask action = [spaces.PointerEvent(x, y, buttonmask)] env.step(action)
def on_mouse_press(x, y, button, modifiers): if self.mouse_grid_stride != None: x_on_grid, y_on_grid = get_xy_on_grid(x, y) next_openai_input = [ self.action_indices[str( [[spaces.PointerEvent(x_on_grid, y_on_grid, 1)]])] ] self.push_action_to_queue(next_openai_input) self.last_input_was_mouse_motion = False
def compile_action(event): if isinstance(event, tuple): if event[0] == 'KeyEvent': name, down = event[1:] return spaces.KeyEvent.by_name(name, down=down).compile() elif event[0] == 'PointerEvent': x, y, buttonmask = event[1:] return spaces.PointerEvent(x, y, buttonmask).compile() else: return event.compile()
def __init__(self, src, x, y, dsts=None, buttonmask=None, hold_after=None, hold_between=None, is_drag=False): if dsts is None: dsts = [] if buttonmask is None: buttonmask = 0 if hold_after is None: hold_after = 0 self.src = src self.dsts = dsts self.x = x self.y = y self.buttonmask = buttonmask self.is_drag = False self.transition_name = '{}->{}'.format(self.src, self.dsts) # Move mouse back and forth every second in order to trigger # hover state. self._prepares = [ [spaces.PointerEvent(x + 1, y)], [spaces.PointerEvent(x, y)], # Probably not completely off-screen but outside of # whatever button is active [spaces.PointerEvent(25, 100)], ] self._apply_down = [spaces.PointerEvent(x, y, buttonmask)] if self.is_drag: self._apply_up = [] else: self._apply_up = [spaces.PointerEvent(x, y)] self.hold_after = hold_after self.hold_between = hold_between self.reset()
def on_mouse_motion(x, y, dx, dy): if self.mouse_grid_stride != None: x_on_grid, y_on_grid = get_xy_on_grid(x, y) next_openai_input = [ self.action_indices[str( [[spaces.PointerEvent(x_on_grid, y_on_grid, 0)]])] ] self.replace_last_in_queue = self.last_input_was_mouse_motion self.push_action_to_queue(next_openai_input) self.replace_last_in_queue = False self.last_input_was_mouse_motion = True
def __init__(self, env=None, mouse_area=(30, 95, 666, 575), mouse_grid_stride=10): """ Wraps an OpenAi environment so that it can be used in combination with the baseline DQNs.""" super(OpenAiEnv, self).__init__(env) self.pixel_skip = 3 self.height, self.width = 768, 1024 self.env.observation_space.shape = ( int(np.ceil(self.height / self.pixel_skip)), int(np.ceil(self.width / self.pixel_skip)), 1 ) # Note that every other pixel row/column is dropped in the preprocessing step. self.keys = ['w', 'a', 's', 'd', 'r'] self.actions = [[[spaces.KeyEvent.by_name(key, down=down_or_up)]] for key in self.keys for down_or_up in [True, False]] # If I see it correctly the game sreen is rendered in a rectangle with coordinates ul_x=30, ul_y=95, lr_x=666, lr_y=575. # In order to have the AI play BulletFury (which I think would be cool) I'll give it three possible positions to place the mouse at, so that it can turn left and right and aim at the center. Also, I'll give it an action for clicking (=firing) at the center. self.game_ulx, self.game_uly, self.game_lrx, self.game_lry = mouse_area if mouse_area != None else ( 0, 0, self.height, self.width) self.mouse_grid_stride = mouse_grid_stride if self.mouse_grid_stride != None: for x in range(self.game_ulx, self.game_lrx + self.mouse_grid_stride, self.mouse_grid_stride): for y in range(self.game_uly, self.game_lry + self.mouse_grid_stride, self.mouse_grid_stride): self.actions.append([[spaces.PointerEvent( x, y, 0)]]) # moves mouse to location on the grid. self.actions.append([[spaces.PointerEvent(x, y, 1)] ]) # clicks at location on the grid. self.env.action_space.n = len(self.actions) self.black_screen = np.zeros(self.env.observation_space.shape) self.manual_reward = 0
def _discrete_to_action(self, i): if self.noAgent: return [] xc, yc = self._points[i] if self._is_clicked: self._is_clicked = False if self.env.spec.id == 'wob.mini.DragBox-v0': return [ vnc_spaces.PointerEvent(xc, yc, buttonmask=0), # release # Click Submit button vnc_spaces.PointerEvent(10 + 47, 75 + 50 + 160 - 38, buttonmask=0), # release vnc_spaces.PointerEvent(10 + 47, 75 + 50 + 160 - 38, buttonmask=1), # click vnc_spaces.PointerEvent(10 + 47, 75 + 50 + 160 - 38, buttonmask=0) # release ] else: return [ vnc_spaces.PointerEvent(xc, yc, buttonmask=0), # release # Click Submit button vnc_spaces.PointerEvent(10 + 55, 75 + 50 + 67, buttonmask=0), # release vnc_spaces.PointerEvent(10 + 55, 75 + 50 + 67, buttonmask=1), # click vnc_spaces.PointerEvent(10 + 55, 75 + 50 + 67, buttonmask=0) # release ] else: self._is_clicked = True return [ vnc_spaces.PointerEvent(xc, yc, buttonmask=0), # release vnc_spaces.PointerEvent(xc, yc, buttonmask=1) # click ]
def _discrete_to_action(self, i): if self.noAgent: return [] else: xc, yc = self._points[i] if self._action_code == 0: self._action_code = 1 return [ # click in text field vnc_spaces.PointerEvent(xc, yc, buttonmask=0), # release vnc_spaces.PointerEvent(xc, yc, buttonmask=1), # click vnc_spaces.PointerEvent(xc, yc, buttonmask=0), # release vnc_spaces.KeyEvent.by_name('ctrl', down=True), # press vnc_spaces.KeyEvent.by_name('a', down=True), # press vnc_spaces.KeyEvent.by_name('ctrl', down=False), # release vnc_spaces.KeyEvent.by_name('a', down=False), # release vnc_spaces.KeyEvent.by_name('ctrl', down=True), # press vnc_spaces.KeyEvent.by_name('c', down=True), # press vnc_spaces.KeyEvent.by_name('ctrl', down=False), # release vnc_spaces.KeyEvent.by_name('c', down=False) # release ] elif self._action_code == 1: self._action_code = 2 return [ # click in empty field vnc_spaces.PointerEvent(xc, yc, buttonmask=0), # release vnc_spaces.PointerEvent(xc, yc, buttonmask=1), # click vnc_spaces.PointerEvent(xc, yc, buttonmask=0), # release vnc_spaces.KeyEvent.by_name('ctrl', down=True), # press vnc_spaces.KeyEvent.by_name('v', down=True), # press vnc_spaces.KeyEvent.by_name('ctrl', down=False), # release vnc_spaces.KeyEvent.by_name('v', down=False) # release ] else: self._action_code = 0 return [ # click submit button vnc_spaces.PointerEvent(xc, yc, buttonmask=0), # release vnc_spaces.PointerEvent(xc, yc, buttonmask=1), # click vnc_spaces.PointerEvent(xc, yc, buttonmask=0), # release ]
def _discrete_to_action(self, i): if self.noAgent: return [] xc, yc = self._points[i] return [ vnc_spaces.PointerEvent(xc, yc, buttonmask=0), # release vnc_spaces.PointerEvent(xc, yc, buttonmask=1), # click vnc_spaces.PointerEvent(xc, yc, buttonmask=0), # release # Click on Submit Button vnc_spaces.PointerEvent(10 + 160 / 2, 75 + 50 + 160 - 30 / 2, buttonmask=0), # release vnc_spaces.PointerEvent(10 + 160 / 2, 75 + 50 + 160 - 30 / 2, buttonmask=1), # click vnc_spaces.PointerEvent(10 + 160 / 2, 75 + 50 + 160 - 30 / 2, buttonmask=0), # release ]
def _discrete_to_action(self, i): if self.noAgent: return [] xc, yc = self._points[i % len(self._points)] if i < len(self._points): # Click on Object return [ vnc_spaces.PointerEvent(xc, yc, buttonmask=0), vnc_spaces.PointerEvent(xc, yc, buttonmask=1) ] elif i < len(self._points * 2): # Drag Object to a place and release it return [vnc_spaces.PointerEvent(xc, yc, buttonmask=0)] else: # Click Submit Button return [ vnc_spaces.PointerEvent(xc, yc, buttonmask=0), vnc_spaces.PointerEvent(xc, yc, buttonmask=1), vnc_spaces.PointerEvent(xc, yc, buttonmask=0) ]
def apply(self, env): apply = [ spaces.PointerEvent(self.x, self.y, self.drag_from_buttonmask), spaces.PointerEvent(self.x, self.y, self.buttonmask) ] env.step(apply)
def _discrete_to_action(self, i): if self.noAgent: return [] else: key = self._keys[i] result = [] # Click in text field if self.env.spec.id == 'wob.mini.SimpleAlgebra-v0': result.append( vnc_spaces.PointerEvent(10 + 110, 75 + 50 + 65, buttonmask=0)) result.append( vnc_spaces.PointerEvent(10 + 110, 75 + 50 + 65, buttonmask=1)) result.append( vnc_spaces.PointerEvent(10 + 110, 75 + 50 + 65, buttonmask=0)) elif self.env.spec.id == 'wob.mini.SimpleArithmetic-v0': result.append( vnc_spaces.PointerEvent(10 + 140, 75 + 50 + 35, buttonmask=0)) result.append( vnc_spaces.PointerEvent(10 + 140, 75 + 50 + 35, buttonmask=1)) result.append( vnc_spaces.PointerEvent(10 + 140, 75 + 50 + 35, buttonmask=0)) elif self.env.spec.id == 'wob.mini.VisualAddition-v0': result.append( vnc_spaces.PointerEvent(10 + 35, 75 + 50 + 125, buttonmask=0)) result.append( vnc_spaces.PointerEvent(10 + 35, 75 + 50 + 125, buttonmask=1)) result.append( vnc_spaces.PointerEvent(10 + 35, 75 + 50 + 125, buttonmask=0)) # Enter number if key >= 0: if key < 10: result.append(vnc_spaces.KeyEvent.by_name(str(key), down=True)) result.append(vnc_spaces.KeyEvent.by_name(str(key), down=False)) else: result.append( vnc_spaces.KeyEvent.by_name(str(key)[:1], down=True)) result.append( vnc_spaces.KeyEvent.by_name(str(key)[:1], down=False)) result.append( vnc_spaces.KeyEvent.by_name(str(key)[1:], down=True)) result.append( vnc_spaces.KeyEvent.by_name(str(key)[1:], down=False)) else: if key > -10: result.append(vnc_spaces.KeyEvent.by_name('-', down=True)) result.append(vnc_spaces.KeyEvent.by_name('-', down=False)) result.append( vnc_spaces.KeyEvent.by_name(str(key)[1:], down=True)) result.append( vnc_spaces.KeyEvent.by_name(str(key)[1:], down=False)) else: result.append(vnc_spaces.KeyEvent.by_name('-', down=True)) result.append(vnc_spaces.KeyEvent.by_name('-', down=False)) result.append( vnc_spaces.KeyEvent.by_name(str(key)[1:2], down=True)) result.append( vnc_spaces.KeyEvent.by_name(str(key)[1:2], down=False)) result.append( vnc_spaces.KeyEvent.by_name(str(key)[2:], down=True)) result.append( vnc_spaces.KeyEvent.by_name(str(key)[2:], down=False)) # Click on submit button if self.env.spec.id == 'wob.mini.SimpleAlgebra-v0': result.append( vnc_spaces.PointerEvent(10 + 160 / 2, 75 + 50 + 115, buttonmask=0)) result.append( vnc_spaces.PointerEvent(10 + 160 / 2, 75 + 50 + 115, buttonmask=1)) result.append( vnc_spaces.PointerEvent(10 + 160 / 2, 75 + 50 + 115, buttonmask=0)) elif self.env.spec.id == 'wob.mini.SimpleArithmetic-v0': result.append( vnc_spaces.PointerEvent(10 + 160 / 2, 75 + 50 + 83, buttonmask=0)) result.append( vnc_spaces.PointerEvent(10 + 160 / 2, 75 + 50 + 83, buttonmask=1)) result.append( vnc_spaces.PointerEvent(10 + 160 / 2, 75 + 50 + 83, buttonmask=0)) elif self.env.spec.id == 'wob.mini.VisualAddition-v0': result.append( vnc_spaces.PointerEvent(10 + 110, 75 + 50 + 125, buttonmask=0)) result.append( vnc_spaces.PointerEvent(10 + 110, 75 + 50 + 125, buttonmask=1)) result.append( vnc_spaces.PointerEvent(10 + 110, 75 + 50 + 125, buttonmask=0)) return result