class AndroidRecorder(BaseRecorder): def __init__(self, device=None): self.hm = HookManager() super(AndroidRecorder, self).__init__(device) self.hm.register(HookConstants.TOUCH_UP, self._on_click) # self.hm.register(HookConstants.GST_DRAG, self._on_drag) # self.hm.register(HookConstants.GST_SWIPE, self._on_swipe) # self.hm.register(HookConstants.GST_PINCH, self._on_pinch) # self.hm.register(HookConstants.ANY_KEY, self._on_key) def attach(self, device): if self.device is not None: self.detach() self.device = device self.hm.set_serial(self.device.serial) def detach(self): self.unhook() self.device = None self.hm.set_serial(None) def hook(self): self.hm.hook() def unhook(self): self.hm.unhook() def _on_click(self, event): pos = (event.x, event.y) print 'touch on', pos self.on_input_event(event) def analyze(self, idx, event, img, uixml): pass
class AndroidAtomRecorder(BaseRecorder): '''record detailed events: TOUCH_UP, TOUCH_DOWN, TOUCH_MOVE, KEY_UP, KEY_DOWN''' def __init__(self, *args, **kwargs): self.hm = HookManager() super(AndroidAtomRecorder, self).__init__(*args, **kwargs) self.hm.register(HC.KEY_ANY, self.input_event) self.hm.register(HC.TOUCH_ANY, self.input_event) def attach(self, device): if self.device is not None: self.detach() self.device = device self.hm.set_serial(self.device.serial) def detach(self): self.unhook() self.device = None self.hm.set_serial(None) def hook(self): self.hm.hook() def unhook(self): self.hm.unhook() def analyze_frame(self, idx, event, status): e = event if e.msg == HC.TOUCH_MOVE: d = { 'action' : 'touch_move', 'args' : (e.slotid, e.x, e.y), } elif e.msg == HC.TOUCH_DOWN: d = { 'action' : 'touch_down', 'args' : (e.slotid, e.x, e.y), } elif e.msg == HC.TOUCH_UP: d = { 'action' : 'touch_up', 'args' : (e.slotid, e.x, e.y), } elif e.msg & HC.KEY_ANY: if e.msg & 0x01: d = { 'action' : 'key_down', 'args' : (e.key,), } else: d = { 'action' : 'key_up', 'args' : (e.key,), } else: return self.case_draft.append(d) def process_draft(self, d): tmpl = { 'touch_move' : 'd.touch_move({:d}, {:d}, {:d})', 'touch_down' : 'd.touch_down({:d}, {:d}, {:d})', 'touch_up' : 'd.touch_up({:d}, {:d}, {:d})', 'key_down' : 'd.key_down("{}")', 'key_up' : 'd.key_up("{}")', } return tmpl[d['action']].format(*d['args'])
class AndroidRecorder(BaseRecorder, ScreenAddon, UixmlAddon, AdbStatusAddon): def __init__(self, *args, **kwargs): self.hm = HookManager() self.uilayout = AndroidLayout() self.nonui_activities = set() self.scene_detector = kwargs.pop('scene_detector', None) super(AndroidRecorder, self).__init__(*args, **kwargs) self.hm.register(HC.KEY_ANY, self.on_key) self.hm.register(HC.GST_TAP, self.input_event) self.hm.register(HC.GST_SWIPE, self.input_event) self.hm.register(HC.GST_DRAG, self.input_event) def attach(self, device): if self.device is not None: self.detach() self.device = device self.hm.set_serial(self.device.serial) def detach(self): self.unhook() self.device = None self.hm.set_serial(None) def hook(self): self.hm.hook() def unhook(self): self.hm.unhook() def add_nonui_activity(self, activity): self.nonui_activities.add(activity) def remove_nonui_activity(self, activity): self.nonui_activities.discard(activity) def on_key(self, event): if not event.msg & 0x01: # key_up print 'KeyEvent', event.key self.input_event(event) def serialize_event(self, event): e = event if e.msg & HC.KEY_ANY: return {'action':'key_event', 'args':(e.key,)} if e.msg == HC.GST_TAP: x, y = e.points[0] return {'action':'click', 'args':(x, y)} if e.msg in (HC.GST_SWIPE, HC.GST_DRAG): sx, sy = e.points[0] ex, ey = e.points[1] return {'action':'swipe', 'args':(sx, sy, ex, ey)} def analyze_frame(self, idx, event, status): e = event d = {'frameidx': idx} if e.msg & HC.KEY_ANY: d['action'] = 'key_event' d['args'] = (e.key,) self.case_draft.append(d) return uixml = status['uixml'] screen = status['screen'] adbstatus = status['adbstatus'] activity = adbstatus['activity'] analyze_ui = False if activity is not None and activity not in self.nonui_activities: if uixml is not None: self.uilayout.parse_xmldata(uixml) analyze_ui = True if event.msg == HC.GST_TAP: x, y = event.points[0] found_ui = False if analyze_ui: node = self.uilayout.find_clickable_node(x, y) if node: found_ui = True pnode, selector, order = self.uilayout.find_selector(node) d['action'] = 'click_ui' d['args'] = (x, y, self.uilayout.nodes.index(pnode)) d['extra'] = (selector, order) # try image first when uinode not found. if not found_ui: if screen is not None: bounds = find_clicked_bound(screen, x, y) d['action'] = 'click_image' d['args'] = (x, y, tuple(bounds)) else: d['action'] = 'click' d['args'] = (x, y) elif event.msg in (HC.GST_SWIPE, HC.GST_DRAG): sx, sy = e.points[0] ex, ey = e.points[1] d['action'] = 'swipe' d['args'] = (sx, sy, ex, ey) elif event.msg == HC.GST_PINCH_IN: #TODO pass else: return self.case_draft.append(d) def process_draft(self, d): if not d: return '' idx = d['frameidx'] frame = self.frames[idx - self.frames[0]['index']] waittime = frame['waittime'] if d['action'] == 'key_event': return 'd.keyevent("{}")'.format(*d['args']) elif d['action'] == 'click': return 'd.click({}, {})'.format(*d['args']) elif d['action'] == 'click_ui': selector, order = d['extra'] if order is None: return 'd(%s).click(timeout=%d)' %\ (', '.join(['%s=u"%s"' % item for item in selector.iteritems()]), 100*(int(waittime*10))) else: return 'objs = d(%s)\nif objs.wait.exists(timeout=%d):\n objs[%d].click()' %\ (', '.join(['%s=u"%s"' % item for item in selector.iteritems()]), 100*(int(waittime*10)), order) elif d['action'] == 'click_image': x, y, bounds = d['args'] desc = get_point_desc(x, y, bounds) screen = cv2.imread(os.path.join(self.framedir, frame['status']['screen'])) l, t, w, h = bounds img = screen[t:t+h, l:l+w] imgname = '%d-click.%dx%d.%s.png' % (idx, self.device_info['width'], self.device_info['height'], desc) imgpath = os.path.join(self.casedir, imgname) cv2.imwrite(imgpath, img) return 'd.click_image("%s")' % (imgname,) elif d['action'] == 'swipe': sx, sy, ex, ey = d['args'] return 'd.swipe(%s, %s, %s, %s, 10)\ntime.sleep(%.2f)' % (sx, sy, ex, ey, waittime) else: print 'unsupported action', d['action'] return ''
class AndroidRecorder(BaseRecorder, ScreenAddon, UixmlAddon, AdbStatusAddon): def __init__(self, *args, **kwargs): self.hm = HookManager() super(AndroidRecorder, self).__init__(*args, **kwargs) self.hm.register(HC.KEY_ANY, self.on_key) self.hm.register(HC.TOUCH_DOWN, self.input_event) self.hm.register(HC.GST_TAP, self.input_event) self.hm.register(HC.GST_SWIPE, self.input_event) self.hm.register(HC.GST_PINCH, self.input_event) self.uilayout = AndroidLayout() self.nonui_activities = set() def attach(self, device): if self.device is not None: self.detach() self.device = device self.hm.set_serial(self.device.serial) def detach(self): self.unhook() self.device = None self.hm.set_serial(None) def hook(self): self.hm.hook() def unhook(self): self.hm.unhook() def add_nonui_activity(self, activity): self.nonui_activities.add(activity) def remove_nonui_activity(self, activity): self.nonui_activities.discard(activity) def on_key(self, event): if not event.msg & 0x01: # key_up self.input_event(event) def analyze_frame(self, idx, event, status): e = event if e.msg & HC.KEY_ANY: d = { 'action' : 'keyevent', 'args' : (e.key,), 'pyscript' : 'd.keyevent("%s")' % (e.key,) } self.case_draft.append(d) return uixml = status['uixml'] screen = status['screen'] adbstatus = status['adbstatus'] activity = adbstatus['activity'] analyze_ui = False if activity is not None and activity not in self.nonui_activities: print 'in activity:', activity if uixml is not None: self.uilayout.parse_xmldata(uixml) analyze_ui = True event.msg = HC.GST_TAP d = {} if event.msg == HC.GST_TAP: x, y = event.x, event.y if analyze_ui: node = self.uilayout.find_clickable_rect(x, y) if node: print node.bounds, x, y d['action'] = 'click_ui' d['args'] = (node.class_name, node.resource_id, node.index) d['pyscript'] = 'd(className="%s", resourceId="%s", index="%s").click()' % d['args'] else: d['action'] = 'click' d['args'] = (x, y) d['pyscript'] = 'd.click(%s, %s)' % d['args'] elif screen is not None: d['action'] = 'click_image' img = find_clicked_img(screen, x, y) imgname = '%d-click.png' % (idx,) imgpath = os.path.join(self.draftdir, imgname) cv2.imwrite(imgpath, img) d['args'] = () d['pyscript'] = 'd.click_image()' else: d['action'] = 'click' d['args'] = (x, y) d['pyscript'] = 'd.click(%s, %s)' % d['args'] elif event.msg == HC.GST_SWIPE: #TODO # sx, sy, ex, ey = event.sx, event.sy, event.ex, event.ey pass elif event.msg == HC.GST_PINCH: #TODO pass else: return self.case_draft.append(d)
class AndroidAtomRecorder(BaseRecorder): '''record detailed events: TOUCH_UP, TOUCH_DOWN, TOUCH_MOVE, KEY_UP, KEY_DOWN''' def __init__(self, *args, **kwargs): self.hm = HookManager() super(AndroidAtomRecorder, self).__init__(*args, **kwargs) self.hm.register(HC.KEY_ANY, self.input_event) self.hm.register(HC.TOUCH_ANY, self.input_event) def attach(self, device): if self.device is not None: self.detach() self.device = device self.hm.set_serial(self.device.serial) def detach(self): self.unhook() self.device = None self.hm.set_serial(None) def hook(self): self.hm.hook() def unhook(self): self.hm.unhook() def analyze_frame(self, idx, event, status): e = event if e.msg == HC.TOUCH_MOVE: d = { 'action' : 'touch_move', 'args' : (e.slotid, e.x, e.y), } elif e.msg == HC.TOUCH_DOWN: d = { 'action' : 'touch_down', 'args' : (e.slotid, e.x, e.y), } elif e.msg == HC.TOUCH_UP: d = { 'action' : 'touch_up', 'args' : (e.slotid, e.x, e.y), } elif e.msg & HC.KEY_ANY: if e.msg & 0x01: d = { 'action' : 'key_down', 'args' : (e.key,), } else: d = { 'action' : 'key_up', 'args' : (e.key,), } else: return self.case_draft.append(d) def process_draft(self, d): tmpl = { 'touch_move' : 'd.touch_move({:d}, {:d}, {:d})', 'touch_down' : 'd.touch_down({:d}, {:d}, {:d})', 'touch_up' : 'd.touch_up({:d}, {:d}, {:d})', 'key_down' : 'd.key_down("{}")', 'key_up' : 'd.key_up("{}")', } return tmpl[d['action']].format(*d['args'])
class AndroidRecorder(BaseRecorder, ScreenAddon, UixmlAddon, AdbStatusAddon, RotationAddon): def __init__(self, *args, **kwargs): self.hm = HookManager() self.uilayout = AndroidLayout() self.nonui_activities = set() self.scene_detector = kwargs.pop('scene_detector', None) super(AndroidRecorder, self).__init__(*args, **kwargs) self.hm.register(HC.KEY_ANY, self.on_key) self.hm.register(HC.GST_TAP, self.input_event) self.hm.register(HC.GST_SWIPE, self.input_event) self.hm.register(HC.GST_DRAG, self.input_event) def attach(self, device): if self.device is not None: self.detach() self.device = device self.hm.set_serial(self.device.serial) def detach(self): self.unhook() self.device = None self.hm.set_serial(None) def hook(self): self.hm.hook() def unhook(self): self.hm.unhook() def add_nonui_activity(self, activity): self.nonui_activities.add(activity) def remove_nonui_activity(self, activity): self.nonui_activities.discard(activity) def on_key(self, event): if not event.msg & 0x01: # key_up print 'KeyEvent', event.key self.input_event(event) def serialize_event(self, event): e = event if e.msg & HC.KEY_ANY: return {'action':'key_event', 'args':(e.key,)} if e.msg == HC.GST_TAP: x, y = e.points[0] return {'action':'click', 'args':(x, y)} if e.msg in (HC.GST_SWIPE, HC.GST_DRAG): sx, sy = e.points[0] ex, ey = e.points[1] return {'action':'swipe', 'args':(sx, sy, ex, ey)} def analyze_frame(self, idx, event, status): e = event d = {'frameidx': idx} if e.msg & HC.KEY_ANY: d['action'] = 'key_event' d['args'] = (e.key,) self.case_draft.append(d) return uixml = status['uixml'] screen = status['screen'] adbstatus = status['adbstatus'] activity = adbstatus and adbstatus['activity'] or None rotation = status['rotation'] analyze_ui = False if activity is not None and activity not in self.nonui_activities: if uixml is not None: self.uilayout.parse_xmldata(uixml) rotation = self.uilayout.rotation analyze_ui = True # default rotation if rotation is None: rotation = 0 w, h = self.device_info['width'], self.device_info['height'] if e.msg == HC.GST_TAP: x, y = touch2screen(w, h, rotation, e.points[0][0], e.points[0][1]) found_ui = False if analyze_ui: node = self.uilayout.find_clickable_node(x, y) if node: found_ui = True pnode, selector, order = self.uilayout.find_selector(node) d['action'] = 'click_ui' d['args'] = (pnode.iterindex, ) d['extra'] = (selector, order) # try image first when uinode not found. if not found_ui: if screen is not None: bounds = find_clicked_bound(screen, x, y) d['action'] = 'click_image' d['args'] = (x, y, tuple(bounds)) else: d['action'] = 'click' d['args'] = (x, y) elif e.msg in (HC.GST_SWIPE, HC.GST_DRAG): sx, sy = touch2screen(w, h, rotation, e.points[0][0], e.points[0][1]) ex, ey = touch2screen(w, h, rotation, e.points[1][0], e.points[1][1]) d['action'] = 'swipe' d['args'] = (sx, sy, ex, ey) elif e.msg == HC.GST_PINCH_IN: #TODO pass else: return self.case_draft.append(d) def process_draft(self, d): if not d: return '' idx = int(d['frameidx']) frame = self.frames[idx] waittime = frame['waittime'] if d['action'] == 'key_event': return 'd.keyevent("{}")'.format(*d['args']) elif d['action'] == 'click': return 'd.click({}, {})'.format(*d['args']) elif d['action'] == 'click_ui': if 'extra' in d: selector, order = d['extra'] else: uixml = open(os.path.join(self.framedir, frame['status']['uixml'])).read() self.uilayout.parse_xmldata(uixml) pnode = self.uilayout.get_index_node(int(d['args'][0])) selector, order = self.uilayout.get_node_selector(pnode) if order is None: return 'd(%s).click(timeout=%d)' %\ (', '.join(['%s=u"%s"' % item for item in selector.iteritems()]), 100*(int(waittime*10))) else: return 'objs = d(%s)\nif objs.wait.exists(timeout=%d):\n objs[%d].click()' %\ (', '.join(['%s=u"%s"' % item for item in selector.iteritems()]), 100*(int(waittime*10)), order) elif d['action'] == 'click_image': x, y, bounds = d['args'] x, y, bounds = int(x), int(y), map(int, bounds) desc = get_point_desc(x, y, bounds) screen = cv2.imread(os.path.join(self.framedir, frame['status']['screen'])) l, t, w, h = bounds img = screen[t:t+h, l:l+w] imgname = '%d-click.%dx%d.%s.png' % (idx, self.device_info['width'], self.device_info['height'], desc) imgpath = os.path.join(self.casedir, imgname) cv2.imwrite(imgpath, img) return 'd.click_image("%s")' % (imgname,) elif d['action'] == 'swipe': sx, sy, ex, ey = d['args'] if waittime == 0: return 'd.swipe(%s, %s, %s, %s, 10)' % (sx, sy, ex, ey) return 'time.sleep(%.2f)\nd.swipe(%s, %s, %s, %s, 10)' % (waittime, sx, sy, ex, ey) else: print 'unsupported action', d['action'] return ''
class AndroidRecorder(BaseRecorder, ScreenAddon, UixmlAddon, AdbStatusAddon): def __init__(self, *args, **kwargs): self.hm = HookManager() self.uilayout = AndroidLayout() self.nonui_activities = set() self.scene_detector = kwargs.pop('scene_detector', None) super(AndroidRecorder, self).__init__(*args, **kwargs) self.hm.register(HC.KEY_ANY, self.on_key) self.hm.register(HC.GST_TAP, self.input_event) self.hm.register(HC.GST_SWIPE, self.input_event) self.hm.register(HC.GST_DRAG, self.input_event) def attach(self, device): if self.device is not None: self.detach() self.device = device self.hm.set_serial(self.device.serial) def detach(self): self.unhook() self.device = None self.hm.set_serial(None) def hook(self): self.hm.hook() def unhook(self): self.hm.unhook() def add_nonui_activity(self, activity): self.nonui_activities.add(activity) def remove_nonui_activity(self, activity): self.nonui_activities.discard(activity) def on_key(self, event): if not event.msg & 0x01: # key_up print 'KeyEvent', event.key self.input_event(event) def serialize_event(self, event): e = event if e.msg & HC.KEY_ANY: return {'action':'keyevent', 'args':(e.key,)} if e.msg == HC.GST_TAP: x, y = e.points[0] return {'action':'touch', 'args':(x, y)} if e.msg in (HC.GST_SWIPE, HC.GST_DRAG): sx, sy = e.points[0] ex, ey = e.points[1] return {'action':'swipe', 'args':(sx, sy, ex, ey)} def analyze_frame(self, idx, event, status, waittime): e = event if e.msg & HC.KEY_ANY: d = { 'action' : 'keyevent', 'args' : (e.key,), 'pyscript' : 'd.keyevent("%s")' % (e.key,) } self.case_draft.append(d) return uixml = status['uixml'] screen = status['screen'] adbstatus = status['adbstatus'] activity = adbstatus['activity'] analyze_ui = False if activity is not None and activity not in self.nonui_activities: if uixml is not None: self.uilayout.parse_xmldata(uixml) analyze_ui = True d = {} if event.msg == HC.GST_TAP: x, y = event.points[0] if analyze_ui: node = self.uilayout.find_clickable_node(x, y) if node: selector, order = self.uilayout.find_selector(node) d['action'] = 'click_ui' d['args'] = (selector, order) if order is None: d['pyscript'] = 'd(%s).click(timeout=%d)' %\ (', '.join(['%s=u"%s"' % item for item in selector.iteritems()]), 100*(int(waittime*10))) else: d['pyscript'] = 'objs = d(%s)\nif objs.wait.exists(timeout=%d):\n objs[%d].click()' %\ (', '.join(['%s=u"%s"' % item for item in selector.iteritems()]), 100*(int(waittime*10)), order) else: d['action'] = 'click' d['args'] = (x, y) d['pyscript'] = 'd.click(%s, %s)' % (x, y) elif screen is not None: d['action'] = 'click_image' img, bounds = find_clicked_img(screen, x, y) imgname = '%d-click.png' % (idx,) imgpath = os.path.join(self.draftdir, imgname) cv2.imwrite(imgpath, img) d['args'] = ((x, y), tuple(bounds)) d['pyscript'] = 'd.click_image("%s")' % (imgname,) else: d['action'] = 'click' d['args'] = (x, y) d['pyscript'] = 'd.click(%s, %s)' % (x, y) elif event.msg in (HC.GST_SWIPE, HC.GST_DRAG): sx, sy = e.points[0] ex, ey = e.points[1] d['action'] = 'swipe' d['args'] = (sx, sy, ex, ey) d['pyscript'] = 'd.swipe(%s, %s, %s, %s, 10)' % (sx, sy, ex, ey) elif event.msg == HC.GST_PINCH_IN: #TODO pass else: return self.case_draft.append(d)