class AndroidRecorder(BaseRecorder):
def __init__(self, device=None):
self.hm = HookManager()
super(AndroidRecorder, self).__init__(device)
self.hm.register(HookConstants.TOUCH_UP, self._on_click)
# self.hm.register(HookConstants.GST_DRAG, self._on_drag)
# self.hm.register(HookConstants.GST_SWIPE, self._on_swipe)
# self.hm.register(HookConstants.GST_PINCH, self._on_pinch)
# self.hm.register(HookConstants.ANY_KEY, self._on_key)
def attach(self, device):
if self.device is not None:
self.detach()
self.device = device
self.hm.set_serial(self.device.serial)
def detach(self):
self.unhook()
self.device = None
self.hm.set_serial(None)
def hook(self):
self.hm.hook()
def unhook(self):
self.hm.unhook()
def _on_click(self, event):
pos = (event.x, event.y)
print 'touch on', pos
self.on_input_event(event)
def analyze(self, idx, event, img, uixml):
pass
class AndroidAtomRecorder(BaseRecorder):
'''record detailed events: TOUCH_UP, TOUCH_DOWN, TOUCH_MOVE, KEY_UP, KEY_DOWN'''
def __init__(self, *args, **kwargs):
self.hm = HookManager()
super(AndroidAtomRecorder, self).__init__(*args, **kwargs)
self.hm.register(HC.KEY_ANY, self.input_event)
self.hm.register(HC.TOUCH_ANY, self.input_event)
def attach(self, device):
if self.device is not None:
self.detach()
self.device = device
self.hm.set_serial(self.device.serial)
def detach(self):
self.unhook()
self.device = None
self.hm.set_serial(None)
def hook(self):
self.hm.hook()
def unhook(self):
self.hm.unhook()
def analyze_frame(self, idx, event, status):
e = event
if e.msg == HC.TOUCH_MOVE:
d = {
'action' : 'touch_move',
'args' : (e.slotid, e.x, e.y),
}
elif e.msg == HC.TOUCH_DOWN:
d = {
'action' : 'touch_down',
'args' : (e.slotid, e.x, e.y),
}
elif e.msg == HC.TOUCH_UP:
d = {
'action' : 'touch_up',
'args' : (e.slotid, e.x, e.y),
}
elif e.msg & HC.KEY_ANY:
if e.msg & 0x01:
d = {
'action' : 'key_down',
'args' : (e.key,),
}
else:
d = {
'action' : 'key_up',
'args' : (e.key,),
}
else:
return
self.case_draft.append(d)
def process_draft(self, d):
tmpl = {
'touch_move' : 'd.touch_move({:d}, {:d}, {:d})',
'touch_down' : 'd.touch_down({:d}, {:d}, {:d})',
'touch_up' : 'd.touch_up({:d}, {:d}, {:d})',
'key_down' : 'd.key_down("{}")',
'key_up' : 'd.key_up("{}")',
}
return tmpl[d['action']].format(*d['args'])
class AndroidRecorder(BaseRecorder, ScreenAddon, UixmlAddon, AdbStatusAddon):
def __init__(self, *args, **kwargs):
self.hm = HookManager()
self.uilayout = AndroidLayout()
self.nonui_activities = set()
self.scene_detector = kwargs.pop('scene_detector', None)
super(AndroidRecorder, self).__init__(*args, **kwargs)
self.hm.register(HC.KEY_ANY, self.on_key)
self.hm.register(HC.GST_TAP, self.input_event)
self.hm.register(HC.GST_SWIPE, self.input_event)
self.hm.register(HC.GST_DRAG, self.input_event)
def attach(self, device):
if self.device is not None:
self.detach()
self.device = device
self.hm.set_serial(self.device.serial)
def detach(self):
self.unhook()
self.device = None
self.hm.set_serial(None)
def hook(self):
self.hm.hook()
def unhook(self):
self.hm.unhook()
def add_nonui_activity(self, activity):
self.nonui_activities.add(activity)
def remove_nonui_activity(self, activity):
self.nonui_activities.discard(activity)
def on_key(self, event):
if not event.msg & 0x01: # key_up
print 'KeyEvent', event.key
self.input_event(event)
def serialize_event(self, event):
e = event
if e.msg & HC.KEY_ANY:
return {'action':'key_event', 'args':(e.key,)}
if e.msg == HC.GST_TAP:
x, y = e.points[0]
return {'action':'click', 'args':(x, y)}
if e.msg in (HC.GST_SWIPE, HC.GST_DRAG):
sx, sy = e.points[0]
ex, ey = e.points[1]
return {'action':'swipe', 'args':(sx, sy, ex, ey)}
def analyze_frame(self, idx, event, status):
e = event
d = {'frameidx': idx}
if e.msg & HC.KEY_ANY:
d['action'] = 'key_event'
d['args'] = (e.key,)
self.case_draft.append(d)
return
uixml = status['uixml']
screen = status['screen']
adbstatus = status['adbstatus']
activity = adbstatus['activity']
analyze_ui = False
if activity is not None and activity not in self.nonui_activities:
if uixml is not None:
self.uilayout.parse_xmldata(uixml)
analyze_ui = True
if event.msg == HC.GST_TAP:
x, y = event.points[0]
found_ui = False
if analyze_ui:
node = self.uilayout.find_clickable_node(x, y)
if node:
found_ui = True
pnode, selector, order = self.uilayout.find_selector(node)
d['action'] = 'click_ui'
d['args'] = (x, y, self.uilayout.nodes.index(pnode))
d['extra'] = (selector, order)
# try image first when uinode not found.
if not found_ui:
if screen is not None:
bounds = find_clicked_bound(screen, x, y)
d['action'] = 'click_image'
d['args'] = (x, y, tuple(bounds))
else:
d['action'] = 'click'
d['args'] = (x, y)
elif event.msg in (HC.GST_SWIPE, HC.GST_DRAG):
sx, sy = e.points[0]
ex, ey = e.points[1]
d['action'] = 'swipe'
d['args'] = (sx, sy, ex, ey)
elif event.msg == HC.GST_PINCH_IN:
#TODO
pass
else:
return
self.case_draft.append(d)
def process_draft(self, d):
if not d:
return ''
idx = d['frameidx']
frame = self.frames[idx - self.frames[0]['index']]
waittime = frame['waittime']
if d['action'] == 'key_event':
return 'd.keyevent("{}")'.format(*d['args'])
elif d['action'] == 'click':
return 'd.click({}, {})'.format(*d['args'])
elif d['action'] == 'click_ui':
selector, order = d['extra']
if order is None:
return 'd(%s).click(timeout=%d)' %\
(', '.join(['%s=u"%s"' % item for item in selector.iteritems()]), 100*(int(waittime*10)))
else:
return 'objs = d(%s)\nif objs.wait.exists(timeout=%d):\n objs[%d].click()' %\
(', '.join(['%s=u"%s"' % item for item in selector.iteritems()]), 100*(int(waittime*10)), order)
elif d['action'] == 'click_image':
x, y, bounds = d['args']
desc = get_point_desc(x, y, bounds)
screen = cv2.imread(os.path.join(self.framedir, frame['status']['screen']))
l, t, w, h = bounds
img = screen[t:t+h, l:l+w]
imgname = '%d-click.%dx%d.%s.png' % (idx, self.device_info['width'], self.device_info['height'], desc)
imgpath = os.path.join(self.casedir, imgname)
cv2.imwrite(imgpath, img)
return 'd.click_image("%s")' % (imgname,)
elif d['action'] == 'swipe':
sx, sy, ex, ey = d['args']
return 'd.swipe(%s, %s, %s, %s, 10)\ntime.sleep(%.2f)' % (sx, sy, ex, ey, waittime)
else:
print 'unsupported action', d['action']
return ''
class AndroidRecorder(BaseRecorder, ScreenAddon, UixmlAddon, AdbStatusAddon):
def __init__(self, *args, **kwargs):
self.hm = HookManager()
super(AndroidRecorder, self).__init__(*args, **kwargs)
self.hm.register(HC.KEY_ANY, self.on_key)
self.hm.register(HC.TOUCH_DOWN, self.input_event)
self.hm.register(HC.GST_TAP, self.input_event)
self.hm.register(HC.GST_SWIPE, self.input_event)
self.hm.register(HC.GST_PINCH, self.input_event)
self.uilayout = AndroidLayout()
self.nonui_activities = set()
def attach(self, device):
if self.device is not None:
self.detach()
self.device = device
self.hm.set_serial(self.device.serial)
def detach(self):
self.unhook()
self.device = None
self.hm.set_serial(None)
def hook(self):
self.hm.hook()
def unhook(self):
self.hm.unhook()
def add_nonui_activity(self, activity):
self.nonui_activities.add(activity)
def remove_nonui_activity(self, activity):
self.nonui_activities.discard(activity)
def on_key(self, event):
if not event.msg & 0x01: # key_up
self.input_event(event)
def analyze_frame(self, idx, event, status):
e = event
if e.msg & HC.KEY_ANY:
d = {
'action' : 'keyevent',
'args' : (e.key,),
'pyscript' : 'd.keyevent("%s")' % (e.key,)
}
self.case_draft.append(d)
return
uixml = status['uixml']
screen = status['screen']
adbstatus = status['adbstatus']
activity = adbstatus['activity']
analyze_ui = False
if activity is not None and activity not in self.nonui_activities:
print 'in activity:', activity
if uixml is not None:
self.uilayout.parse_xmldata(uixml)
analyze_ui = True
event.msg = HC.GST_TAP
d = {}
if event.msg == HC.GST_TAP:
x, y = event.x, event.y
if analyze_ui:
node = self.uilayout.find_clickable_rect(x, y)
if node:
print node.bounds, x, y
d['action'] = 'click_ui'
d['args'] = (node.class_name, node.resource_id, node.index)
d['pyscript'] = 'd(className="%s", resourceId="%s", index="%s").click()' % d['args']
else:
d['action'] = 'click'
d['args'] = (x, y)
d['pyscript'] = 'd.click(%s, %s)' % d['args']
elif screen is not None:
d['action'] = 'click_image'
img = find_clicked_img(screen, x, y)
imgname = '%d-click.png' % (idx,)
imgpath = os.path.join(self.draftdir, imgname)
cv2.imwrite(imgpath, img)
d['args'] = ()
d['pyscript'] = 'd.click_image()'
else:
d['action'] = 'click'
d['args'] = (x, y)
d['pyscript'] = 'd.click(%s, %s)' % d['args']
elif event.msg == HC.GST_SWIPE:
#TODO
# sx, sy, ex, ey = event.sx, event.sy, event.ex, event.ey
pass
elif event.msg == HC.GST_PINCH:
#TODO
pass
else:
return
self.case_draft.append(d)
class AndroidRecorder(BaseRecorder, ScreenAddon, UixmlAddon, AdbStatusAddon):
def __init__(self, *args, **kwargs):
self.hm = HookManager()
self.uilayout = AndroidLayout()
self.nonui_activities = set()
self.scene_detector = kwargs.pop('scene_detector', None)
super(AndroidRecorder, self).__init__(*args, **kwargs)
self.hm.register(HC.KEY_ANY, self.on_key)
self.hm.register(HC.GST_TAP, self.input_event)
self.hm.register(HC.GST_SWIPE, self.input_event)
self.hm.register(HC.GST_DRAG, self.input_event)
def attach(self, device):
if self.device is not None:
self.detach()
self.device = device
self.hm.set_serial(self.device.serial)
def detach(self):
self.unhook()
self.device = None
self.hm.set_serial(None)
def hook(self):
self.hm.hook()
def unhook(self):
self.hm.unhook()
def add_nonui_activity(self, activity):
self.nonui_activities.add(activity)
def remove_nonui_activity(self, activity):
self.nonui_activities.discard(activity)
def on_key(self, event):
if not event.msg & 0x01: # key_up
print 'KeyEvent', event.key
self.input_event(event)
def serialize_event(self, event):
e = event
if e.msg & HC.KEY_ANY:
return {'action':'keyevent', 'args':(e.key,)}
if e.msg == HC.GST_TAP:
x, y = e.points[0]
return {'action':'touch', 'args':(x, y)}
if e.msg in (HC.GST_SWIPE, HC.GST_DRAG):
sx, sy = e.points[0]
ex, ey = e.points[1]
return {'action':'swipe', 'args':(sx, sy, ex, ey)}
def analyze_frame(self, idx, event, status, waittime):
e = event
if e.msg & HC.KEY_ANY:
d = {
'action' : 'keyevent',
'args' : (e.key,),
'pyscript' : 'd.keyevent("%s")' % (e.key,)
}
self.case_draft.append(d)
return
uixml = status['uixml']
screen = status['screen']
adbstatus = status['adbstatus']
activity = adbstatus['activity']
analyze_ui = False
if activity is not None and activity not in self.nonui_activities:
if uixml is not None:
self.uilayout.parse_xmldata(uixml)
analyze_ui = True
d = {}
if event.msg == HC.GST_TAP:
x, y = event.points[0]
if analyze_ui:
node = self.uilayout.find_clickable_node(x, y)
if node:
selector, order = self.uilayout.find_selector(node)
d['action'] = 'click_ui'
d['args'] = (selector, order)
if order is None:
d['pyscript'] = 'd(%s).click(timeout=%d)' %\
(', '.join(['%s=u"%s"' % item for item in selector.iteritems()]), 100*(int(waittime*10)))
else:
d['pyscript'] = 'objs = d(%s)\nif objs.wait.exists(timeout=%d):\n objs[%d].click()' %\
(', '.join(['%s=u"%s"' % item for item in selector.iteritems()]), 100*(int(waittime*10)), order)
else:
d['action'] = 'click'
d['args'] = (x, y)
d['pyscript'] = 'd.click(%s, %s)' % (x, y)
elif screen is not None:
d['action'] = 'click_image'
img, bounds = find_clicked_img(screen, x, y)
imgname = '%d-click.png' % (idx,)
imgpath = os.path.join(self.draftdir, imgname)
cv2.imwrite(imgpath, img)
d['args'] = ((x, y), tuple(bounds))
d['pyscript'] = 'd.click_image("%s")' % (imgname,)
else:
d['action'] = 'click'
d['args'] = (x, y)
d['pyscript'] = 'd.click(%s, %s)' % (x, y)
elif event.msg in (HC.GST_SWIPE, HC.GST_DRAG):
sx, sy = e.points[0]
ex, ey = e.points[1]
d['action'] = 'swipe'
d['args'] = (sx, sy, ex, ey)
d['pyscript'] = 'd.swipe(%s, %s, %s, %s, 10)' % (sx, sy, ex, ey)
elif event.msg == HC.GST_PINCH_IN:
#TODO
pass
else:
return
self.case_draft.append(d)
class AndroidAtomRecorder(BaseRecorder):
'''record detailed events: TOUCH_UP, TOUCH_DOWN, TOUCH_MOVE, KEY_UP, KEY_DOWN'''
def __init__(self, *args, **kwargs):
self.hm = HookManager()
super(AndroidAtomRecorder, self).__init__(*args, **kwargs)
self.hm.register(HC.KEY_ANY, self.input_event)
self.hm.register(HC.TOUCH_ANY, self.input_event)
def attach(self, device):
if self.device is not None:
self.detach()
self.device = device
self.hm.set_serial(self.device.serial)
def detach(self):
self.unhook()
self.device = None
self.hm.set_serial(None)
def hook(self):
self.hm.hook()
def unhook(self):
self.hm.unhook()
def analyze_frame(self, idx, event, status):
e = event
if e.msg == HC.TOUCH_MOVE:
d = {
'action': 'touch_move',
'args': (e.slotid, e.x, e.y),
}
elif e.msg == HC.TOUCH_DOWN:
d = {
'action': 'touch_down',
'args': (e.slotid, e.x, e.y),
}
elif e.msg == HC.TOUCH_UP:
d = {
'action': 'touch_up',
'args': (e.slotid, e.x, e.y),
}
elif e.msg & HC.KEY_ANY:
if e.msg & 0x01:
d = {
'action': 'key_down',
'args': (e.key, ),
}
else:
d = {
'action': 'key_up',
'args': (e.key, ),
}
else:
return
self.case_draft.append(d)
def process_draft(self, d):
tmpl = {
'touch_move': 'd.touch_move({:d}, {:d}, {:d})',
'touch_down': 'd.touch_down({:d}, {:d}, {:d})',
'touch_up': 'd.touch_up({:d}, {:d}, {:d})',
'key_down': 'd.key_down("{}")',
'key_up': 'd.key_up("{}")',
}
return tmpl[d['action']].format(*d['args'])
class AndroidRecorder(BaseRecorder, ScreenAddon, UixmlAddon, AdbStatusAddon,
RotationAddon):
def __init__(self, *args, **kwargs):
self.hm = HookManager()
self.uilayout = AndroidLayout()
self.nonui_activities = set()
self.scene_detector = kwargs.pop('scene_detector', None)
super(AndroidRecorder, self).__init__(*args, **kwargs)
self.hm.register(HC.KEY_ANY, self.on_key)
self.hm.register(HC.GST_TAP, self.input_event)
self.hm.register(HC.GST_SWIPE, self.input_event)
self.hm.register(HC.GST_DRAG, self.input_event)
def attach(self, device):
if self.device is not None:
self.detach()
self.device = device
self.hm.set_serial(self.device.serial)
def detach(self):
self.unhook()
self.device = None
self.hm.set_serial(None)
def hook(self):
self.hm.hook()
def unhook(self):
self.hm.unhook()
def add_nonui_activity(self, activity):
self.nonui_activities.add(activity)
def remove_nonui_activity(self, activity):
self.nonui_activities.discard(activity)
def on_key(self, event):
if not event.msg & 0x01: # key_up
print 'KeyEvent', event.key
self.input_event(event)
def serialize_event(self, event):
e = event
if e.msg & HC.KEY_ANY:
return {'action': 'key_event', 'args': (e.key, )}
if e.msg == HC.GST_TAP:
x, y = e.points[0]
return {'action': 'click', 'args': (x, y)}
if e.msg in (HC.GST_SWIPE, HC.GST_DRAG):
sx, sy = e.points[0]
ex, ey = e.points[1]
return {'action': 'swipe', 'args': (sx, sy, ex, ey)}
def analyze_frame(self, idx, event, status):
e = event
d = {'frameidx': idx}
if e.msg & HC.KEY_ANY:
d['action'] = 'key_event'
d['args'] = (e.key, )
self.case_draft.append(d)
return
uixml = status['uixml']
screen = status['screen']
adbstatus = status['adbstatus']
activity = adbstatus and adbstatus['activity'] or None
rotation = status['rotation']
analyze_ui = False
if activity is not None and activity not in self.nonui_activities:
if uixml is not None:
self.uilayout.parse_xmldata(uixml)
rotation = self.uilayout.rotation
analyze_ui = True
# default rotation
if rotation is None:
rotation = 0
w, h = self.device_info['width'], self.device_info['height']
if e.msg == HC.GST_TAP:
x, y = touch2screen(w, h, rotation, e.points[0][0], e.points[0][1])
found_ui = False
if analyze_ui:
node = self.uilayout.find_clickable_node(x, y)
if node:
found_ui = True
pnode, selector, order = self.uilayout.find_selector(node)
d['action'] = 'click_ui'
d['args'] = (pnode.iterindex, )
d['extra'] = (selector, order)
# try image first when uinode not found.
if not found_ui:
if screen is not None:
bounds = find_clicked_bound(screen, x, y)
d['action'] = 'click_image'
d['args'] = (x, y, tuple(bounds))
else:
d['action'] = 'click'
d['args'] = (x, y)
elif e.msg in (HC.GST_SWIPE, HC.GST_DRAG):
sx, sy = touch2screen(w, h, rotation, e.points[0][0],
e.points[0][1])
ex, ey = touch2screen(w, h, rotation, e.points[1][0],
e.points[1][1])
d['action'] = 'swipe'
d['args'] = (sx, sy, ex, ey)
elif e.msg == HC.GST_PINCH_IN:
#TODO
pass
else:
return
self.case_draft.append(d)
def process_draft(self, d):
if not d:
return ''
idx = int(d['frameidx'])
frame = self.frames[idx]
waittime = frame['waittime']
if d['action'] == 'key_event':
return 'd.keyevent("{}")'.format(*d['args'])
elif d['action'] == 'click':
return 'd.click({}, {})'.format(*d['args'])
elif d['action'] == 'click_ui':
if 'extra' in d:
selector, order = d['extra']
else:
uixml = open(
os.path.join(self.framedir,
frame['status']['uixml'])).read()
self.uilayout.parse_xmldata(uixml)
pnode = self.uilayout.get_index_node(int(d['args'][0]))
selector, order = self.uilayout.get_node_selector(pnode)
if order is None:
return 'd(%s).click(timeout=%d)' %\
(', '.join(['%s=u"%s"' % item for item in selector.iteritems()]), 100*(int(waittime*10)))
else:
return 'objs = d(%s)\nif objs.wait.exists(timeout=%d):\n objs[%d].click()' %\
(', '.join(['%s=u"%s"' % item for item in selector.iteritems()]), 100*(int(waittime*10)), order)
elif d['action'] == 'click_image':
x, y, bounds = d['args']
x, y, bounds = int(x), int(y), map(int, bounds)
desc = get_point_desc(x, y, bounds)
screen = cv2.imread(
os.path.join(self.framedir, frame['status']['screen']))
l, t, w, h = bounds
img = screen[t:t + h, l:l + w]
imgname = '%d-click.%dx%d.%s.png' % (
idx, self.device_info['width'], self.device_info['height'],
desc)
imgpath = os.path.join(self.casedir, imgname)
cv2.imwrite(imgpath, img)
return 'd.click_image("%s")' % (imgname, )
elif d['action'] == 'swipe':
sx, sy, ex, ey = d['args']
if waittime == 0:
return 'd.swipe(%s, %s, %s, %s, 10)' % (sx, sy, ex, ey)
return 'time.sleep(%.2f)\nd.swipe(%s, %s, %s, %s, 10)' % (
waittime, sx, sy, ex, ey)
else:
print 'unsupported action', d['action']
return ''