def drag_screen_to_left(device):
Saver.save_crawler_log(device.logPath, "Step : drag screen to left")
# x_max = str(int(device.screenResolution[0]) - 50)
# x_min = str(int(resolution[0]) * 0.5)[:str(int(resolution[0]) * 0.5).index('.')]
# command = 'adb -s ' + device.id + ' shell input swipe ' + x_max + ' 100 ' + '20' + ' 100'
command = 'adb -s ' + device.id + ' shell input keyevent 22'
os.popen(command)
def node_is_shown_in_page(device, node, page):
if node in page.nodesList:
Saver.save_crawler_log(device.logPath, "node is shown in page now")
return True
else:
Saver.save_crawler_log(device.logPath, "node is not shown in page now")
return False
def pages_are_different(device, page1, page2):
if page1.nodesList == page2.nodesList:
Saver.save_crawler_log(device.logPath, "pages are same ")
return False
else:
Saver.save_crawler_log(device.logPath, "pages are different ")
return True
def get_nodes_list(device):
Saver.save_crawler_log(device.logPath, "Step : get nodes list")
try:
dom = xml.dom.minidom.parse(device.logPath + '/Uidump.xml')
root = dom.documentElement
nodes = root.getElementsByTagName('node')
return nodes
except:
return ''
def close_sys_alert(plan, app, device, page_now):
if page_now is None:
return page_now
for node in page_now.clickableNodes:
info = [node.package, node.resource_id, node.text]
if info in Setting.AuthorizationAlert:
Saver.save_crawler_log(device.logPath, "Step : close sys alert")
save_screen(device, node, False)
tap_node(device, node)
page_now = get_page_info(plan, app, device)
return page_now
def save_screen(device, node, model):
if Setting.SaveScreen:
try:
global screenshot_num
Saver.save_crawler_log(device.logPath, "Step : save screenshot ")
get_screenshot_command = 'adb -s ' + device.id + ' shell /system/bin/screencap -p /sdcard/screenshot.png'
activity = node.currentActivity
resource_id = node.resource_id
resource_id = resource_id[resource_id.find('/') + 1:]
location = node.location
if model:
local_png = device.screenshotPath + '/' + str(
screenshot_num) + '-' + str(activity) + '-' + str(
resource_id) + '-' + str(location[0]) + '-' + str(
location[1]) + '.png'
else:
local_png = device.screenshotPath + '/' + str(
screenshot_num) + '-' + 'unCrawl' + '-' + str(
activity) + '-' + str(resource_id) + '-' + str(
location[0]) + '-' + str(location[1]) + '.png'
pull_screenshot_command = 'adb -s ' + device.id + ' pull /sdcard/screenshot.png ' + local_png
os.popen(get_screenshot_command)
os.popen(pull_screenshot_command)
screenshot_num += 1
# command = 'adb shell screencap -p | gsed s/' + '\r' + '$//> ' + local_png
# os.popen(command)
bounds = node.bounds
image = pl.array(Image.open(local_png))
pl.figure(figsize=(float(device.screenResolution[0]) / 100,
float(device.screenResolution[1]) / 100),
dpi=100)
pl.imshow(image)
x = [bounds[0], bounds[0], bounds[2], bounds[2], bounds[0]]
y = [bounds[1], bounds[3], bounds[3], bounds[1], bounds[1]]
pl.axis('off')
pl.axis('scaled')
pl.axis([
0,
int(device.screenResolution[0]),
int(device.screenResolution[1]), 0
])
pl.plot(x[:5], y[:5], 'r', linewidth=2)
pl.savefig(local_png)
im = Image.open(local_png)
box = (float(device.screenResolution[0]) / 8,
float(device.screenResolution[1]) / 9,
float(device.screenResolution[0]) * 65 / 72,
float(device.screenResolution[1]) * 8 / 9)
region = im.crop(box)
region.save(local_png)
pl.close()
except:
Saver.save_crawler_log(device.logPath, "save screen error")
def get_device_list(self):
device_list = []
string = ' '
outLine = os.popen('adb devices').readlines()
for line in outLine:
if string in line:
device_id = line[0:line.index(string)]
device = Device(self, device_id)
device_list.append(device)
index = device_list.index(device)
device.update_device_account(Setting.AccountList[index])
Saver.save_crawler_log(self.logPath, device_list)
return device_list
def get_screen_resolution(self):
Saver.save_crawler_log(self.logPath, "Step : get screen resolution")
command = 'adb -s ' + self.id + ' shell wm size'
resolution = []
result = os.popen(command).readlines()
for line in result:
if 'Physical size: ' in line:
r = re.findall(r'\d+', line)
x = r[0]
y = r[1]
resolution.append(x)
resolution.append(y)
Saver.save_crawler_log(self.logPath, resolution)
return resolution
def get_device_name(self):
# linux:
# adb shell cat /system/build.prop | grep "product"
# windows
# adb -s 84B5T15A10010101 shell cat /system/build.prop | findstr "product"
command = 'adb -s ' + self.id + ' shell cat /system/build.prop | grep "product" '
result = os.popen(command).readlines()
for line in result:
key = 'ro.product.model='
if key in line:
device_name = line[line.find(key) + len(key):-2]
break
Saver.save_crawler_log(self.logPath, "device name : " + device_name)
return device_name
def save_logcat(plan, device, finish):
Saver.save_crawler_log(plan.logPath,
"Step : save device log : " + device.id)
if not os.path.exists(os.getcwd()):
os.makedirs(os.getcwd())
command = 'adb -s ' + device.id + ' logcat -d >> ' + device.logPath + '/logcat.txt'
os.popen(command)
if not finish:
get_log_commend = 'adb -s ' + device.id + ' logcat -d'
log = os.popen(get_log_commend).readlines()
for line in log:
if line.find('System.err') != -1:
device.update_crawl_statue('HasCrashed')
elif line.find('ANR') != -1:
device.update_crawl_statue('HasANR')
def start_activity(device, packagename, activity):
Saver.save_crawler_log(device.logPath,
'Step : start up activity : ' + activity)
time1 = time.time()
result = True
while result:
command = 'adb -s ' + device.id + ' shell am start -n ' + packagename + '/' + activity
Saver.save_crawler_log(device.logPath,
'start up activity: ' + activity)
os.popen(command)
if time.time() - time1 < 10:
top_activity_info = get_top_activity_info(device)
top_packagename = top_activity_info['packagename']
if top_packagename == packagename:
result = False
else:
result = False
def save_screen_jump_out(device, package, activity):
if Setting.SaveJumpOutScreen:
try:
global jump_out_time, screenshot_num
Saver.save_crawler_log(device.logPath,
"Step : jump out . save screenshot ")
get_screenshot_command = 'adb -s ' + device.id + ' shell /system/bin/screencap -p /sdcard/screenshot.png'
local_png = device.screenshotPath + '/' + str(
screenshot_num) + '-' + str(package) + '-' + str(
activity) + '-Jump' + str(jump_out_time) + '.png'
pull_screenshot_command = 'adb -s ' + device.id + ' pull /sdcard/screenshot.png ' + local_png
os.popen(get_screenshot_command)
os.popen(pull_screenshot_command)
screenshot_num += 1
jump_out_time += 1
except:
Saver.save_crawler_log(device, "save screen error")
def __init__(self, plan, device_id):
Saver.save_crawler_log(plan.logPath, "Step : Init device : " + device_id)
self.id = device_id
Saver.save_crawler_log(plan.logPath, "id : " + self.id)
self.logPath = self.create_device_folder(plan)
self.name = self.get_device_name()
self.model = self.get_device_model()
self.version = self.get_device_sys_version()
self.accountInfo = []
self.screenResolution = self.get_screen_resolution()
self.screenshotPath = self.create_screenshot_folder()
self.beginCrawlTime = int(time.strftime('%Y%m%d%H%M%S', time.localtime(time.time())))
self.unCrawledNodes = []
self.hasCrawledNodes = []
self.hasCrawlPage = []
self.hasCrawledActivities = []
self.crawlStatue = "Inited"
def get_top_activity_info(device):
Saver.save_crawler_log(device.logPath, "Step : get top activity info")
# linux:
# adb shell dumpsys activity | grep "mFocusedActivity"
# windows:
# adb shell dumpsys activity | findstr "mFocusedActivity"
info = {}
# command = 'adb -s ' + device.id + ' shell dumpsys activity | grep "mFocusedActivity"'
# sometime mResumedActivity is Right
command = 'adb -s ' + device.id + ' shell dumpsys activity | grep "mResumedActivity"'
result = os.popen(command).read()
packagename = ''
activity = ''
if 'u0' not in result and ' com.' not in result:
result = os.popen(command).read()
if 'u0 ' in result:
packagename = result[result.find('u0 ') + len('u0 '):result.find('/')]
elif ' com.' in result:
packagename = result[result.find(' com.') + 1:result.find('/')]
if ' t' in result:
activity = result[result.find('/') + len('/'):result.find(' t')]
elif '}' in result:
activity = result[result.find('/') + len('/'):result.find('}')]
info['packagename'] = packagename
info['activity'] = activity
Saver.save_crawler_log(device.logPath, 'Top activity is :' + activity)
Saver.save_crawler_log(device.logPath, 'Top package is :' + packagename)
return info
def recover_node_shown(plan, app, device, page_now, page_before_run, node):
t = 1
r = False
while page_now is not None and page_now.nodesNum != 0 and node.nodeInfo not in page_now.nodesInfoList:
if get_node_recover_way(device, page_now, page_before_run, node, []):
r = True
break
Saver.save_crawler_log(device.logPath,
"Step : no recover way , click back")
save_screen_jump_out(device, page_now.package,
page_now.currentActivity)
click_back(device)
page_now = get_page_info(plan, app, device)
t += 1
if t > 3:
Saver.save_crawler_log(device.logPath,
"can't find the node after back 3 times.")
break
if r:
Saver.save_crawler_log(device.logPath, "Step : recover node shown")
for n in node.recoverWay:
save_screen(device, n, False)
if n.crawlOperation == 'tap':
tap_node(device, n)
elif n.crawlOperation == 'longclick':
long_click_node(device, n)
elif n.crawlOperation == 'type':
t = get_random_text(8)
type_text(device, n, t)
check_page_after_operation(plan, app, device)
if t < 4:
r = True
return r
def crawl_edittext(plan, app, device, page_before_run, page_now, init):
for node in get_random_nodes(page_before_run.editTexts):
# if crash and not keep run , break from deep run .page_need_crawled will be None
if page_now is None:
Saver.save_crawler_log(device.logPath, 'Jump out to crawl')
break
# sometimes the need tap node is not shown after one deep run
if not recover_node_shown(plan, app, device, page_now, page_before_run,
node):
continue
save_screen(device, node, True)
text = get_random_text(8)
type_text(device, node, text)
device.update_crawled_activity(node.currentActivity)
device.update_crawled_nodes(node.nodeInfo)
device.delete_uncrawled_nodes(node.nodeInfo)
# if jump out the test app, try to go back & return the final page
page_after_operation = check_page_after_operation(plan, app, device)
if page_after_operation is None:
Saver.save_crawler_log(device.logPath, 'Jump out to crawl')
page_now = page_after_operation
break
# compare two pages before & after click .
# update the after page . leave the new clickable/scrollable/longclickable/edittext nodes only.
page_now = get_need_crawl_page(plan, app, device, page_before_run,
page_after_operation)
if page_is_crawlable(plan, app, device, page_now):
page_now.add_last_page(page_before_run)
page_now.add_entry(node)
# deep run
if init:
page_now = crawl_init_nodes(plan, app, device, page_now)
else:
page_now = crawl_main_nodes(plan, app, device, page_now)
else:
page_now = page_after_operation
# if page no crawlable nodes , back to last Page, until has crawlable nodes, if back time >3, break
page_now = recover_page_to_crawlable(plan, app, device, page_now)
return page_now
def get_page_info(plan, app, device):
if Setting.TimeModel == 'Limit':
time_now = int(
time.strftime('%Y%m%d%H%M%S', time.localtime(time.time())))
print time_now
print time_now - device.beginCrawlTime
print Setting.LimitTime * 100
if (time_now - device.beginCrawlTime) > (Setting.LimitTime * 100):
Saver.save_crawler_log_both(
plan.logPath, device.logPath,
"Step : crawl time out , finish crawl.")
return None
Saver.save_crawler_log(device.logPath, "get all nodes in this page")
page = Page()
result = False
while not result:
try:
get_uidump_xml_file(device)
dom = xml.dom.minidom.parse(device.logPath + '/Uidump.xml')
result = True
except:
result = False
root = dom.documentElement
nodes = root.getElementsByTagName('node')
Saver.save_crawler_log(device.logPath, len(nodes))
info = get_top_activity_info(device)
for node in nodes:
n = Node(node)
n.update_current_activity(info['activity'])
if n.resource_id in app.firstClickViews:
save_screen(device, n, False)
tap_node(device, n)
page = get_page_info(plan, app, device)
if page is not None:
page.add_entry(n)
page.add_node(device, app, n)
page = close_sys_alert(plan, app, device, page)
return page
def get_need_crawl_page(plan, app, device, page_before_run, page_after_run):
Saver.save_crawler_log(device.logPath,
"Step : get need crawl page now ...")
if len(page_after_run.nodesList) == 0:
page_after_run = get_page_info(plan, app, device)
if page_after_run is not None and len(page_after_run.nodesList) != 0:
for node in page_after_run.nodesList:
if node in page_before_run.nodesList:
page_after_run.remove_clickable_node(node)
page_after_run.remove_scrollable_node(node)
page_after_run.remove_longclickable_node(node)
page_after_run.remove_edit_text(node)
# after type text in edit text, text & bounds will change , don't need to crawl the edit text again
if node.isEditText:
info = [
node.index, node.resource_id, node.package,
node.content_desc
]
for n in page_after_run.editTexts:
i = [n.index, n.resource_id, n.package, n.content_desc]
if i == info:
page_after_run.remove_edit_text(n)
break
return page_after_run
def recover_page_to_crawlable(plan, app, device, page_now):
t = 1
while page_now is not None and no_uncrawled_clickable_nodes_now(plan, device, page_now) \
and no_uncrawled_longclickable_nodes_now(plan, device, page_now) \
and no_uncrawled_edit_text_now(plan, device, page_now):
if page_now.backBtn is not None \
and node_is_shown_in_page(device, page_now.backBtn, page_now):
Saver.save_crawler_log(device.logPath,
"Step : find the back btn and tap ")
save_screen(device, page_now.backBtn, False)
tap_node(device, page_now.backBtn)
t += 1
page_now = get_page_info(plan, app, device)
else:
Saver.save_crawler_log(device.logPath,
"Step : no back btn , click back")
save_screen_jump_out(device, page_now.package,
page_now.currentActivity)
click_back(device)
page_now = get_page_info(plan, app, device)
t += 1
if t > 2:
break
return page_now
def app_is_running(device, app):
Saver.save_crawler_log(device.logPath,
"Step : check app is running or not")
command = "adb -s " + device.id + " shell top -n 1 | grep " + app.packageName
output = os.popen(command)
lines = output.readlines()
if len(lines) == 0:
Saver.save_crawler_log(device.logPath, "app is not running")
return False
else:
Saver.save_crawler_log(device.logPath, "app is running")
return True
def keyboard_is_shown(device):
Saver.save_crawler_log(device.logPath, "Step : check keyboard")
command = 'adb -s ' + device.id + ' shell dumpsys input_method'
result = os.popen(command).read()
key = 'mInputShown='
keyboard_status = result[result.index(key) + len(key):result.index(key) +
len(key) + 5]
if keyboard_status == 'true':
Saver.save_crawler_log(device.logPath, "keyboard is shown ")
return True
else:
Saver.save_crawler_log(device.logPath, "keyboard is not shown")
return False
def page_is_crawlable(plan, app, device, page):
Saver.save_crawler_log(device.logPath,
"Step : check page is crawlable or not")
if page.nodesInfoList not in device.hasCrawlPage \
and page.package == app.packageName \
and (page.clickableNodesNum != 0
or page.scrollableNodesNum != 0
or page.longClickableNodesNum != 0
or page.editTextsNum != 0):
Saver.save_crawler_log(device.logPath, "page is crawlable")
return True
else:
Saver.save_crawler_log(device.logPath, "page is not crawlable")
return False
def get_clickable_nodes(device):
Saver.save_crawler_log(device.logPath, "Step : get clickable nodes")
nodes = get_nodes_list(device)
clickable_nodes = []
if not len(nodes) == 0:
for node in nodes:
if not node_is_edittext(node) and node_is_clickable(node):
node_info = {
'index': node.getAttribute('index'),
'text': node.getAttribute('text'),
'resource_id': node.getAttribute('resource-id'),
'class': node.getAttribute('class'),
'package': node.getAttribute('package'),
'bounds': node.getAttribute('bounds'),
'content_desc': node.getAttribute('content-desc')
}
clickable_nodes.append(node_info)
Saver.save_crawler_log(device.logPath, clickable_nodes)
Saver.save_crawler_log(device.logPath, len(clickable_nodes))
else:
Saver.save_crawler_log(device.logPath, "no clickable nodes ...")
return clickable_nodes
def no_uncrawled_edit_text_now(plan, device, page_now):
if page_now is None:
return True
Saver.save_crawler_log(
device.logPath,
"Step : Check there are uncCrawled editTexts in the page now or not")
result = True
for node in page_now.editTexts:
if device.is_in_uncrawled_nodes(node.nodeInfo):
result = False
break
if result:
Saver.save_crawler_log(device.logPath,
"no uncrawled editTexts in this page now")
return True
else:
Saver.save_crawler_log(
device.logPath, "have some uncrawled editTexts in this page now")
return False
def no_uncrawled_clickable_nodes_now(plan, device, page_now):
if page_now is None:
return True
Saver.save_crawler_log(
device.logPath,
"Step : Check there are uncCrawled clickable Nodes in the page now or not"
)
result = True
for node in page_now.nodesList:
if node_is_clickable(node) and device.is_in_uncrawled_nodes(
node.nodeInfo):
result = False
break
if result:
Saver.save_crawler_log(
device.logPath, "no uncrawled clickable nodes in this page now")
return True
else:
Saver.save_crawler_log(
device.logPath,
"have some uncrawled clickable nodes in this page now")
return False
def update_device_account(self, account):
Saver.save_crawler_log(self.logPath, "Step : Update account : " + str(account))
self.accountInfo = account
def update_crawl_statue(self, statue):
Saver.save_crawler_log(self.logPath, "Step : Update crawl statue from " + self.crawlStatue + ' to ' + statue)
self.crawlStatue = statue
def get_device_sys_version(self):
command = 'adb -s ' + self.id + ' shell getprop ro.build.version.release'
result = os.popen(command).read()
Saver.save_crawler_log(self.logPath, "sys version : " + result)
version = int(result.replace('.', ''))
return version
def get_device_model(self):
command = 'adb -s ' + self.id + ' shell getprop ro.product.model'
model = os.popen(command).read()
Saver.save_crawler_log(self.logPath, "device model : " + model)
return model
def create_screenshot_folder(self):
path = self.logPath + '/Screenshot'
Saver.save_crawler_log(self.logPath, "Step : creat screenshot folder")
if not os.path.exists(path):
os.makedirs(path)
return path