def start(self, interval, start_time):
self.interval = interval
self.starttime = start_time
self.dataworker_thread = threading.Thread(
target=self._handle_data_thread)
self.dataworker_thread.start()
logger.debug("DataWorker started...")
def killOccupy5037Process():
if ADB.get_os_name() == "Windows":
sub = subprocess.Popen('netstat -ano|findstr \"5037\"',
stdout=subprocess.PIPE,
shell=True)
ret = sub.stdout.read()
sub.wait()
if not ret:
logger.debug("netstat is empty")
return
lines = ret.splitlines()
for line in lines:
if "LISTENING" in line:
logger.debug(line)
pid = line.split()[-1]
sub = subprocess.Popen('tasklist |findstr %s' % pid,
stdout=subprocess.PIPE,
shell=True)
ret = sub.stdout.read()
sub.wait()
process = ret.split()[0]
logger.debug("pid:%s ,process:%s occupy 5037 port" %
(pid, process))
# DDMS会用到adb 杀了adb会导致 IDE调试或控制台可能不正常，后面需要改环境变量
subprocess.Popen("taskkill /T /F /PID %s" % pid,
stdout=subprocess.PIPE,
shell=True)
logger.debug("kill process %s" % process)
break
else:
logger.debug("don't have process occupy 5037")
def _collect_thread_num_thread(self, start_time):
end_time = time.time() + self._timeout
thread_list_titile = ("datatime", "packagename", "pid", "thread_num")
thread_num_file = os.path.join(RuntimeData.package_save_path,
'thread_num.csv')
try:
with open(thread_num_file, 'a+') as df:
csv.writer(df,
lineterminator='\n').writerow(thread_list_titile)
if self.thread_queue:
thread_file_dic = {'thread_file': thread_num_file}
self.thread_queue.put(thread_file_dic)
except RuntimeError as e:
logger.error(e)
while not self._stop_event.is_set() and time.time() < end_time:
try:
before = time.time()
logger.debug(
"-----------into _collect_thread_num_thread loop, thread is : "
+ str(threading.current_thread().name))
# 获取pakagename的thread num信息
thread_pck_info = self.get_process_thread_num(self.packagename)
logger.debug(thread_pck_info)
current_time = TimeUtils.getCurrentTime()
if not thread_pck_info:
continue
else:
logger.debug("current time: " + current_time +
", processname: " + thread_pck_info[1] +
", pid: " + str(thread_pck_info[2]) +
" thread num: " + str(thread_pck_info[3]))
if self.thread_queue:
self.thread_queue.put(thread_pck_info)
if not self.thread_queue: #为了本地单个文件运行
try:
with open(thread_num_file, 'a+') as thread_writer:
writer_p = csv.writer(thread_writer,
lineterminator='\n')
thread_pck_info[0] = current_time
writer_p.writerow(thread_pck_info)
except RuntimeError as e:
logger.error(e)
after = time.time()
time_consume = after - before
delta_inter = self._interval - time_consume
logger.debug("time_consume for thread num infos: " +
str(time_consume))
if delta_inter > 0:
time.sleep(delta_inter)
except:
logger.error(
"an exception hanpend in thread num thread, reason unkown!"
)
s = traceback.format_exc()
logger.debug(s)
if self.thread_queue:
self.thread_queue.task_done()
def check_config_option(self, config_dic, parse, section, option):
if parse.has_option(section, option):
try:
config_dic[option] = parse.get(section, option)
if option == 'frequency':
config_dic[option] = (int)(parse.get(section, option))
if option == 'timeout':#timeout 的单位是分钟
config_dic[option] = (int)(parse.get(section, option))*60
if option in ["exceptionlog" ,"phone_log_path","space_size_check_path","package","pid_change_focus_package",
"watcher_users","main_activity","activity_list"]:
if option == "activity_list" or option == "main_activity":
config_dic[option] = parse.get(section, option).strip().replace("\n","").split(";")
else:
config_dic[option] = parse.get(section, option).split(";")
except:#配置项中数值发生错误
if option != 'serialnum':
logger.debug("config option error:"+option)
self._config_error()
else:
config_dic[option] = ''
else:#配置项没有配置
if option not in ['serialnum',"main_activity","activity_list","pid_change_focus_package","shell_file"]:
logger.debug("config option error:" + option)
self._config_error()
else:
config_dic[option] = ''
return config_dic
def update_launch_list(self, content, timestamp):
# if self.packagename in content[1]:
self.launch_list.append(content)
tmp_file = os.path.join(RuntimeData.package_save_path,
'launch_logcat.csv')
perf_data = {
"task_id": "",
'launch_time': [],
'cpu': [],
"mem": [],
'traffic': [],
"fluency": [],
'power': [],
}
dic = {
"time": timestamp,
"act_name": content[1],
"this_time": content[2],
"total_time": content[3],
"launch_type": content[4]
}
perf_data['launch_time'].append(dic)
# perf_queue.put(perf_data)
with open(tmp_file, "a+") as f:
csvwriter = csv.writer(f, lineterminator='\n') #这种方式可以去除csv的空行
logger.debug("save launchtime data to csv: " +
str(self.launch_list))
csvwriter.writerows(self.launch_list)
del self.launch_list[:]
def _parse(self):
sp_lines = self.source.split('\n')
for line in sp_lines:
if self.uid and self.uid in line:
# logger.debug(" target uid : "+str(self.uid))
tart_list = line.split()
tag = tart_list[2]
# logger.debug(" tag is： " +tag)
if tag == '0x0': #tag即acct_tag_hex这一列，默认是0，表示与这个uid关联的流量，有时候用户需要在自己的uid内添加一个其他
# tag表示这个模块中的子模块的流量，就可以通过setThreadTag
# logger.debug(" tart_list: " + str(tart_list))
self.rx_uid_bytes += int(
tart_list[5]) #不区分网络的类型，直接算总和,wifi和mobile, lo数据的总和
# logger.debug(self.rx_uid_bytes)
self.rx_uid_packets += int(tart_list[6])
self.tx_uid_bytes += int(tart_list[7])
self.tx_uid_packets += int(tart_list[8])
self.total_uid_bytes = self.tx_uid_bytes + self.rx_uid_bytes
self.total_uid_packets = self.tx_uid_packets + self.rx_uid_packets
if (tart_list[1] == 'lo'): #对应着iface这列，表示本地流量
self.lo_uid_bytes += int(tart_list[5]) + int(
tart_list[7])
# logger.debug(" lo_uid_bytes: " + str(self.lo_uid_bytes))
if (int(tart_list[4]) == 0): #统计后台流量
self.bg_bytes += int(tart_list[5]) + int(tart_list[7])
# logger.debug(" backgroud data ： " +str(self.bg_bytes))
elif (int(tart_list[4]) == 1): #统计前台流量
self.fg_bytes += int(tart_list[5]) + int(tart_list[7])
# logger.debug(" fg data: " +str(self.fg_bytes))
logger.debug(" total uid bytes : " + str(self.total_uid_bytes))
def list_dir_between_time(self, dir_path, start_time, end_time):
'''列取目录下 起止时间点之间的文件
start_time end_time 时间戳
返回文件绝对路径 列表
'''
# ls - l
# -rwxrwx--- 1 root root 19897899 2018-12-27 18:02 com.alibaba.ailabs.ar.fireeye2_dumpheap_2018_12_27_18_02_52.hprof
result = self.run_shell_cmd('ls -l %s' % dir_path)
if not result:
return ""
result = result.replace('\r\r\n', '\n')
if 'No such file or directory' in result:
logger.error('文件(夹) %s 不存在' % dir_path)
file_list = []
re_time = re.compile(r'\S*\s+(\d+-\d+-\d+\s+\d+:\d+)\s+\S+')
for line in result.split('\n'):
items = line.split()
match = re_time.search(line)
if match:
last_modify_time = match.group(1)
logger.debug(last_modify_time)
last_modify_timestamp = TimeUtils.getTimeStamp(last_modify_time, "%Y-%m-%d %H:%M")
# logger.debug(last_modify_timestamp)
if start_time < last_modify_timestamp and last_modify_timestamp < end_time:
logger.debug("append file:" + items[-1])
file_list.append('%s/%s' % (dir_path, items[-1]))
return file_list
def get_data_from_threadstart(self, traffic_snapshot):
# 获取从当前线程开始的流量值
traffic_snapshot.total_uid_bytes = traffic_snapshot.total_uid_bytes - self.traffic_init_dic[
'total'] if (traffic_snapshot.total_uid_bytes -
self.traffic_init_dic['total']) >= 0 else 0
traffic_snapshot.total_uid_packets = traffic_snapshot.total_uid_packets - self.traffic_init_dic[
'total_packets'] if (
traffic_snapshot.total_uid_packets -
self.traffic_init_dic['total_packets']) >= 0 else 0
traffic_snapshot.rx_uid_bytes = traffic_snapshot.rx_uid_bytes - self.traffic_init_dic[
'rx'] if (traffic_snapshot.rx_uid_bytes -
self.traffic_init_dic['rx']) >= 0 else 0
traffic_snapshot.rx_uid_packets = traffic_snapshot.rx_uid_packets - self.traffic_init_dic[
'rx_packets'] if (traffic_snapshot.rx_uid_packets -
self.traffic_init_dic['rx_packets']) >= 0 else 0
traffic_snapshot.tx_uid_bytes = traffic_snapshot.tx_uid_bytes - self.traffic_init_dic[
'tx'] if (traffic_snapshot.tx_uid_bytes -
self.traffic_init_dic['tx']) >= 0 else 0
traffic_snapshot.tx_uid_packets = traffic_snapshot.tx_uid_packets - self.traffic_init_dic[
'tx_packets'] if (traffic_snapshot.tx_uid_packets -
self.traffic_init_dic['tx_packets']) >= 0 else 0
traffic_snapshot.fg_bytes = traffic_snapshot.fg_bytes - self.traffic_init_dic[
'fg'] if (traffic_snapshot.fg_bytes -
self.traffic_init_dic['fg']) >= 0 else 0
traffic_snapshot.bg_bytes = traffic_snapshot.bg_bytes - self.traffic_init_dic[
'bg'] if (traffic_snapshot.bg_bytes -
self.traffic_init_dic['bg']) >= 0 else 0
traffic_snapshot.lo_uid_bytes = traffic_snapshot.lo_uid_bytes - self.traffic_init_dic[
'lo'] if (traffic_snapshot.lo_uid_bytes -
self.traffic_init_dic['lo']) >= 0 else 0
logger.debug(traffic_snapshot)
return traffic_snapshot
def get_focus_activity(self):
'''
通过dumpsys window windows获取activity名称 window名?
'''
activity_name = ''
activity_line = ''
activity_line_split = ''
dumpsys_result = self.run_shell_cmd('dumpsys window windows')
dumpsys_result_list = dumpsys_result.split('\n')
for line in dumpsys_result_list:
if line.find('mCurrentFocus') != -1:
activity_line = line.strip()
# Android
# Android 8.0 mCurrentFocus的输出行
# mCurrentFocus=Window{2f4cb8b u0 com.google.android.apps.photos/com.google.android.apps.photos.home.HomeActivity}
if activity_line:
activity_line_split = activity_line.split(' ')
else:
return activity_name
logger.debug('dumpsys window windows命令activity_line_split结果: %s' % activity_line_split)
if len(activity_line_split) > 1:
if activity_line_split[1] == 'u0':
activity_name = activity_line_split[2].rstrip('}')
else:
activity_name = activity_line_split[1]
return activity_name
def _dumpsys_process_meminfo(self, process):
'''
dump 进程详细内存 耗时 1s以内
:param process:
:return:
'''
time_old = time.time()
out = self.device.adb.run_shell_cmd('dumpsys meminfo %s' % process)
# self.num = self.num + 1
# if self.num % 10 == 0:
#避免：在windows 无法创建文件名，不能有冒号:
process_rename = process.replace(":", "_")
meminfo_file = os.path.join(RuntimeData.package_save_path,
'dumpsys_meminfo_%s.txt' % process_rename)
with open(meminfo_file, "a+") as writer:
writer.write(TimeUtils.getCurrentTime() +
" dumpsys meminfo package info:\n")
if out:
writer.write(out + "\n\n")
# self.num = 0
passedtime = time.time() - time_old #测试meminfo这个命令的耗时，执行的时长在400多ms
logger.debug("dumpsys meminfo package time consume:" + str(passedtime))
out.replace('\r', '')
return MemInfoPackage(dump=out)
def _parse(self):
sp_lines = self.source.split('\n')
for line in sp_lines:
# wlan0: 1241508864 840739 0 0 0 0 0 7 7149177 73416 0 6 0 0 0 0
# 获取其中 接受流量1241508864 发送流量7149177
if "wlan0:" in line:
items = line.split()
self.wifi_rx = int(items[1])
self.wifi_tx = int(items[9])
self.wifi_total = self.wifi_rx + self.wifi_tx
logger.debug("wifi_rx : " + items[1] + " wifi_tx : " +
items[9] + " total wifi:" + str(self.wifi_total))
# 移动 3 4 5G 流量
# rmnet0: 362133448 298441 0 0 0 0 0 0 10641124 91012 0 0 0 0 0 0
if "rmnet0:" in line:
items = line.split()
self.mobile_rx = int(items[1])
self.mobile_tx = int(items[9])
self.mobile_total = self.wifi_rx + self.wifi_tx
logger.debug("mobile_rx : " + items[1] + " mobile_tx : " +
items[9] + " total mobile:" +
str(self.mobile_total))
self.rx = self.wifi_rx + self.mobile_rx
self.tx = self.wifi_tx + self.mobile_tx
self.total = self.wifi_total + self.mobile_total
def start(self, start_time):
'''
启动一个cpu监控器，监控cpu信息
:return:
'''
self.cpu_collector.start(start_time)
logger.debug("INFO: CpuMonitor has started...")
def _get_activity_save(self, activity_data, timestamp):
if self.first_time:
activity_title = ("datetime", "current_activity")
self.first_time = False
self.activity_file = os.path.join(RuntimeData.package_save_path,
'current_activity.csv')
try:
with open(self.activity_file, 'a+') as af:
csv.writer(af,
lineterminator='\n').writerow(activity_title)
except Exception as e:
logger.error("file not found: " + str(self.activity_file))
else:
try:
activity_data[0] = timestamp
dic = {
"time": activity_data[0] * 1000,
"name": activity_data[1]
}
self.perf_data['activity'].append(dic)
with open(self.activity_file, 'a+') as writer:
if isinstance(activity_data[0], float):
activity_data[0] = TimeUtils.formatTimeStamp(
activity_data[0])
tmp_dic = copy.deepcopy(dic)
tmp_dic["time"] = activity_data[0]
logger.debug(tmp_dic)
writer_p = csv.writer(writer, lineterminator='\n')
writer_p.writerow(activity_data)
except Exception as e:
logger.error("activity save error ")
s = traceback.format_exc()
logger.debug(s)
def start(self,start_time):
if not RuntimeData.package_save_path:
RuntimeData.package_save_path = os.path.join(os.path.abspath(os.path.join(os.getcwd(), "../..")),'results',self.device.adb._device_id,start_time)
if not os.path.exists(RuntimeData.package_save_path):
os.makedirs(RuntimeData.package_save_path)
self.start_time = start_time
self.power_collector.start(start_time)
logger.debug("INFO: PowerMonitor has started...")
def start(self,start_time):
'''
启动一个搜集器来启动一个新的线程搜集cpu信息
:return:
'''
self.collect_device_cpu_thread = threading.Thread(target=self._collect_cpu_thread, args=(start_time,))
self.collect_device_cpu_thread.start()
logger.debug("INFO: CpuCollector started...")
def stop(self):
logger.debug("INFO: TrafficCollecor stop...")
if (self.collect_traffic_thread.isAlive()):
self._stop_event.set()
self.collect_traffic_thread.join(timeout=1)
self.collect_traffic_thread = None
if self.traffic_queue:
self.traffic_queue.task_done()
def stop_logcat(self):
'''停止logcat进程
'''
self._logcat_running = False
logger.debug("stop logcat")
if hasattr(self, '_log_pipe'):
if self._log_pipe.poll() == None: # 判断logcat进程是否存在
self._log_pipe.terminate()
def recover():
if ADB.checkAdbNormal():
logger.debug("adb is normal")
return
else:
logger.error("adb is not normal")
ADB.kill_server()
ADB.start_server()
def filter_file_names(self, device):
csv_files = []
logger.debug(device)
for f in os.listdir(device):
if os.path.isfile(os.path.join(device, f)) and os.path.basename(
f) in self.summary_csf_file.keys():
logger.debug(os.path.join(device, f))
csv_files.append(f)
return csv_files
def _cat_current(self):
current = 0
# cat /sys/class/power_supply/Battery/current_now Android9 上没权限
reg = self.device.adb.run_shell_cmd('cat /sys/class/power_supply/battery/current_now')
if isinstance(reg, str) and "No such file or directory"==reg:
logger.debug("can't get current from file /sys/class/power_supply/battery/current_now")
elif reg:
current = reg
return current
def stop(self):
logger.debug("INFO: ThreadNumPackageCollector stop... ")
if (self.collect_thread_num_thread.isAlive()):
self._stop_event.set()
self.collect_thread_num_thread.join(timeout=1)
self.collect_thread_num_thread = None
#结束的时候，发送一个任务完成的信号，以结束队列
if self.thread_queue:
self.thread_queue.task_done()
def _monkey_thread_func(self,save_dir):
'''获取monkey线程，保存monkey日志，monkey Crash日志暂不处理，后续有需要再处理
'''
self.append_log_line_num = 0
self.file_log_line_num = 0
self.log_file_create_time = None
log_is_none = 0
logs = []
logger.debug("monkey_thread_func")
if RuntimeData.start_time is None:
RuntimeData.start_time = TimeUtils.getCurrentTime()
while self.running:
try:
log = self._log_pipe.stdout.readline().strip()
if not isinstance(log, str):
# 先编码为unicode
try:
log = str(log, "utf8")
except Exception as e:
log = repr(log)
logger.error('str error:' + log)
logger.error(e)
if log:
logs.append(log)
self.append_log_line_num = self.append_log_line_num + 1
self.file_log_line_num = self.file_log_line_num + 1
# if self.append_log_line_num > 1000:
if self.append_log_line_num > 100:
if not self.log_file_create_time:
self.log_file_create_time = TimeUtils.getCurrentTimeUnderline()
log_file = os.path.join(save_dir,
'monkey_%s.log' % self.log_file_create_time)
self.append_log_line_num = 0
# 降低音量，避免音量过大，导致语音指令失败
self.device.adb.run_shell_cmd("input keyevent 25")
self.save(log_file, logs)
logs = []
# 新建文件
if self.file_log_line_num > 600000:
# if self.file_log_line_num > 200:
self.file_log_line_num = 0
self.log_file_create_time = TimeUtils.getCurrentTimeUnderline()
log_file = os.path.join(save_dir, 'monkey_%s.log' % self.log_file_create_time)
self.save(log_file, logs)
logs = []
else:
log_is_none = log_is_none + 1
if log_is_none % 1000 == 0:
logger.info("log is none")
if not self.device.adb.is_process_running("com.android.commands.monkey") and self.running:
self.device.adb.kill_process("com.android.commands.monkey")
self._log_pipe = self.device.adb.run_shell_cmd(self.monkey_cmd, sync=False)
except:
logger.error("an exception hanpend in monkey thread, reason unkown!")
s = traceback.format_exc()
logger.debug(s)
def start(self, start_time):
if not RuntimeData.package_save_path:
RuntimeData.package_save_path = os.path.join(
os.path.abspath(os.path.join(os.getcwd(), "../..")), 'results',
self.package, start_time)
if not os.path.exists(RuntimeData.package_save_path):
os.makedirs(RuntimeData.package_save_path)
self.start_time = start_time
self.traffic_colloctor.start(start_time)
logger.debug("INFO: TrafficMonitor has started...")
def stop(self):
'''结束logcat日志监控器
'''
logger.debug("logcat monitor: stop...")
self.remove_log_handle(self.launchtime.handle_launchtime) # 删除回调
logger.debug("logcat monitor: stopped")
if self.exception_log_list:
self.remove_log_handle(self.handle_exception)
self.device.adb.stop_logcat()
self.running = False
def get_pckinfo_from_ps(self, packagename):
'''
从ps中获取应用的信息:pid,uid,packagename
:param packagename: 目标包名
:return: 返回目标包名的列表信息
'''
# get list_process 有重复代码
results = self.run_shell_cmd('ps -A')
lines = results.replace('\r', '').splitlines()
# logger.debug(lines)
busybox = False
if lines[0].startswith('PID'): busybox = True
result_list = []
for line in lines:
# logger.debug(line)
if packagename in line:
items = line.split()
if not busybox:
if len(items) < 9:
err_msg = "ps命令返回格式错误：\n%s" % line
if len(items) == 8:
target_pkg = items[7]
# 只保存与packagename完全相同的进程，对于以包名开头的其他进程将不会被保存，例如packagename:com.xx，com.xx:play不会保存
if packagename == target_pkg:
result_list.append(
{'pid': int(items[1]), 'uid': items[0], 'proc_name': items[-1],
'status': items[-2]})
else:
logger.error(err_msg)
else:
target_pck = items[-1]
# 只保存与packagename完全相同的进程，对于以包名开头的其他进程将不会被保存，例如packagename:com.xx，com.xx:play不会保存
if packagename == target_pck:
result_list.append(
{'pid': int(items[1]), 'uid': items[0], 'proc_name': items[-1], 'status': items[-2]})
else: # busybox = True
idx = 4
cmd = items[idx]
if len(cmd) == 1:
# 有时候发现此处会有“N”
idx += 1
cmd = items[idx]
idx += 1
if cmd[0] == '{' and cmd[-1] == '}': cmd = items[idx]
target_pkg = items[-1]
if packagename == target_pkg:
result_list.append(
{'pid': int(items[0]), 'uid': items[1], 'proc_name': cmd, 'status': items[-2]})
if len(result_list) > 0:
logger.debug(" get pckinfo from ps: " + str(result_list))
return result_list
def parse_data_from_config(self):
'''
从配置文件中解析出需要的信息，包名，时间间隔，设备的序列号等
:return:配置文件中读出来的数值的字典
'''
config_dic = {}
configpath = os.path.join(RuntimeData.top_dir, "config.conf")
logger.debug("configpath:%s" % configpath)
if not os.path.isfile(configpath):
logger.error("the config file didn't exist: " + configpath)
raise RuntimeError("the config file didn't exist: " + configpath)
# 避免windows会用系统默认的gbk打开
with open(configpath, encoding="utf-8") as f:
content = f.read()
# Window下用记事本打开配置文件并修改保存后，编码为UNICODE或UTF-8的文件的文件头
# 会被相应的加上\xff\xfe（\xff\xfe）或\xef\xbb\xbf，然后再传递给ConfigParser解析的时候会出错
# ，因此解析之前，先替换掉
content = re.sub(r"\xfe\xff", "", content)
content = re.sub(r"\xff\xfe", "", content)
content = re.sub(r"\xef\xbb\xbf", "", content)
open(configpath, 'w', encoding="utf-8").write(content)
paser = ConfigParser()
paser.read(configpath, encoding="utf-8")
config_dic = self.check_config_option(config_dic, paser, "Common",
"package")
config_dic = self.check_config_option(config_dic, paser, "Common",
"pid_change_focus_package")
config_dic = self.check_config_option(config_dic, paser, "Common",
"frequency")
config_dic = self.check_config_option(config_dic, paser, "Common",
"dumpheap_freq")
config_dic = self.check_config_option(config_dic, paser, "Common",
"timeout")
config_dic = self.check_config_option(config_dic, paser, "Common",
"serialnum")
config_dic = self.check_config_option(config_dic, paser, "Common",
"mailbox")
config_dic = self.check_config_option(config_dic, paser, "Common",
"exceptionlog")
config_dic = self.check_config_option(config_dic, paser, "Common",
"save_path")
config_dic = self.check_config_option(config_dic, paser, "Common",
"phone_log_path")
# 读取monkey配置
config_dic = self.check_config_option(config_dic, paser, "Common",
"monkey")
config_dic = self.check_config_option(config_dic, paser, "Common",
"main_activity")
config_dic = self.check_config_option(config_dic, paser, "Common",
"activity_list")
logger.debug(config_dic)
return config_dic
def get_process_thread_num(self, process):
pid = self.device.adb.get_pid_from_pck(self.packagename)
out = self.device.adb.run_shell_cmd('ls -lt /proc/%s/task' % pid)
collection_time = time.time()
logger.debug("collection time in thread_num info is : " +
str(collection_time))
if out:
# logger.debug("thread num out:"+out)
thread_num = len(out.split("\n"))
return [collection_time, self.packagename, pid, thread_num]
else:
return []
