def _monkey_thread_func(self,save_dir):
'''获取monkey线程，保存monkey日志，monkey Crash日志暂不处理，后续有需要再处理
'''
self.append_log_line_num = 0
self.file_log_line_num = 0
self.log_file_create_time = None
log_is_none = 0
logs = []
logger.debug("monkey_thread_func")
if RuntimeData.start_time is None:
RuntimeData.start_time = TimeUtils.getCurrentTime()
while self.running:
try:
log = self._log_pipe.stdout.readline().strip()
if not isinstance(log, str):
# 先编码为unicode
try:
log = str(log, "utf8")
except Exception as e:
log = repr(log)
logger.error('str error:' + log)
logger.error(e)
if log:
logs.append(log)
self.append_log_line_num = self.append_log_line_num + 1
self.file_log_line_num = self.file_log_line_num + 1
# if self.append_log_line_num > 1000:
if self.append_log_line_num > 100:
if not self.log_file_create_time:
self.log_file_create_time = TimeUtils.getCurrentTimeUnderline()
log_file = os.path.join(save_dir,
'monkey_%s.log' % self.log_file_create_time)
self.append_log_line_num = 0
# 降低音量，避免音量过大，导致语音指令失败
self.device.adb.run_shell_cmd("input keyevent 25")
self.save(log_file, logs)
logs = []
# 新建文件
if self.file_log_line_num > 600000:
# if self.file_log_line_num > 200:
self.file_log_line_num = 0
self.log_file_create_time = TimeUtils.getCurrentTimeUnderline()
log_file = os.path.join(save_dir, 'monkey_%s.log' % self.log_file_create_time)
self.save(log_file, logs)
logs = []
else:
log_is_none = log_is_none + 1
if log_is_none % 1000 == 0:
logger.info("log is none")
if not self.device.adb.is_process_running("com.android.commands.monkey") and self.running:
self.device.adb.kill_process("com.android.commands.monkey")
self._log_pipe = self.device.adb.run_shell_cmd(self.monkey_cmd, sync=False)
except:
logger.error("an exception hanpend in monkey thread, reason unkown!")
s = traceback.format_exc()
logger.debug(s)
def __init__(self, csv_dir, packages=[]):
os.chdir(csv_dir)
# 需要画曲线的csv文件名
self.summary_csf_file = {
"cpuinfo.csv": {
"table_name": "pid_cpu",
"x_axis": "datatime",
"y_axis": "%",
"values": ["pid_cpu%", "total_pid_cpu%"]
},
"meminfo.csv": {
"table_name": "pid_pss",
"x_axis": "datatime",
"y_axis": "mem(MB)",
"values": ["pid_pss(MB)", "total_pss(MB)"]
},
"pid_change.csv": {
"table_name": "pid",
"x_axis": "datatime",
"y_axis": "pid_num",
"values": ["pid"]
},
}
self.packages = packages
if len(self.packages) > 0:
for package in self.packages:
pss_detail_dic = {
"table_name": "pss_detail",
"x_axis": "datatime",
"y_axis": "mem(MB)",
"values": ["pss", "java_heap", "native_heap", "system"]
}
# 文件名太长会导致写excel失败
self.summary_csf_file["pss_%s.csv" %
package.split(".")[-1].replace(
":", "_")] = pss_detail_dic
logger.debug(self.packages)
logger.debug(self.summary_csf_file)
logger.info('create report for %s' % csv_dir)
file_names = self.filter_file_names(csv_dir)
logger.debug('%s' % file_names)
if file_names:
book_name = 'summary_%s.xlsx' % TimeUtils.getCurrentTimeUnderline()
excel = Excel(book_name)
for file_name in file_names:
logger.debug('get csv %s to excel' % file_name)
values = self.summary_csf_file[file_name]
excel.csv_to_xlsx(file_name, values["table_name"],
values["x_axis"], values["y_axis"],
values["values"])
logger.info('wait to save %s' % book_name)
excel.save()
def run(self, time_out=None):
self.clear_heapdump()
# objgraph.show_growth()
# 对设备连接情况的检查
if not self.serialnum:
# androiddevice 没传 serialnum，默认执行adb shell
logger.info(
"serialnum in config file is null,default get connected phone")
is_device_connect = False
for i in range(0, 5):
if self.device.adb.is_connected(self.serialnum):
is_device_connect = True
break
else:
logger.error("device not found:" + self.serialnum)
time.sleep(2)
if not is_device_connect:
logger.error("after 5 times check,device not found:" +
self.serialnum)
return
# 对是否安装被测app的检查 只在最开始检查一次
if not self.device.adb.is_app_installed(self.packages[0]):
logger.error("test app not installed:" + self.packages[0])
return
try:
#初始化数据处理的类,将没有消息队列传递过去，以便获取数据，并处理
# datahandle = DataWorker(self.get_queue_dic())
# 将queue传进去，与datahandle那个线程交互
self.add_monitor(
CpuMonitor(self.serialnum, self.packages, self.frequency,
self.timeout))
self.add_monitor(
MemMonitor(self.serialnum, self.packages, self.frequency,
self.timeout))
self.add_monitor(
TrafficMonitor(self.serialnum, self.packages[0],
self.frequency, self.timeout))
# 软件方式 获取电量不准，已用硬件方案测试功耗
# self.add_monitor(PowerMonitor(self.serialnum, self.frequency,self.timeout))
self.add_monitor(
FPSMonitor(self.serialnum, self.packages[0], self.frequency,
self.timeout))
# 6.0以下能采集到fd数据，7.0以上没权限
if self.device.adb.get_sdk_version() <= 23:
self.add_monitor(
FdMonitor(self.serialnum, self.packages[0], self.frequency,
self.timeout))
self.add_monitor(
ThreadNumMonitor(self.serialnum, self.packages[0],
self.frequency, self.timeout))
if self.config_dic["monkey"] == "true":
self.add_monitor(Monkey(self.serialnum, self.packages[0]))
if self.config_dic["main_activity"] and self.config_dic[
"activity_list"]:
self.add_monitor(
DeviceMonitor(self.serialnum, self.packages[0],
self.frequency,
self.config_dic["main_activity"],
self.config_dic["activity_list"],
RuntimeData.exit_event))
if len(self.monitors):
start_time = TimeUtils.getCurrentTimeUnderline()
RuntimeData.start_time = start_time
if self.config_dic["save_path"]:
RuntimeData.package_save_path = os.path.join(
self.config_dic["save_path"], self.packages[0],
start_time)
else:
RuntimeData.package_save_path = os.path.join(
RuntimeData.top_dir, 'results', self.packages[0],
start_time)
FileUtils.makedir(RuntimeData.package_save_path)
self.save_device_info()
for monitor in self.monitors:
#启动所有的monitors
try:
monitor.start(start_time)
except Exception as e:
logger.error(e)
# logcat的代码可能会引起死锁，拎出来单独处理logcat
try:
self.logcat_monitor = LogcatMonitor(
self.serialnum, self.packages[0])
# 如果有异常日志标志，才启动这个模块
if self.exceptionlog_list:
self.logcat_monitor.set_exception_list(
self.exceptionlog_list)
self.logcat_monitor.add_log_handle(
self.logcat_monitor.handle_exception)
time.sleep(1)
self.logcat_monitor.start(start_time)
except Exception as e:
logger.error(e)
timeout = time_out if time_out != None else self.config_dic[
'timeout']
endtime = time.time() + timeout
while (time.time() < endtime): #吊着主线程防止线程中断
# 时间到或测试过程中检测到异常
if self.check_exit_signal_quit():
logger.error("app " + str(self.packages[0]) +
" exit signal, quit!")
break
time.sleep(self.frequency)
logger.debug("time is up,finish!!!")
self.stop()
# try:
# datahandle.stop()
# time.sleep(self.frequency*2)
# # 延迟一点时间结束上报，已让数据上报完
# # report.stop()
# except:
# logger.debug("report or datahandle stop exception")
# finally:
# logger.info("time is up, end")
# os._exit(0)
except KeyboardInterrupt: #捕获键盘异常的事件，例如ctrl c
logger.debug(" catch keyboardInterrupt, goodbye!!!")
# 收尾工作
self.stop()
os._exit(0)
except Exception as e:
logger.error("Exception in run")
logger.error(e)
def _logcat_thread_func(self, save_dir, process_list, params=""):
'''获取logcat线程
'''
self.append_log_line_num = 0
self.file_log_line_num = 0
self.log_file_create_time = None
logs = []
logger.debug("logcat_thread_func")
log_is_none = 0
while self._logcat_running:
try:
log = self._log_pipe.stdout.readline().strip()
if not isinstance(log, str):
try:
log = str(log,"utf8")
except Exception as e:
log = repr(log)
logger.error('str error:'+log)
logger.error(e)
if log:
log_is_none = 0
# logger.debug(log)
logs.append(log)
# if self._log_pipe.poll() != None:
# logger.debug('process:%s have exited' % self._log_pipe.pid)
# if self._logcat_running :
# self._log_pipe = self.run_shell_cmd('logcat ' + params, sync=False)
# else :
# break
for _handle in self._logcat_handle:
try:
_handle(log)
except Exception as e:
logger.error("an exception happen in logcat handle log , reason unkown!, e:")
logger.error(e)
self.append_log_line_num = self.append_log_line_num + 1
self.file_log_line_num = self.file_log_line_num + 1
# if self.append_log_line_num > 1000:
if self.append_log_line_num > 100:
if not self.log_file_create_time:
self.log_file_create_time = TimeUtils.getCurrentTimeUnderline()
logcat_file = os.path.join(save_dir,
'logcat_%s.log' % self.log_file_create_time)
self.append_log_line_num = 0
self.save(logcat_file, logs)
logs = []
# 新建文件
if self.file_log_line_num > 600000:
# if self.file_log_line_num > 200:
self.file_log_line_num = 0
self.log_file_create_time = TimeUtils.getCurrentTimeUnderline()
logcat_file = os.path.join(save_dir, 'logcat_%s.log' % self.log_file_create_time)
self.save(logcat_file, logs)
logs = []
else:
log_is_none = log_is_none + 1
if log_is_none % 1000 == 0:
logger.info("log is none")
self._log_pipe = self.run_shell_cmd('logcat -v threadtime ' + params, sync=False)
except:
logger.error("an exception hanpend in logcat thread, reason unkown!")
s = traceback.format_exc()
logger.debug(s)
def _run_cmd_once(self, cmd, *argv, **kwds):
'''执行一次adb命令：cmd
:param str cmd: 命令字符串
:param list argv: 可变参数
:param dict kwds: 可选关键字参数 (超时/异步)
:return: 执行adb命令的子进程或执行的结果
:rtype: Popen or str
'''
import locale
import codecs
if self._device_id:
cmdlet = [self._adb_path, '-s', self._device_id, cmd]
else:
cmdlet = [self._adb_path, cmd]
for i in range(len(argv)):
arg = argv[i]
if not isinstance(argv[i], str):
arg = arg.decode('utf8')
cmdlet.append(arg)
# logger.debug('ADB cmd:' + " ".join(cmdlet))
# logger.debug(cmdlet)
cmdStr = " ".join(cmdlet)
logger.debug(cmdStr)
process = None
# windows上 不要传cmdStr 目录有空格，会报错
# if ADB.os_name == "Windows":
# process = subprocess.Popen(cmdlet, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE,shell=True)
# # mac上传list会报错:Android Debug Bridge version
# else:
# windows ["adb devices"] 提示没有命令 ，改为str执行
process = subprocess.Popen(cmdStr, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
shell=True)
if "sync" in kwds and kwds['sync'] == False:
# 异步执行命令，不等待结果，返回该子进程对象
return process
before = time.time()
timeout = 10
if "timeout" in kwds:
timeout = kwds['timeout']
if timeout != None and timeout > 0:
# timeout = None 或者小于等于0时，一直等待执行结果
threading.Thread(None, self._timer, (process, timeout))
(out, error) = process.communicate()
# 执行错误 mac out无输出 error有输出 返回值非0
# 执行错误 windows out有输出 error没有输出，返回值0
if process.poll() != 0: # 返回码为非0，表示命令未执行成功返回
# logger.logError("adb执行出错或超时，出错命令是：\n%s"%cmdlet)
if error and len(error) != 0:
logger.debug("adb error info:\n%s" % error)
if "no devices/emulators found" in str(out) or "no devices/emulators found" in str(error):
logger.error("no devices/emulators found,please reconnect phone,make sure adb shell normal")
return ""
# 退出整个进程
if "killing" in str(out) or "killing" in str(error):
logger.error(
"adb 5037 port is occupied,please stop the process occupied 5037 port,make sure adb devices normal")
return ""
if "device not found" in str(out) or "device not found" in str(error):
logger.error("device not found,please reconnect phone,make sure adb devices normal")
self.before_connect = False
self.after_connect = False
return ""
if "offline" in str(out) or "offline" in str(error):
logger.error("device offline,please reconnect phone,make sure adb devices normal")
return ""
if "more than one" in str(out) or "more than one" in str(error):
logger.error("more than one device,please input device serialnum!")
# sys.exit(1)
if "Android Debug Bridge version" in str(out) or "Android Debug Bridge version" in str(error):
logger.error("adb cmd error!:" + out)
# sys.exit(1)
if str(out, "utf-8") == '':
# logger.debug("out is empty ,use error")
out = error
self.after_connect = True
after = time.time()
time_consume = after - before
logger.info(cmdStr + " time consume: " + str(time_consume))
if not isinstance(out, str):
try:
out = str(out, "utf8")
except Exception as e:
out = repr(out)
return out.strip()
def _collect_memory_thread(self, start_time):
end_time = time.time() + self._timeout
mem_list_titile = ["datatime", "total_ram(MB)", "free_ram(MB)"]
pid_list_titile = ["datatime"]
pss_detail_titile = [
"datatime", "package", "pid", "pss", "java_heap", "native_heap",
"system"
]
for i in range(0, len(self.packages)):
mem_list_titile.extend(["package", "pid", "pid_pss(MB)"])
pid_list_titile.extend(["package", "pid"])
if len(self.packages) > 1:
mem_list_titile.append("total_pss(MB)")
mem_file = os.path.join(RuntimeData.package_save_path, 'meminfo.csv')
pid_file = os.path.join(RuntimeData.package_save_path,
'pid_change.csv')
for package in self.packages:
pss_detail_file = os.path.join(
RuntimeData.package_save_path,
'pss_%s.csv' % package.split(".")[-1].replace(":", "_"))
with open(pss_detail_file, 'a+') as df:
csv.writer(df, lineterminator='\n').writerow(pss_detail_titile)
try:
with open(mem_file, 'a+') as df:
csv.writer(df, lineterminator='\n').writerow(mem_list_titile)
if self.mem_queue:
mem_file_dic = {'mem_file': mem_file}
self.mem_queue.put(mem_file_dic)
with open(pid_file, 'a+') as df:
csv.writer(df, lineterminator='\n').writerow(pid_list_titile)
except RuntimeError as e:
logger.error(e)
starttime_stamp = TimeUtils.getTimeStamp(start_time,
"%Y_%m_%d_%H_%M_%S")
old_package_pid_pss_list = []
dumpsys_mem_times = 0
# D系统上会报错 System server has no access to file context
# hprof_path = "/sdcard/hprof"
hprof_path = "/data/local/tmp"
self.device.adb.run_shell_cmd("mkdir " + hprof_path)
# sdcard 卡目录下dump需要打开这个开关
self.device.adb.run_shell_cmd("setenforce 0")
first_dump = True
while not self._stop_event.is_set() and time.time() < end_time:
try:
before = time.time()
logger.debug(
"-----------into _collect_mem_thread loop, thread is : " +
str(threading.current_thread().name))
collection_time = time.time()
# # 获取主进程的详细信息
for package in self.packages:
mem_pck_snapshot = self._dumpsys_process_meminfo(package)
if 0 == mem_pck_snapshot.totalPSS:
logger.error("package total pss is 0:%s" % package)
continue
pss_detail_file = os.path.join(
RuntimeData.package_save_path, 'pss_%s.csv' %
package.split(".")[-1].replace(":", "_"))
pss_detail_list = [
TimeUtils.formatTimeStamp(collection_time), package,
mem_pck_snapshot.pid, mem_pck_snapshot.totalPSS,
mem_pck_snapshot.javaHeap, mem_pck_snapshot.nativeHeap,
mem_pck_snapshot.system
]
with open(pss_detail_file, 'a+') as pss_writer:
writer_p = csv.writer(pss_writer, lineterminator='\n')
writer_p.writerow(pss_detail_list)
# 写到pss_detail表格中
# 手机每5分钟 dumpheap一次
if (before - starttime_stamp) > 300 or first_dump:
# 先清理hprof文件
filelist = self.device.adb.list_dir(hprof_path)
if filelist:
for file in filelist:
for package in self.packages:
if package in file:
self.device.adb.delete_file(hprof_path +
"/" + file)
# if (before - starttime_stamp) % 60 < self._interval and "D" in self.device.adb.get_system_version():
for package in self.packages:
self.device.adb.dumpheap(package,
RuntimeData.package_save_path)
starttime_stamp = before
# self.device.adb.run_shell_cmd("kill -10 %s"%str(mem_pck_snapshot.pid))
# dumpsys meminfo 耗时长，可能会导致system server cpu占用变高，降低采集频率
dumpsys_mem_times = dumpsys_mem_times + 1
# 10倍率frequency dumpsys meminfo一次
if dumpsys_mem_times % 10 == 0 or first_dump:
mem_device_snapshot = self._dumpsys_meminfo()
# 如果没有采集到dumpsys meminfo的信息，正常情况totalmem不可能为0
if mem_device_snapshot == None or not mem_device_snapshot.package_pid_pss_list or mem_device_snapshot.totalmem == 0:
logger.error("mem_device_snapshot is none")
# 如果获取不到结果，继续延长采集间隔
dumpsys_mem_times = dumpsys_mem_times - 1
continue
first_dump = False
logger.debug("current time: " +
TimeUtils.getCurrentTime() +
", processname: " + ",total pss:" +
str(mem_device_snapshot.total_pss))
logger.debug("collection time in meminfo is : " +
TimeUtils.getCurrentTime())
gather_list = [
TimeUtils.formatTimeStamp(collection_time),
mem_device_snapshot.totalmem,
mem_device_snapshot.freemem
]
pid_list = [TimeUtils.formatTimeStamp(collection_time)]
pid_change = False
for i in range(0, len(self.packages)):
if len(mem_device_snapshot.package_pid_pss_list
) == len(self.packages):
gather_list.extend([
mem_device_snapshot.package_pid_pss_list[i]
["package"],
mem_device_snapshot.package_pid_pss_list[i]
["pid"],
mem_device_snapshot.package_pid_pss_list[i]
["pss"]
])
if not old_package_pid_pss_list:
old_package_pid_pss_list = mem_device_snapshot.package_pid_pss_list
pid_change = True
else:
for i in range(0, len(self.packages)):
package = mem_device_snapshot.package_pid_pss_list[
i]["package"]
if mem_device_snapshot.package_pid_pss_list[i]["pid"] and \
old_package_pid_pss_list[i]["pid"]!=mem_device_snapshot.package_pid_pss_list[i]["pid"]:
pid_change = True
# 确保上次pid也有
if old_package_pid_pss_list[i]["pid"]:
if package and package in RuntimeData.config_dic[
"pid_change_focus_package"]:
# 确保有tombstones文件才提单
self.device.adb.pull_file(
"/data/vendor/tombstones",
RuntimeData.package_save_path)
if pid_change:
old_package_pid_pss_list = mem_device_snapshot.package_pid_pss_list
for i in range(0, len(self.packages)):
if len(old_package_pid_pss_list) == len(
self.packages):
pid_list.extend([
old_package_pid_pss_list[i]["package"],
old_package_pid_pss_list[i]["pid"]
])
try:
with open(pid_file, 'a+') as pid_writer:
writer_p = csv.writer(pid_writer,
lineterminator='\n')
writer_p.writerow(pid_list)
logger.debug("write to file:" + pid_file)
logger.debug(pid_list)
except RuntimeError as e:
logger.error(e)
if len(self.packages) > 1:
gather_list.append(mem_device_snapshot.total_pss)
if self.mem_queue:
gather_list[0] = collection_time
self.mem_queue.put(gather_list)
if not self.mem_queue: #为了本地单个文件运行
try:
with open(mem_file, 'a+') as mem_writer:
writer_p = csv.writer(mem_writer,
lineterminator='\n')
writer_p.writerow(gather_list)
logger.debug("write to file:" + mem_file)
logger.debug(gather_list)
except RuntimeError as e:
logger.error(e)
after = time.time()
time_consume = after - before
delta_inter = self._interval - time_consume
logger.info("time consume for meminfos: " + str(time_consume))
if delta_inter > 0:
time.sleep(delta_inter)
except:
logger.error(
"an exception hanpend in meminfo thread, reason unkown!")
s = traceback.format_exc()
logger.debug(s)
if self.mem_queue:
self.mem_queue.task_done()
logger.debug("stop event is set or timeout")
def _handle_data_thread(self):
'''
该方法在独立线程中运行，是一个消费者，负责处理所有的生产者搜集的数据
:return:
'''
time.sleep(1)
while not self._stop_event.is_set():
try:
# 消息队列会一直阻塞直到 取到数据为止，如果生产者的线程发生了异常，则需要生产者线程会往对应的queue发出一个task_done以结束队列
# power 数据处理
self.perf_data = {
"task_id": "",
"activity": [],
'launch_time': [],
'cpu': [],
"mem": [],
'traffic': [],
"fluency": [],
'power': [],
"fd": [],
"thread": []
}
try:
power_data = self.power_queue.get()
if isinstance(power_data, list):
self.timestamp = power_data[0]
logger.debug("dataworker logger timestamp: " +
str(self.timestamp))
logger.info("now collecting data")
self._get_power_save(power_data, self.timestamp)
except queue.Empty:
logger.debug("dataworker power queue Empty")
# traffic数据的处理
try:
traffic_data = self.traffic_queue.get(timeout=2)
# logger.debug(traffic_data)
self._get_traffic_save(traffic_data, self.timestamp)
except queue.Empty:
logger.debug("dataworker traffic queue Empty")
#fps数据的处理
try:
fps_data = self.fps_queue.get(timeout=2)
self._get_fps_save(fps_data, self.timestamp)
except queue.Empty:
logger.debug("dataworker fps queue Empty")
#cpu数据处理
try:
cpu_data = self.cpu_queue.get(timeout=2)
# logger.debug(cpu_data)
self._get_cpu_save(cpu_data, self.timestamp)
except queue.Empty:
logger.debug("dataworker cpu queue Empty")
#mem 数据处理
try:
mem_data = self.mem_queue.get(timeout=2)
# logger.debug(mem_data)
self._get_mem_save(mem_data, self.timestamp)
except queue.Empty:
logger.debug("dataworker mem queue Empty")
try:
fd_data = self.fd_queue.get(timeout=2)
# logger.debug("fd_data")
# logger.debug(fd_data)
self._get_fd_save(fd_data, self.timestamp)
except queue.Empty:
logger.debug("dataworker fd_data queue Empty")
try:
thread_data = self.thread_queue.get(timeout=2)
# logger.debug("thread_data")
# logger.debug(thread_data)
self._get_thread_save(thread_data, self.timestamp)
except queue.Empty:
logger.debug("dataworker thread queue Empty")
try:
activity_data = self.activity_queue.get(timeout=2)
self._get_activity_save(activity_data, self.timestamp)
except queue.Empty:
logger.debug("dataworker activity queue Empty")
except Exception as e:
logger.error(e)
s = traceback.format_exc()
logger.debug(s) # 将堆栈信息打印到log中
#logger.info("dataworker interval: " + str(self.interval))
# 封装后最后统一放入 perf_queue中
logger.debug("perf_str put in queue:" + json.dumps(self.perf_data))
# 没有启动上报线程，避免无限增长
# perf_queue.put(self.perf_data)
time.sleep(self.interval)
