def _monkey_thread_func(self,save_dir):
'''获取monkey线程，保存monkey日志，monkey Crash日志暂不处理，后续有需要再处理
'''
self.append_log_line_num = 0
self.file_log_line_num = 0
self.log_file_create_time = None
log_is_none = 0
logs = []
logger.debug("monkey_thread_func")
if RuntimeData.start_time is None:
RuntimeData.start_time = TimeUtils.getCurrentTime()
while self.running:
try:
log = self._log_pipe.stdout.readline().strip()
if not isinstance(log, str):
# 先编码为unicode
try:
log = str(log, "utf8")
except Exception as e:
log = repr(log)
logger.error('str error:' + log)
logger.error(e)
if log:
logs.append(log)
self.append_log_line_num = self.append_log_line_num + 1
self.file_log_line_num = self.file_log_line_num + 1
# if self.append_log_line_num > 1000:
if self.append_log_line_num > 100:
if not self.log_file_create_time:
self.log_file_create_time = TimeUtils.getCurrentTimeUnderline()
log_file = os.path.join(save_dir,
'monkey_%s.log' % self.log_file_create_time)
self.append_log_line_num = 0
# 降低音量，避免音量过大，导致语音指令失败
self.device.adb.run_shell_cmd("input keyevent 25")
self.save(log_file, logs)
logs = []
# 新建文件
if self.file_log_line_num > 600000:
# if self.file_log_line_num > 200:
self.file_log_line_num = 0
self.log_file_create_time = TimeUtils.getCurrentTimeUnderline()
log_file = os.path.join(save_dir, 'monkey_%s.log' % self.log_file_create_time)
self.save(log_file, logs)
logs = []
else:
log_is_none = log_is_none + 1
if log_is_none % 1000 == 0:
logger.info("log is none")
if not self.device.adb.is_process_running("com.android.commands.monkey") and self.running:
self.device.adb.kill_process("com.android.commands.monkey")
self._log_pipe = self.device.adb.run_shell_cmd(self.monkey_cmd, sync=False)
except:
logger.error("an exception hanpend in monkey thread, reason unkown!")
s = traceback.format_exc()
logger.debug(s)
def stop(self):
for monitor in self.monitors:
try:
monitor.stop()
except Exception as e: # 捕获所有的异常，防止其中一个monitor的stop操作发生异常退出时，影响其他的monitor的stop操作
logger.error(e)
try:
if self.logcat_monitor:
self.logcat_monitor.stop()
except Exception as e:
logger.error("stop exception for logcat monitor")
logger.error(e)
if self.config_dic["monkey"] == "true":
self.device.adb.kill_process("com.android.commands.monkey")
# 统计测试时长
cost_time = round((float)(time.time() - TimeUtils.getTimeStamp(
RuntimeData.start_time, TimeUtils.UnderLineFormatter)) / 3600, 2)
self.add_device_info("test cost time:", str(cost_time) + "h")
# 根据csv生成excel汇总文件
Report(RuntimeData.package_save_path, self.packages)
self.pull_heapdump()
self.pull_log_files()
# self.memory_analyse()
# self.device.adb.bugreport(RuntimeData.package_save_path)
os._exit(0)
def _collect_thread_num_thread(self, start_time):
end_time = time.time() + self._timeout
thread_list_titile = ("datatime", "packagename", "pid", "thread_num")
thread_num_file = os.path.join(RuntimeData.package_save_path,
'thread_num.csv')
try:
with open(thread_num_file, 'a+') as df:
csv.writer(df,
lineterminator='\n').writerow(thread_list_titile)
if self.thread_queue:
thread_file_dic = {'thread_file': thread_num_file}
self.thread_queue.put(thread_file_dic)
except RuntimeError as e:
logger.error(e)
while not self._stop_event.is_set() and time.time() < end_time:
try:
before = time.time()
logger.debug(
"-----------into _collect_thread_num_thread loop, thread is : "
+ str(threading.current_thread().name))
# 获取pakagename的thread num信息
thread_pck_info = self.get_process_thread_num(self.packagename)
logger.debug(thread_pck_info)
current_time = TimeUtils.getCurrentTime()
if not thread_pck_info:
continue
else:
logger.debug("current time: " + current_time +
", processname: " + thread_pck_info[1] +
", pid: " + str(thread_pck_info[2]) +
" thread num: " + str(thread_pck_info[3]))
if self.thread_queue:
self.thread_queue.put(thread_pck_info)
if not self.thread_queue: #为了本地单个文件运行
try:
with open(thread_num_file, 'a+') as thread_writer:
writer_p = csv.writer(thread_writer,
lineterminator='\n')
thread_pck_info[0] = current_time
writer_p.writerow(thread_pck_info)
except RuntimeError as e:
logger.error(e)
after = time.time()
time_consume = after - before
delta_inter = self._interval - time_consume
logger.debug("time_consume for thread num infos: " +
str(time_consume))
if delta_inter > 0:
time.sleep(delta_inter)
except:
logger.error(
"an exception hanpend in thread num thread, reason unkown!"
)
s = traceback.format_exc()
logger.debug(s)
if self.thread_queue:
self.thread_queue.task_done()
def _get_activity_save(self, activity_data, timestamp):
if self.first_time:
activity_title = ("datetime", "current_activity")
self.first_time = False
self.activity_file = os.path.join(RuntimeData.package_save_path,
'current_activity.csv')
try:
with open(self.activity_file, 'a+') as af:
csv.writer(af,
lineterminator='\n').writerow(activity_title)
except Exception as e:
logger.error("file not found: " + str(self.activity_file))
else:
try:
activity_data[0] = timestamp
dic = {
"time": activity_data[0] * 1000,
"name": activity_data[1]
}
self.perf_data['activity'].append(dic)
with open(self.activity_file, 'a+') as writer:
if isinstance(activity_data[0], float):
activity_data[0] = TimeUtils.formatTimeStamp(
activity_data[0])
tmp_dic = copy.deepcopy(dic)
tmp_dic["time"] = activity_data[0]
logger.debug(tmp_dic)
writer_p = csv.writer(writer, lineterminator='\n')
writer_p.writerow(activity_data)
except Exception as e:
logger.error("activity save error ")
s = traceback.format_exc()
logger.debug(s)
def list_dir_between_time(self, dir_path, start_time, end_time):
'''列取目录下 起止时间点之间的文件
start_time end_time 时间戳
返回文件绝对路径 列表
'''
# ls - l
# -rwxrwx--- 1 root root 19897899 2018-12-27 18:02 com.alibaba.ailabs.ar.fireeye2_dumpheap_2018_12_27_18_02_52.hprof
result = self.run_shell_cmd('ls -l %s' % dir_path)
if not result:
return ""
result = result.replace('\r\r\n', '\n')
if 'No such file or directory' in result:
logger.error('文件(夹) %s 不存在' % dir_path)
file_list = []
re_time = re.compile(r'\S*\s+(\d+-\d+-\d+\s+\d+:\d+)\s+\S+')
for line in result.split('\n'):
items = line.split()
match = re_time.search(line)
if match:
last_modify_time = match.group(1)
logger.debug(last_modify_time)
last_modify_timestamp = TimeUtils.getTimeStamp(last_modify_time, "%Y-%m-%d %H:%M")
# logger.debug(last_modify_timestamp)
if start_time < last_modify_timestamp and last_modify_timestamp < end_time:
logger.debug("append file:" + items[-1])
file_list.append('%s/%s' % (dir_path, items[-1]))
return file_list
def _get_fps_save(self, fps_data, timestamp):
if isinstance(fps_data, dict):
self.fps_filename = fps_data['fps_file']
logger.debug("fps_filename: " + str(self.fps_filename))
else:
try:
'''0 1 2 3
fps_data: ("datetime", "activity","fps", "jank")
对应的值是:[formatter(collection_time), activity,fps,jank],
'''
fps_data[0] = timestamp
dic = {
"time": fps_data[0] * 1000,
"activity": fps_data[1],
"fps": fps_data[2],
"jank": fps_data[3]
}
self.perf_data['fluency'].append(dic)
with open(self.fps_filename, 'a+') as writer:
logger.debug(
"dataworker write fps data in dataworker。fps timestamp: "
+ str(fps_data[0]))
fps_data[0] = TimeUtils.formatTimeStamp(fps_data[0])
tmp_dic = copy.deepcopy(dic)
tmp_dic["time"] = fps_data[0]
logger.debug(tmp_dic)
writer_p = csv.writer(writer, lineterminator='\n')
writer_p.writerow(fps_data)
except Exception as e:
s = traceback.format_exc()
logger.error(s) # 将堆栈信息打印到log中
logger.error("fps save error")
def _get_fd_save(self, fd_data, timestamp):
if isinstance(fd_data, dict):
self.fd_file = fd_data['fd_file']
logger.debug("fd_file: " + str(self.fd_file))
else:
try:
''' 0 1 2 3
fd_data: ("datatime", "pckagename", "pid", "fd num")
对应的值是：[formatTimeStamp(collection_time), packagename, pid,fd_num]
'''
fd_data[0] = timestamp
dic = {
"time": fd_data[0] * 1000,
"package": fd_data[1],
"pid": fd_data[2],
"fd": fd_data[3]
}
self.perf_data['fd'].append(dic)
with open(self.fd_file, 'a+') as writer:
logger.debug(
"write fd data in dataworker。。。。。。 fd timestamp: " +
str(fd_data[0]))
if isinstance(fd_data[0], float):
fd_data[0] = TimeUtils.formatTimeStamp(fd_data[0])
tmp_dic = copy.deepcopy(dic)
tmp_dic["time"] = fd_data[0]
logger.debug(tmp_dic)
writer_p = csv.writer(writer, lineterminator='\n')
writer_p.writerow(fd_data)
except Exception as e:
logger.error('fd save error')
s = traceback.format_exc()
logger.debug(s)
def _dumpsys_process_meminfo(self, process):
'''
dump 进程详细内存 耗时 1s以内
:param process:
:return:
'''
time_old = time.time()
out = self.device.adb.run_shell_cmd('dumpsys meminfo %s' % process)
# self.num = self.num + 1
# if self.num % 10 == 0:
#避免：在windows 无法创建文件名，不能有冒号:
process_rename = process.replace(":", "_")
meminfo_file = os.path.join(RuntimeData.package_save_path,
'dumpsys_meminfo_%s.txt' % process_rename)
with open(meminfo_file, "a+") as writer:
writer.write(TimeUtils.getCurrentTime() +
" dumpsys meminfo package info:\n")
if out:
writer.write(out + "\n\n")
# self.num = 0
passedtime = time.time() - time_old #测试meminfo这个命令的耗时，执行的时长在400多ms
logger.debug("dumpsys meminfo package time consume:" + str(passedtime))
out.replace('\r', '')
return MemInfoPackage(dump=out)
def get_max_freq(self):
out = self.device.adb.run_shell_cmd(
"cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_max_freq")
out.replace('\r', '')
max_freq_file = os.path.join(RuntimeData.package_save_path,
'scaling_max_freq.txt')
with open(max_freq_file, "a+", encoding="utf-8") as writer:
writer.write(TimeUtils.getCurrentTime() + " scaling_max_freq:\n")
writer.write(out + "\n\n")
def _collect_power_thread(self,start_time):
'''
搜集电池信息的线程
:return:
'''
end_time = time.time() + self._timeout
power_list_titile = ("datetime","level","voltage(V)","tempreture(C)","current(mA)")
power_device_file = os.path.join(RuntimeData.package_save_path, 'powerinfo.csv')
try:
with open(power_device_file, 'a+') as df:
csv.writer(df, lineterminator='\n').writerow(power_list_titile)
if self.power_queue:
power_file_dic = {'power_file':power_device_file}
self.power_queue.put(power_file_dic)
except RuntimeError as e:
logger.error(e)
while not self._stop_event.is_set() and time.time() < end_time:
try:
before = time.time()
logger.debug("------------into _collect_power_thread loop thread is : " + str(threading.current_thread().name))
device_power_info = self._get_battaryproperties()
if device_power_info.source == '':
logger.debug("can't get power info , break!")
break
device_power_info = self.trim_data(device_power_info)#debug
collection_time = time.time()
logger.debug(" collection time in powerconsumption is : " + str(collection_time))
power_tmp_list = [collection_time, device_power_info.level, device_power_info.voltage,
device_power_info.temp, device_power_info.current]
if self.power_queue:
self.power_queue.put(power_tmp_list)
if not self.power_queue:#为了本地单个脚本运行
power_tmp_list[0] = TimeUtils.formatTimeStamp(power_tmp_list[0])
try:
with open(power_device_file,'a+',encoding="utf-8") as writer:
writer_p = csv.writer(writer, lineterminator='\n')
writer_p.writerow(power_tmp_list)
except RuntimeError as e:
logger.error(e)
after = time.time()
time_consume = after - before
delta_inter = self._interval - time_consume
if delta_inter > 0:
time.sleep(delta_inter)
except:
logger.error("an exception hanpend in powerconsumption thread , reason unkown!")
s = traceback.format_exc()
logger.debug(s)
if self.power_queue:
self.power_queue.task_done()
def __init__(self, csv_dir, packages=[]):
os.chdir(csv_dir)
# 需要画曲线的csv文件名
self.summary_csf_file = {
"cpuinfo.csv": {
"table_name": "pid_cpu",
"x_axis": "datatime",
"y_axis": "%",
"values": ["pid_cpu%", "total_pid_cpu%"]
},
"meminfo.csv": {
"table_name": "pid_pss",
"x_axis": "datatime",
"y_axis": "mem(MB)",
"values": ["pid_pss(MB)", "total_pss(MB)"]
},
"pid_change.csv": {
"table_name": "pid",
"x_axis": "datatime",
"y_axis": "pid_num",
"values": ["pid"]
},
}
self.packages = packages
if len(self.packages) > 0:
for package in self.packages:
pss_detail_dic = {
"table_name": "pss_detail",
"x_axis": "datatime",
"y_axis": "mem(MB)",
"values": ["pss", "java_heap", "native_heap", "system"]
}
# 文件名太长会导致写excel失败
self.summary_csf_file["pss_%s.csv" %
package.split(".")[-1].replace(
":", "_")] = pss_detail_dic
logger.debug(self.packages)
logger.debug(self.summary_csf_file)
logger.info('create report for %s' % csv_dir)
file_names = self.filter_file_names(csv_dir)
logger.debug('%s' % file_names)
if file_names:
book_name = 'summary_%s.xlsx' % TimeUtils.getCurrentTimeUnderline()
excel = Excel(book_name)
for file_name in file_names:
logger.debug('get csv %s to excel' % file_name)
values = self.summary_csf_file[file_name]
excel.csv_to_xlsx(file_name, values["table_name"],
values["x_axis"], values["y_axis"],
values["values"])
logger.info('wait to save %s' % book_name)
excel.save()
def _parse_package(self):
'''
解析top命令中的包的cpu信息
:return:
'''
if self.packages == None or self.packages == "":
logger.error("no process name input, please input")
for package in self.packages:
package_dic = {"package": package, "pid": "", "pid_cpu": ""}
sp_lines = self.source.split('\n')
for line in sp_lines:
# logger.debug(line)
if package in line: #解析进程cpu信息
tmp = line.split()
self.pid = tmp[0]
target_pck = tmp[-1] #从中解析出的最后一个值是包名
self.datetime = TimeUtils.getCurrentTime()
logger.debug("cpuinfos, _parse top target_pck is : " +
str(target_pck) + " , self.pacakgename : " +
package)
if package == target_pck: #只统计包名完全相同的进程
if int(self.pid) > 0:
logger.debug(
"cpuinfos, into _parse_pck packege is target package, pid is :"
+ str(self.pid))
# logger.debug("into _parse_pck packege is target package, pid is :" + str(self.pid))
cpu_index = self.get_cpucol_index()
uid_index = self.get_uidcol_index()
if (len(tmp) > cpu_index):
self.pck_cpu_rate = tmp[cpu_index]
# CPU% 9% 有的格式会有%
self.pck_cpu_rate = self.pck_cpu_rate.replace(
"%", "")
if (len(tmp) > uid_index):
self.uid = tmp[uid_index]
package_dic = {
"package": package,
"pid": self.pid,
"pid_cpu": str(self.pck_cpu_rate),
"uid": self.uid
}
# self.package_list.append(package_dic)
# 将top中解析出来的信息保存在一个列表中，作为一条记录添加在package_list中
logger.debug("package: " + package +
", cpu_rate: " +
str(self.pck_cpu_rate))
self.total_pid_cpu = self.total_pid_cpu + float(
self.pck_cpu_rate)
break
self.package_list.append(package_dic)
logger.debug(package_dic)
def run_shell_cmd(self, cmd, **kwds):
'''执行 adb shell 命令
'''
# 如果失去连接后，adb又正常连接了
if not self.before_connect and self.after_connect:
cpu_uptime_file = os.path.join(RuntimeData.package_save_path, "uptime.txt")
with open(cpu_uptime_file, "a+",encoding = "utf-8") as writer:
writer.write(TimeUtils.getCurrentTimeUnderline() + " /proc/uptime:" + self.run_adb_cmd("shell cat /proc/uptime") + "\n")
self.before_connect = True
ret = self.run_adb_cmd('shell', '%s' % cmd, **kwds)
# 当 adb 命令传入 sync=False时，ret是Poen对象
if ret == None:
logger.error(u'adb cmd failed:%s ' % cmd)
return ret
def _activity_monitor_thread(self):
activity_title = ("datetime", "current_activity")
self.activity_file = os.path.join(RuntimeData.package_save_path,
'current_activity.csv')
try:
with open(self.activity_file, 'a+') as af:
csv.writer(af, lineterminator='\n').writerow(activity_title)
except Exception as e:
logger.error("file not found: " + str(self.activity_file))
while not self.stop_event.is_set():
try:
before = time.time()
self.current_activity = self.device.adb.get_current_activity()
collection_time = time.time()
activity_list = [collection_time, self.current_activity]
if self.activity_queue:
logger.debug("activity monitor thread activity_list: " +
str(activity_list))
self.activity_queue.put(activity_list)
if self.current_activity:
logger.debug("current activity: " + self.current_activity)
if self.main_activity and self.activity_list:
if self.current_activity not in self.activity_list:
start_activity = self.packagename + "/" + self.main_activity[
random.randint(0,
len(self.main_activity) - 1)]
logger.debug("start_activity:" + start_activity)
self.device.adb.start_activity(start_activity)
activity_tuple = (TimeUtils.getCurrentTime(),
self.current_activity)
# 写文件
try:
with open(self.activity_file, 'a+') as writer:
writer_p = csv.writer(writer, lineterminator='\n')
writer_p.writerow(activity_tuple)
except RuntimeError as e:
logger.error(e)
time_consume = time.time() - before
delta_inter = self.interval - time_consume
logger.debug("get app activity time consumed: " +
str(time_consume))
if delta_inter > 0:
time.sleep(delta_inter)
except Exception as e:
s = traceback.format_exc()
logger.debug(s) # 将堆栈信息打印到log中
if self.activity_queue:
self.activity_queue.task_done()
def _get_traffic_save(self, traffic_data, timestamp):
if isinstance(traffic_data, dict):
self.traffic_filename = traffic_data['traffic_file']
logger.debug("dataworker traffic_filename: " +
str(self.traffic_filename))
else:
try:
''' 0 1 2 3 4 5 6 7 8 9 10 11
traffic_data: ("datetime","packagename","uid","uid_total(KB)", "uid_total_packets", "rx(KB)", "rx_packets","tx(KB)","tx_packets","fg(KB)","bg(KB)","lo(KB)")
example： [collection_time, traffic_snapshot.packagename, traffic_snapshot.uid,TrafficUtils.byte2kb(traffic_snapshot.total_uid_bytes), traffic_snapshot.total_uid_packets,
TrafficUtils.byte2kb(traffic_snapshot.rx_uid_bytes),traffic_snapshot.rx_uid_packets, TrafficUtils.byte2kb(traffic_snapshot.tx_uid_bytes),
traffic_snapshot.tx_uid_packets, TrafficUtils.byte2kb(traffic_snapshot.fg_bytes), TrafficUtils.byte2kb(traffic_snapshot.bg_bytes),
TrafficUtils.byte2kb(traffic_snapshot.lo_uid_bytes)]
'''
traffic_data[0] = timestamp
dic = {
"time": traffic_data[0] * 1000,
"total": traffic_data[3],
"total_packets": traffic_data[4],
"rx": traffic_data[5],
"rx_packets": traffic_data[6],
"tx": traffic_data[7],
"tx_packets": traffic_data[8],
"fg": traffic_data[9],
"bg": traffic_data[10],
"lo": traffic_data[11]
}
self.perf_data['traffic'].append(dic)
with open(self.traffic_filename, 'a+') as writer:
logger.debug(
"write traffic data in dataworker traffic data timestamp: "
+ str(traffic_data[0]))
if isinstance(traffic_data[0], float):
traffic_data[0] = TimeUtils.formatTimeStamp(
traffic_data[0])
tmp_dic = copy.deepcopy(dic)
tmp_dic["time"] = traffic_data[0]
logger.debug(tmp_dic)
writer_p = csv.writer(writer, lineterminator='\n')
# logger.debug("------------------ dataworker trafficdata: " + str(traffic_data))
writer_p.writerow(traffic_data)
except Exception as e:
logger.error("traffic save error")
s = traceback.format_exc()
logger.debug(s)
def _top_cpuinfo(self):
self._top_pipe = self.device.adb.run_shell_cmd(self.top_cmd,
sync=False)
out = self._top_pipe.stdout.read()
error = self._top_pipe.stderr.read()
if error:
logger.error("into cpuinfos error : " + str(error))
return
out = str(out, "utf-8")
out.replace('\r', '')
top_file = os.path.join(RuntimeData.package_save_path, 'top.txt')
with open(top_file, "a+", encoding="utf-8") as writer:
writer.write(TimeUtils.getCurrentTime() + " top info:\n")
writer.write(out + "\n\n")
#避免文件过大，超过100M清理
if FileUtils.get_FileSize(top_file) > 100:
os.remove(top_file)
return PckCpuinfo(self.packages, out, self.sdkversion)
def _dumpsys_meminfo(self):
'''
总内存 各进程内存都从dumpsys meminfo中获取
这个方法挺耗时 约6 7秒才能完成
:return:
'''
time_old = time.time()
out = self.device.adb.run_shell_cmd('dumpsys meminfo')
meminfo_file = os.path.join(RuntimeData.package_save_path, 'dumpsys_meminfo.txt')
with open(meminfo_file, "a+",encoding="utf-8") as writer:
writer.write(TimeUtils.getCurrentTime()+" dumpsys meminfo info:\n")
writer.write(out+"\n\n")
# self.num = 0
passedtime = time.time() - time_old#测试meminfo这个命令的耗时，执行的时长在400多ms
logger.debug("dumpsys meminfo time consume:" + str(passedtime))
out.replace('\r', '')
return MemInfoDevice(dump=out,packages=self.packages)
def handle_launchtime(self, log_line):
'''
这个方法在每次一个启动时间的log产生时回调
:param log_line:最近一条的log 内容
:param tag:启动的方式，是normal的启动，还是自定义方式的启动：fullydrawnlaunch
#如果监控到到fully drawn这样的log，则优先统计这种log，它表示了到起始界面自定义界面的启动时间
:return:void
'''
# logger.debug(log_line)
# add begin by liurui
# 08-28 10:57:30.229 18882 19137 D IC5: CLogProducer == > code = 0, uuid = 4FE71E350379C64611CCD905938C10CA, eventType = performance, eventName = am_activity_launch_timeme, \
# log_time = 2019-08-28 10:57:30.229, contextInfo = {"tag": "am_activity_launch_time", "start_time": "2019-08-28 10:57:16",
# "activity_name_original": "com.android.settings\/.FallbackHome",
# "activity_name": "com.android.settings#com.android.settings.FallbackHome",
# "this_time": "916", "total_time": "916", "start_type": "code_start",
# "gmt_create": "2019-08-28 10:57:16.742", "uploadtime": "2019-08-28 10:57:30.173",
# "boottime": "2019-08-28 10:57:18.502", "firstupload": "2019-08-28 10:57:25.733"}
ltag = ""
if ("am_activity_launch_time" in log_line
or "am_activity_fully_drawn_time" in log_line):
# 最近增加的一条如果是启动时间相关的log，那么回调所有注册的_handle
if "am_activity_launch_time" in log_line:
ltag = "normal launch"
elif "am_activity_fully_drawn_time" in log_line:
ltag = "fullydrawn launch"
logger.debug("launchtime log:" + log_line)
if ltag:
content = []
timestamp = time.time()
content.append(TimeUtils.formatTimeStamp(timestamp))
temp_list = log_line.split()[-1].replace("[", "").replace(
"]", "").split(',')[2:5]
for i in range(len(temp_list)):
content.append(temp_list[i])
content.append(ltag)
logger.debug("Launch Info: " + str(content))
if len(content) == 5:
content = self.trim_value(content)
if content:
self.update_launch_list(content, timestamp)
def _get_power_save(self, power_data, timestamp):
if isinstance(power_data, dict):
self.power_filename = power_data['power_file']
logger.debug("dataworker power_filename: " +
str(self.power_filename))
else:
try:
''' 0 1 2 3 4
power_data: ("datetime","level","voltage(V)","tempreture(C)","current(mA)")
example: [collection_time, device_power_info.level, device_power_info.voltage,
device_power_info.temp, device_power_info.current]
'''
power_data[0] = timestamp
dic = {
"time": power_data[0] * 1000,
"level": power_data[1],
"vol": power_data[2],
"temp": power_data[3],
"current": power_data[4]
}
self.perf_data['power'].append(dic)
with open(self.power_filename, 'a+') as writer:
logger.debug(
"write power data in dataworker。。。。。。 timestamp:" +
str(power_data[0]))
if isinstance(power_data[0], float):
power_data[0] = TimeUtils.formatTimeStamp(
power_data[0])
tmp_dic = copy.deepcopy(dic)
tmp_dic["time"] = power_data[0]
logger.debug(tmp_dic)
writer_p = csv.writer(writer, lineterminator='\n')
# logger.debug("------------------ dataworker power data: " + str(power_data))
writer_p.writerow(power_data)
except Exception as e:
logger.error('power save error')
s = traceback.format_exc()
logger.debug(s)
def _get_cpu_save(self, cpu_data, timestamp):
if isinstance(cpu_data, dict):
self.cpu_filename = cpu_data['cpu_file']
logger.debug("cpu_filename: " + str(self.cpu_filename))
else:
try:
''' 0 1 2 3 4 5 6 7 8 9
cpu_data: ("datetime", " cpu_rate%", "user%", "system%", "all_jiffies","packagename", "pid", "uid", "pck_jiffies", "pid_cpu%")
对应的值是:[collection_time, cpu_info.cpu_rate, cpu_info.user_rate, cpu_info.system_rate, cpu_info.cpu_jiffs, cpu_pck_info.pckagename,
cpu_pck_info.pid, cpu_pck_info.uid,cpu_pck_info.p_cpu_jiffs, cpu_pck_info.p_cpu_rate],
'''
cpu_data[0] = timestamp
dic = {
"time": cpu_data[0] * 1000,
"total": cpu_data[1],
"cpu_jiffies": cpu_data[4],
"user": cpu_data[2],
"sys": cpu_data[3],
"pck_jiffies": cpu_data[8],
"pid_cpu": cpu_data[9]
}
self.perf_data['cpu'].append(dic)
with open(self.cpu_filename, 'a+') as writer:
logger.debug(
"write cpu data in dataworker mem timestamp: " +
str(cpu_data[0]))
cpu_data[0] = TimeUtils.formatTimeStamp(cpu_data[0])
tmp_dic = copy.deepcopy(dic)
tmp_dic["time"] = cpu_data[0]
logger.debug(tmp_dic)
writer_p = csv.writer(writer, lineterminator='\n')
# logger.debug("------------------ dataworker cpudate: " + str(cpu_data))
writer_p.writerow(cpu_data)
except Exception as e:
logger.error('cpu save error')
s = traceback.format_exc()
logger.error(s)
def _get_mem_save(self, mem_data, timestamp):
if isinstance(mem_data, dict):
self.mem_filename = mem_data['mem_file']
logger.debug("mem_filename: " + str(self.mem_filename))
else:
try:
''' 0 1 2 3 4 5 6
mem_data: ("datatime", "total_ram(KB)", "free_ram(KB)", "pckagename", "pid", "pid_pss(KB)", "pid_alloc_heap(KB)")
对应的值是：[formatTimeStamp(collection_time), cpu_info.cpu_rate, cpu_info.user_rate, cpu_info.system_rate, cpu_info.cpu_jiffs, cpu_pck_info.pckagename,
cpu_pck_info.pid, cpu_pck_info.uid,cpu_pck_info.p_cpu_jiffs, cpu_pck_info.p_cpu_rate]
'''
mem_data[0] = timestamp
dic = {
"time": mem_data[0] * 1000,
"total": mem_data[1],
"free": mem_data[2],
"pss": mem_data[5],
"heap": mem_data[6]
}
self.perf_data['mem'].append(dic)
with open(self.mem_filename, 'a+') as writer:
logger.debug(
"write mem data in dataworker。。。。。。 mem timestamp: " +
str(mem_data[0]))
if isinstance(mem_data[0], float):
mem_data[0] = TimeUtils.formatTimeStamp(mem_data[0])
tmp_dic = copy.deepcopy(dic)
tmp_dic["time"] = mem_data[0]
logger.debug(tmp_dic)
writer_p = csv.writer(writer, lineterminator='\n')
# logger.debug("------------------ dataworker memdata: " + str(mem_data))
writer_p.writerow(mem_data)
except Exception as e:
logger.error('mem save error')
s = traceback.format_exc()
logger.debug(s)
def is_overtime_days(self, filepath, days=7):
result = self.run_shell_cmd('ls -l %s' % filepath)
if not result:
return False
result = result.replace('\r\r\n', '\n')
if 'No such file or directory' in result:
logger.error('文件(夹) %s 不存在' % filepath)
return False
re_time = re.compile(r'\S*\s+(\d+-\d+-\d+\s+\d+:\d+)\s+\S+')
match = re_time.search(result)
if match:
last_modify_time = match.group(1)
logger.debug(last_modify_time)
last_modify_timestamp = TimeUtils.getTimeStamp(last_modify_time, "%Y-%m-%d %H:%M")
# logger.debug(last_modify_timestamp)
if last_modify_timestamp < (time.time() - days * 24 * 60 * 60):
logger.debug(filepath + " is overtime days:" + str(days))
return True
else:
logger.debug(filepath + " is not overtime days:" + str(days))
return False
logger.debug(filepath + " not have match time formatter")
return False
def handle_exception(self, log_line):
'''
这个方法在每次有log时回调
:param log_line:最近一条的log 内容
异常日志写一个文件
:return:void
'''
for tag in self.exception_log_list:
if tag in log_line:
logger.debug("exception Info: " + log_line)
tmp_file = os.path.join(RuntimeData.package_save_path,
'exception.log')
with open(tmp_file, 'a+') as f:
f.write(log_line + '\n')
# 这个路径 空格会有影响
process_stack_log_file = os.path.join(
RuntimeData.package_save_path, 'process_stack_%s_%s.log' %
(self.package, TimeUtils.getCurrentTimeUnderline()))
# 如果进程挂了，pid会变 ，抓变后进程pid的堆栈没有意义
# self.logmonitor.device.adb.get_process_stack(self.package,process_stack_log_file)
if RuntimeData.old_pid:
self.device.adb.get_process_stack_from_pid(
RuntimeData.old_pid, process_stack_log_file)
def run(self, time_out=None):
self.clear_heapdump()
# objgraph.show_growth()
# 对设备连接情况的检查
if not self.serialnum:
# androiddevice 没传 serialnum，默认执行adb shell
logger.info(
"serialnum in config file is null,default get connected phone")
is_device_connect = False
for i in range(0, 5):
if self.device.adb.is_connected(self.serialnum):
is_device_connect = True
break
else:
logger.error("device not found:" + self.serialnum)
time.sleep(2)
if not is_device_connect:
logger.error("after 5 times check,device not found:" +
self.serialnum)
return
# 对是否安装被测app的检查 只在最开始检查一次
if not self.device.adb.is_app_installed(self.packages[0]):
logger.error("test app not installed:" + self.packages[0])
return
try:
#初始化数据处理的类,将没有消息队列传递过去，以便获取数据，并处理
# datahandle = DataWorker(self.get_queue_dic())
# 将queue传进去，与datahandle那个线程交互
self.add_monitor(
CpuMonitor(self.serialnum, self.packages, self.frequency,
self.timeout))
self.add_monitor(
MemMonitor(self.serialnum, self.packages, self.frequency,
self.timeout))
self.add_monitor(
TrafficMonitor(self.serialnum, self.packages[0],
self.frequency, self.timeout))
# 软件方式 获取电量不准，已用硬件方案测试功耗
# self.add_monitor(PowerMonitor(self.serialnum, self.frequency,self.timeout))
self.add_monitor(
FPSMonitor(self.serialnum, self.packages[0], self.frequency,
self.timeout))
# 6.0以下能采集到fd数据，7.0以上没权限
if self.device.adb.get_sdk_version() <= 23:
self.add_monitor(
FdMonitor(self.serialnum, self.packages[0], self.frequency,
self.timeout))
self.add_monitor(
ThreadNumMonitor(self.serialnum, self.packages[0],
self.frequency, self.timeout))
if self.config_dic["monkey"] == "true":
self.add_monitor(Monkey(self.serialnum, self.packages[0]))
if self.config_dic["main_activity"] and self.config_dic[
"activity_list"]:
self.add_monitor(
DeviceMonitor(self.serialnum, self.packages[0],
self.frequency,
self.config_dic["main_activity"],
self.config_dic["activity_list"],
RuntimeData.exit_event))
if len(self.monitors):
start_time = TimeUtils.getCurrentTimeUnderline()
RuntimeData.start_time = start_time
if self.config_dic["save_path"]:
RuntimeData.package_save_path = os.path.join(
self.config_dic["save_path"], self.packages[0],
start_time)
else:
RuntimeData.package_save_path = os.path.join(
RuntimeData.top_dir, 'results', self.packages[0],
start_time)
FileUtils.makedir(RuntimeData.package_save_path)
self.save_device_info()
for monitor in self.monitors:
#启动所有的monitors
try:
monitor.start(start_time)
except Exception as e:
logger.error(e)
# logcat的代码可能会引起死锁，拎出来单独处理logcat
try:
self.logcat_monitor = LogcatMonitor(
self.serialnum, self.packages[0])
# 如果有异常日志标志，才启动这个模块
if self.exceptionlog_list:
self.logcat_monitor.set_exception_list(
self.exceptionlog_list)
self.logcat_monitor.add_log_handle(
self.logcat_monitor.handle_exception)
time.sleep(1)
self.logcat_monitor.start(start_time)
except Exception as e:
logger.error(e)
timeout = time_out if time_out != None else self.config_dic[
'timeout']
endtime = time.time() + timeout
while (time.time() < endtime): #吊着主线程防止线程中断
# 时间到或测试过程中检测到异常
if self.check_exit_signal_quit():
logger.error("app " + str(self.packages[0]) +
" exit signal, quit!")
break
time.sleep(self.frequency)
logger.debug("time is up,finish!!!")
self.stop()
# try:
# datahandle.stop()
# time.sleep(self.frequency*2)
# # 延迟一点时间结束上报，已让数据上报完
# # report.stop()
# except:
# logger.debug("report or datahandle stop exception")
# finally:
# logger.info("time is up, end")
# os._exit(0)
except KeyboardInterrupt: #捕获键盘异常的事件，例如ctrl c
logger.debug(" catch keyboardInterrupt, goodbye!!!")
# 收尾工作
self.stop()
os._exit(0)
except Exception as e:
logger.error("Exception in run")
logger.error(e)
def _calculator_thread(self, start_time):
'''处理surfaceflinger数据
'''
fps_file = os.path.join(RuntimeData.package_save_path, 'fps.csv')
if self.use_legacy_method:
fps_title = ['datetime', 'fps']
else:
fps_title = ['datetime', "activity window", 'fps', 'jank']
try:
with open(fps_file, 'a+') as df:
csv.writer(df, lineterminator='\n').writerow(fps_title)
if self.fps_queue:
fps_file_dic = {'fps_file': fps_file}
self.fps_queue.put(fps_file_dic)
except RuntimeError as e:
logger.exception(e)
while True:
try:
data = self.data_queue.get()
if isinstance(data, str) and data == 'Stop':
break
before = time.time()
if self.use_legacy_method:
td = data['timestamp'] - self.surface_before['timestamp']
seconds = td.seconds + td.microseconds / 1e6
frame_count = (data['page_flip_count'] -
self.surface_before['page_flip_count'])
fps = int(round(frame_count / seconds))
if fps > 60:
fps = 60
self.surface_before = data
logger.debug('FPS:%2s' % fps)
tmp_list = [TimeUtils.getCurrentTimeUnderline(), fps]
try:
with open(fps_file, 'a+') as f:
# tmp_list[0] = TimeUtils.formatTimeStamp(tmp_list[0])
csv.writer(f,
lineterminator='\n').writerow(tmp_list)
except RuntimeError as e:
logger.exception(e)
else:
refresh_period = data[0]
timestamps = data[1]
collect_time = data[2]
fps, jank = self._calculate_results(
refresh_period, timestamps)
logger.debug('FPS:%2s Jank:%s' % (fps, jank))
fps_list = [collect_time, self.focus_window, fps, jank]
if self.fps_queue:
self.fps_queue.put(fps_list)
if not self.fps_queue: #为了让单个脚本运行时保存数据
try:
with open(fps_file, 'a+') as f:
tmp_list = copy.deepcopy(fps_list)
tmp_list[0] = TimeUtils.formatTimeStamp(
tmp_list[0])
csv.writer(
f, lineterminator='\n').writerow(tmp_list)
except RuntimeError as e:
logger.exception(e)
time_consume = time.time() - before
delta_inter = self.frequency - time_consume
if delta_inter > 0:
time.sleep(delta_inter)
except:
logger.error(
"an exception hanpend in fps _calculator_thread ,reason unkown!"
)
s = traceback.format_exc()
logger.debug(s)
if self.fps_queue:
self.fps_queue.task_done()
def save(self):
pass
def parse(self, file_path):
'''解析
:param str file_path: 要解析数据文件的路径
'''
pass
def get_fps_collector(self):
'''获得fps收集器，收集器里保存着time fps jank的列表
:return: fps收集器
:rtype: SurfaceStatsCollector
'''
return self.fpscollector
if __name__ == '__main__':
# tulanduo android8.0 api level 27
monitor = FPSMonitor('TC79SSDMO7HEY5Z9',
"com.alibaba.ailabs.genie.smartapp", 1)
# mate 9 android8.0
# monitor = FPSMonitor('MKJNW18226007860',"com.sankuai.meituan",2)
# android8.0 Google Pixel 2
# monitor = FPSMonitor('HT7B81A05143',package_name = "com.alibaba.ailibs.genie.contacts",1)
monitor.start(TimeUtils.getCurrentTimeUnderline())
time.sleep(600)
monitor.stop()
class ThreadNumMonitor(object):
def __init__(self,
device_id,
packagename,
interval=1.0,
timeout=24 * 60 * 60,
thread_queue=None):
self.device = AndroidDevice(device_id)
if not packagename:
packagename = self.device.adb.get_foreground_process()
self.thread_package_collector = ThreadNumPackageCollector(
self.device, packagename, interval, timeout, thread_queue)
def start(self, start_time):
self.start_time = start_time
self.thread_package_collector.start(start_time)
def stop(self):
self.thread_package_collector.stop()
def save(self):
pass
if __name__ == "__main__":
monitor = ThreadNumMonitor("", "com.yunos.tv.alitvasr", 3)
monitor.start(TimeUtils.getCurrentTime())
time.sleep(20)
monitor.stop()
# monitor.save()
def _logcat_thread_func(self, save_dir, process_list, params=""):
'''获取logcat线程
'''
self.append_log_line_num = 0
self.file_log_line_num = 0
self.log_file_create_time = None
logs = []
logger.debug("logcat_thread_func")
log_is_none = 0
while self._logcat_running:
try:
log = self._log_pipe.stdout.readline().strip()
if not isinstance(log, str):
try:
log = str(log,"utf8")
except Exception as e:
log = repr(log)
logger.error('str error:'+log)
logger.error(e)
if log:
log_is_none = 0
# logger.debug(log)
logs.append(log)
# if self._log_pipe.poll() != None:
# logger.debug('process:%s have exited' % self._log_pipe.pid)
# if self._logcat_running :
# self._log_pipe = self.run_shell_cmd('logcat ' + params, sync=False)
# else :
# break
for _handle in self._logcat_handle:
try:
_handle(log)
except Exception as e:
logger.error("an exception happen in logcat handle log , reason unkown!, e:")
logger.error(e)
self.append_log_line_num = self.append_log_line_num + 1
self.file_log_line_num = self.file_log_line_num + 1
# if self.append_log_line_num > 1000:
if self.append_log_line_num > 100:
if not self.log_file_create_time:
self.log_file_create_time = TimeUtils.getCurrentTimeUnderline()
logcat_file = os.path.join(save_dir,
'logcat_%s.log' % self.log_file_create_time)
self.append_log_line_num = 0
self.save(logcat_file, logs)
logs = []
# 新建文件
if self.file_log_line_num > 600000:
# if self.file_log_line_num > 200:
self.file_log_line_num = 0
self.log_file_create_time = TimeUtils.getCurrentTimeUnderline()
logcat_file = os.path.join(save_dir, 'logcat_%s.log' % self.log_file_create_time)
self.save(logcat_file, logs)
logs = []
else:
log_is_none = log_is_none + 1
if log_is_none % 1000 == 0:
logger.info("log is none")
self._log_pipe = self.run_shell_cmd('logcat -v threadtime ' + params, sync=False)
except:
logger.error("an exception hanpend in logcat thread, reason unkown!")
s = traceback.format_exc()
logger.debug(s)
def dump_native_heap(self, package, save_path):
native_heap_file = "/data/local/tmp/%s_native_heap_%s.txt" % (package, TimeUtils.getCurrentTimeUnderline())
self.run_shell_cmd("am dumpheap -n %s %s" % (package, native_heap_file))
def dumpheap(self, package, save_path):
heapfile = "/data/local/tmp/%s_dumpheap_%s.hprof" % (package, TimeUtils.getCurrentTimeUnderline())
self.run_shell_cmd("am dumpheap %s %s" % (package, heapfile))
time.sleep(10)
self.pull_file(heapfile,save_path)
