def _monkey_thread_func(self,save_dir): '''获取monkey线程,保存monkey日志,monkey Crash日志暂不处理,后续有需要再处理 ''' self.append_log_line_num = 0 self.file_log_line_num = 0 self.log_file_create_time = None log_is_none = 0 logs = [] logger.debug("monkey_thread_func") if RuntimeData.start_time is None: RuntimeData.start_time = TimeUtils.getCurrentTime() while self.running: try: log = self._log_pipe.stdout.readline().strip() if not isinstance(log, str): # 先编码为unicode try: log = str(log, "utf8") except Exception as e: log = repr(log) logger.error('str error:' + log) logger.error(e) if log: logs.append(log) self.append_log_line_num = self.append_log_line_num + 1 self.file_log_line_num = self.file_log_line_num + 1 # if self.append_log_line_num > 1000: if self.append_log_line_num > 100: if not self.log_file_create_time: self.log_file_create_time = TimeUtils.getCurrentTimeUnderline() log_file = os.path.join(save_dir, 'monkey_%s.log' % self.log_file_create_time) self.append_log_line_num = 0 # 降低音量,避免音量过大,导致语音指令失败 self.device.adb.run_shell_cmd("input keyevent 25") self.save(log_file, logs) logs = [] # 新建文件 if self.file_log_line_num > 600000: # if self.file_log_line_num > 200: self.file_log_line_num = 0 self.log_file_create_time = TimeUtils.getCurrentTimeUnderline() log_file = os.path.join(save_dir, 'monkey_%s.log' % self.log_file_create_time) self.save(log_file, logs) logs = [] else: log_is_none = log_is_none + 1 if log_is_none % 1000 == 0: logger.info("log is none") if not self.device.adb.is_process_running("com.android.commands.monkey") and self.running: self.device.adb.kill_process("com.android.commands.monkey") self._log_pipe = self.device.adb.run_shell_cmd(self.monkey_cmd, sync=False) except: logger.error("an exception hanpend in monkey thread, reason unkown!") s = traceback.format_exc() logger.debug(s)
def stop(self): for monitor in self.monitors: try: monitor.stop() except Exception as e: # 捕获所有的异常,防止其中一个monitor的stop操作发生异常退出时,影响其他的monitor的stop操作 logger.error(e) try: if self.logcat_monitor: self.logcat_monitor.stop() except Exception as e: logger.error("stop exception for logcat monitor") logger.error(e) if self.config_dic["monkey"] == "true": self.device.adb.kill_process("com.android.commands.monkey") # 统计测试时长 cost_time = round((float)(time.time() - TimeUtils.getTimeStamp( RuntimeData.start_time, TimeUtils.UnderLineFormatter)) / 3600, 2) self.add_device_info("test cost time:", str(cost_time) + "h") # 根据csv生成excel汇总文件 Report(RuntimeData.package_save_path, self.packages) self.pull_heapdump() self.pull_log_files() # self.memory_analyse() # self.device.adb.bugreport(RuntimeData.package_save_path) os._exit(0)
def _collect_thread_num_thread(self, start_time): end_time = time.time() + self._timeout thread_list_titile = ("datatime", "packagename", "pid", "thread_num") thread_num_file = os.path.join(RuntimeData.package_save_path, 'thread_num.csv') try: with open(thread_num_file, 'a+') as df: csv.writer(df, lineterminator='\n').writerow(thread_list_titile) if self.thread_queue: thread_file_dic = {'thread_file': thread_num_file} self.thread_queue.put(thread_file_dic) except RuntimeError as e: logger.error(e) while not self._stop_event.is_set() and time.time() < end_time: try: before = time.time() logger.debug( "-----------into _collect_thread_num_thread loop, thread is : " + str(threading.current_thread().name)) # 获取pakagename的thread num信息 thread_pck_info = self.get_process_thread_num(self.packagename) logger.debug(thread_pck_info) current_time = TimeUtils.getCurrentTime() if not thread_pck_info: continue else: logger.debug("current time: " + current_time + ", processname: " + thread_pck_info[1] + ", pid: " + str(thread_pck_info[2]) + " thread num: " + str(thread_pck_info[3])) if self.thread_queue: self.thread_queue.put(thread_pck_info) if not self.thread_queue: #为了本地单个文件运行 try: with open(thread_num_file, 'a+') as thread_writer: writer_p = csv.writer(thread_writer, lineterminator='\n') thread_pck_info[0] = current_time writer_p.writerow(thread_pck_info) except RuntimeError as e: logger.error(e) after = time.time() time_consume = after - before delta_inter = self._interval - time_consume logger.debug("time_consume for thread num infos: " + str(time_consume)) if delta_inter > 0: time.sleep(delta_inter) except: logger.error( "an exception hanpend in thread num thread, reason unkown!" ) s = traceback.format_exc() logger.debug(s) if self.thread_queue: self.thread_queue.task_done()
def _get_activity_save(self, activity_data, timestamp): if self.first_time: activity_title = ("datetime", "current_activity") self.first_time = False self.activity_file = os.path.join(RuntimeData.package_save_path, 'current_activity.csv') try: with open(self.activity_file, 'a+') as af: csv.writer(af, lineterminator='\n').writerow(activity_title) except Exception as e: logger.error("file not found: " + str(self.activity_file)) else: try: activity_data[0] = timestamp dic = { "time": activity_data[0] * 1000, "name": activity_data[1] } self.perf_data['activity'].append(dic) with open(self.activity_file, 'a+') as writer: if isinstance(activity_data[0], float): activity_data[0] = TimeUtils.formatTimeStamp( activity_data[0]) tmp_dic = copy.deepcopy(dic) tmp_dic["time"] = activity_data[0] logger.debug(tmp_dic) writer_p = csv.writer(writer, lineterminator='\n') writer_p.writerow(activity_data) except Exception as e: logger.error("activity save error ") s = traceback.format_exc() logger.debug(s)
def list_dir_between_time(self, dir_path, start_time, end_time): '''列取目录下 起止时间点之间的文件 start_time end_time 时间戳 返回文件绝对路径 列表 ''' # ls - l # -rwxrwx--- 1 root root 19897899 2018-12-27 18:02 com.alibaba.ailabs.ar.fireeye2_dumpheap_2018_12_27_18_02_52.hprof result = self.run_shell_cmd('ls -l %s' % dir_path) if not result: return "" result = result.replace('\r\r\n', '\n') if 'No such file or directory' in result: logger.error('文件(夹) %s 不存在' % dir_path) file_list = [] re_time = re.compile(r'\S*\s+(\d+-\d+-\d+\s+\d+:\d+)\s+\S+') for line in result.split('\n'): items = line.split() match = re_time.search(line) if match: last_modify_time = match.group(1) logger.debug(last_modify_time) last_modify_timestamp = TimeUtils.getTimeStamp(last_modify_time, "%Y-%m-%d %H:%M") # logger.debug(last_modify_timestamp) if start_time < last_modify_timestamp and last_modify_timestamp < end_time: logger.debug("append file:" + items[-1]) file_list.append('%s/%s' % (dir_path, items[-1])) return file_list
def _get_fps_save(self, fps_data, timestamp): if isinstance(fps_data, dict): self.fps_filename = fps_data['fps_file'] logger.debug("fps_filename: " + str(self.fps_filename)) else: try: '''0 1 2 3 fps_data: ("datetime", "activity","fps", "jank") 对应的值是:[formatter(collection_time), activity,fps,jank], ''' fps_data[0] = timestamp dic = { "time": fps_data[0] * 1000, "activity": fps_data[1], "fps": fps_data[2], "jank": fps_data[3] } self.perf_data['fluency'].append(dic) with open(self.fps_filename, 'a+') as writer: logger.debug( "dataworker write fps data in dataworker。fps timestamp: " + str(fps_data[0])) fps_data[0] = TimeUtils.formatTimeStamp(fps_data[0]) tmp_dic = copy.deepcopy(dic) tmp_dic["time"] = fps_data[0] logger.debug(tmp_dic) writer_p = csv.writer(writer, lineterminator='\n') writer_p.writerow(fps_data) except Exception as e: s = traceback.format_exc() logger.error(s) # 将堆栈信息打印到log中 logger.error("fps save error")
def _get_fd_save(self, fd_data, timestamp): if isinstance(fd_data, dict): self.fd_file = fd_data['fd_file'] logger.debug("fd_file: " + str(self.fd_file)) else: try: ''' 0 1 2 3 fd_data: ("datatime", "pckagename", "pid", "fd num") 对应的值是:[formatTimeStamp(collection_time), packagename, pid,fd_num] ''' fd_data[0] = timestamp dic = { "time": fd_data[0] * 1000, "package": fd_data[1], "pid": fd_data[2], "fd": fd_data[3] } self.perf_data['fd'].append(dic) with open(self.fd_file, 'a+') as writer: logger.debug( "write fd data in dataworker。。。。。。 fd timestamp: " + str(fd_data[0])) if isinstance(fd_data[0], float): fd_data[0] = TimeUtils.formatTimeStamp(fd_data[0]) tmp_dic = copy.deepcopy(dic) tmp_dic["time"] = fd_data[0] logger.debug(tmp_dic) writer_p = csv.writer(writer, lineterminator='\n') writer_p.writerow(fd_data) except Exception as e: logger.error('fd save error') s = traceback.format_exc() logger.debug(s)
def _dumpsys_process_meminfo(self, process): ''' dump 进程详细内存 耗时 1s以内 :param process: :return: ''' time_old = time.time() out = self.device.adb.run_shell_cmd('dumpsys meminfo %s' % process) # self.num = self.num + 1 # if self.num % 10 == 0: #避免:在windows 无法创建文件名,不能有冒号: process_rename = process.replace(":", "_") meminfo_file = os.path.join(RuntimeData.package_save_path, 'dumpsys_meminfo_%s.txt' % process_rename) with open(meminfo_file, "a+") as writer: writer.write(TimeUtils.getCurrentTime() + " dumpsys meminfo package info:\n") if out: writer.write(out + "\n\n") # self.num = 0 passedtime = time.time() - time_old #测试meminfo这个命令的耗时,执行的时长在400多ms logger.debug("dumpsys meminfo package time consume:" + str(passedtime)) out.replace('\r', '') return MemInfoPackage(dump=out)
def get_max_freq(self): out = self.device.adb.run_shell_cmd( "cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_max_freq") out.replace('\r', '') max_freq_file = os.path.join(RuntimeData.package_save_path, 'scaling_max_freq.txt') with open(max_freq_file, "a+", encoding="utf-8") as writer: writer.write(TimeUtils.getCurrentTime() + " scaling_max_freq:\n") writer.write(out + "\n\n")
def _collect_power_thread(self,start_time): ''' 搜集电池信息的线程 :return: ''' end_time = time.time() + self._timeout power_list_titile = ("datetime","level","voltage(V)","tempreture(C)","current(mA)") power_device_file = os.path.join(RuntimeData.package_save_path, 'powerinfo.csv') try: with open(power_device_file, 'a+') as df: csv.writer(df, lineterminator='\n').writerow(power_list_titile) if self.power_queue: power_file_dic = {'power_file':power_device_file} self.power_queue.put(power_file_dic) except RuntimeError as e: logger.error(e) while not self._stop_event.is_set() and time.time() < end_time: try: before = time.time() logger.debug("------------into _collect_power_thread loop thread is : " + str(threading.current_thread().name)) device_power_info = self._get_battaryproperties() if device_power_info.source == '': logger.debug("can't get power info , break!") break device_power_info = self.trim_data(device_power_info)#debug collection_time = time.time() logger.debug(" collection time in powerconsumption is : " + str(collection_time)) power_tmp_list = [collection_time, device_power_info.level, device_power_info.voltage, device_power_info.temp, device_power_info.current] if self.power_queue: self.power_queue.put(power_tmp_list) if not self.power_queue:#为了本地单个脚本运行 power_tmp_list[0] = TimeUtils.formatTimeStamp(power_tmp_list[0]) try: with open(power_device_file,'a+',encoding="utf-8") as writer: writer_p = csv.writer(writer, lineterminator='\n') writer_p.writerow(power_tmp_list) except RuntimeError as e: logger.error(e) after = time.time() time_consume = after - before delta_inter = self._interval - time_consume if delta_inter > 0: time.sleep(delta_inter) except: logger.error("an exception hanpend in powerconsumption thread , reason unkown!") s = traceback.format_exc() logger.debug(s) if self.power_queue: self.power_queue.task_done()
def __init__(self, csv_dir, packages=[]): os.chdir(csv_dir) # 需要画曲线的csv文件名 self.summary_csf_file = { "cpuinfo.csv": { "table_name": "pid_cpu", "x_axis": "datatime", "y_axis": "%", "values": ["pid_cpu%", "total_pid_cpu%"] }, "meminfo.csv": { "table_name": "pid_pss", "x_axis": "datatime", "y_axis": "mem(MB)", "values": ["pid_pss(MB)", "total_pss(MB)"] }, "pid_change.csv": { "table_name": "pid", "x_axis": "datatime", "y_axis": "pid_num", "values": ["pid"] }, } self.packages = packages if len(self.packages) > 0: for package in self.packages: pss_detail_dic = { "table_name": "pss_detail", "x_axis": "datatime", "y_axis": "mem(MB)", "values": ["pss", "java_heap", "native_heap", "system"] } # 文件名太长会导致写excel失败 self.summary_csf_file["pss_%s.csv" % package.split(".")[-1].replace( ":", "_")] = pss_detail_dic logger.debug(self.packages) logger.debug(self.summary_csf_file) logger.info('create report for %s' % csv_dir) file_names = self.filter_file_names(csv_dir) logger.debug('%s' % file_names) if file_names: book_name = 'summary_%s.xlsx' % TimeUtils.getCurrentTimeUnderline() excel = Excel(book_name) for file_name in file_names: logger.debug('get csv %s to excel' % file_name) values = self.summary_csf_file[file_name] excel.csv_to_xlsx(file_name, values["table_name"], values["x_axis"], values["y_axis"], values["values"]) logger.info('wait to save %s' % book_name) excel.save()
def _parse_package(self): ''' 解析top命令中的包的cpu信息 :return: ''' if self.packages == None or self.packages == "": logger.error("no process name input, please input") for package in self.packages: package_dic = {"package": package, "pid": "", "pid_cpu": ""} sp_lines = self.source.split('\n') for line in sp_lines: # logger.debug(line) if package in line: #解析进程cpu信息 tmp = line.split() self.pid = tmp[0] target_pck = tmp[-1] #从中解析出的最后一个值是包名 self.datetime = TimeUtils.getCurrentTime() logger.debug("cpuinfos, _parse top target_pck is : " + str(target_pck) + " , self.pacakgename : " + package) if package == target_pck: #只统计包名完全相同的进程 if int(self.pid) > 0: logger.debug( "cpuinfos, into _parse_pck packege is target package, pid is :" + str(self.pid)) # logger.debug("into _parse_pck packege is target package, pid is :" + str(self.pid)) cpu_index = self.get_cpucol_index() uid_index = self.get_uidcol_index() if (len(tmp) > cpu_index): self.pck_cpu_rate = tmp[cpu_index] # CPU% 9% 有的格式会有% self.pck_cpu_rate = self.pck_cpu_rate.replace( "%", "") if (len(tmp) > uid_index): self.uid = tmp[uid_index] package_dic = { "package": package, "pid": self.pid, "pid_cpu": str(self.pck_cpu_rate), "uid": self.uid } # self.package_list.append(package_dic) # 将top中解析出来的信息保存在一个列表中,作为一条记录添加在package_list中 logger.debug("package: " + package + ", cpu_rate: " + str(self.pck_cpu_rate)) self.total_pid_cpu = self.total_pid_cpu + float( self.pck_cpu_rate) break self.package_list.append(package_dic) logger.debug(package_dic)
def run_shell_cmd(self, cmd, **kwds): '''执行 adb shell 命令 ''' # 如果失去连接后,adb又正常连接了 if not self.before_connect and self.after_connect: cpu_uptime_file = os.path.join(RuntimeData.package_save_path, "uptime.txt") with open(cpu_uptime_file, "a+",encoding = "utf-8") as writer: writer.write(TimeUtils.getCurrentTimeUnderline() + " /proc/uptime:" + self.run_adb_cmd("shell cat /proc/uptime") + "\n") self.before_connect = True ret = self.run_adb_cmd('shell', '%s' % cmd, **kwds) # 当 adb 命令传入 sync=False时,ret是Poen对象 if ret == None: logger.error(u'adb cmd failed:%s ' % cmd) return ret
def _activity_monitor_thread(self): activity_title = ("datetime", "current_activity") self.activity_file = os.path.join(RuntimeData.package_save_path, 'current_activity.csv') try: with open(self.activity_file, 'a+') as af: csv.writer(af, lineterminator='\n').writerow(activity_title) except Exception as e: logger.error("file not found: " + str(self.activity_file)) while not self.stop_event.is_set(): try: before = time.time() self.current_activity = self.device.adb.get_current_activity() collection_time = time.time() activity_list = [collection_time, self.current_activity] if self.activity_queue: logger.debug("activity monitor thread activity_list: " + str(activity_list)) self.activity_queue.put(activity_list) if self.current_activity: logger.debug("current activity: " + self.current_activity) if self.main_activity and self.activity_list: if self.current_activity not in self.activity_list: start_activity = self.packagename + "/" + self.main_activity[ random.randint(0, len(self.main_activity) - 1)] logger.debug("start_activity:" + start_activity) self.device.adb.start_activity(start_activity) activity_tuple = (TimeUtils.getCurrentTime(), self.current_activity) # 写文件 try: with open(self.activity_file, 'a+') as writer: writer_p = csv.writer(writer, lineterminator='\n') writer_p.writerow(activity_tuple) except RuntimeError as e: logger.error(e) time_consume = time.time() - before delta_inter = self.interval - time_consume logger.debug("get app activity time consumed: " + str(time_consume)) if delta_inter > 0: time.sleep(delta_inter) except Exception as e: s = traceback.format_exc() logger.debug(s) # 将堆栈信息打印到log中 if self.activity_queue: self.activity_queue.task_done()
def _get_traffic_save(self, traffic_data, timestamp): if isinstance(traffic_data, dict): self.traffic_filename = traffic_data['traffic_file'] logger.debug("dataworker traffic_filename: " + str(self.traffic_filename)) else: try: ''' 0 1 2 3 4 5 6 7 8 9 10 11 traffic_data: ("datetime","packagename","uid","uid_total(KB)", "uid_total_packets", "rx(KB)", "rx_packets","tx(KB)","tx_packets","fg(KB)","bg(KB)","lo(KB)") example: [collection_time, traffic_snapshot.packagename, traffic_snapshot.uid,TrafficUtils.byte2kb(traffic_snapshot.total_uid_bytes), traffic_snapshot.total_uid_packets, TrafficUtils.byte2kb(traffic_snapshot.rx_uid_bytes),traffic_snapshot.rx_uid_packets, TrafficUtils.byte2kb(traffic_snapshot.tx_uid_bytes), traffic_snapshot.tx_uid_packets, TrafficUtils.byte2kb(traffic_snapshot.fg_bytes), TrafficUtils.byte2kb(traffic_snapshot.bg_bytes), TrafficUtils.byte2kb(traffic_snapshot.lo_uid_bytes)] ''' traffic_data[0] = timestamp dic = { "time": traffic_data[0] * 1000, "total": traffic_data[3], "total_packets": traffic_data[4], "rx": traffic_data[5], "rx_packets": traffic_data[6], "tx": traffic_data[7], "tx_packets": traffic_data[8], "fg": traffic_data[9], "bg": traffic_data[10], "lo": traffic_data[11] } self.perf_data['traffic'].append(dic) with open(self.traffic_filename, 'a+') as writer: logger.debug( "write traffic data in dataworker traffic data timestamp: " + str(traffic_data[0])) if isinstance(traffic_data[0], float): traffic_data[0] = TimeUtils.formatTimeStamp( traffic_data[0]) tmp_dic = copy.deepcopy(dic) tmp_dic["time"] = traffic_data[0] logger.debug(tmp_dic) writer_p = csv.writer(writer, lineterminator='\n') # logger.debug("------------------ dataworker trafficdata: " + str(traffic_data)) writer_p.writerow(traffic_data) except Exception as e: logger.error("traffic save error") s = traceback.format_exc() logger.debug(s)
def _top_cpuinfo(self): self._top_pipe = self.device.adb.run_shell_cmd(self.top_cmd, sync=False) out = self._top_pipe.stdout.read() error = self._top_pipe.stderr.read() if error: logger.error("into cpuinfos error : " + str(error)) return out = str(out, "utf-8") out.replace('\r', '') top_file = os.path.join(RuntimeData.package_save_path, 'top.txt') with open(top_file, "a+", encoding="utf-8") as writer: writer.write(TimeUtils.getCurrentTime() + " top info:\n") writer.write(out + "\n\n") #避免文件过大,超过100M清理 if FileUtils.get_FileSize(top_file) > 100: os.remove(top_file) return PckCpuinfo(self.packages, out, self.sdkversion)
def _dumpsys_meminfo(self): ''' 总内存 各进程内存都从dumpsys meminfo中获取 这个方法挺耗时 约6 7秒才能完成 :return: ''' time_old = time.time() out = self.device.adb.run_shell_cmd('dumpsys meminfo') meminfo_file = os.path.join(RuntimeData.package_save_path, 'dumpsys_meminfo.txt') with open(meminfo_file, "a+",encoding="utf-8") as writer: writer.write(TimeUtils.getCurrentTime()+" dumpsys meminfo info:\n") writer.write(out+"\n\n") # self.num = 0 passedtime = time.time() - time_old#测试meminfo这个命令的耗时,执行的时长在400多ms logger.debug("dumpsys meminfo time consume:" + str(passedtime)) out.replace('\r', '') return MemInfoDevice(dump=out,packages=self.packages)
def handle_launchtime(self, log_line): ''' 这个方法在每次一个启动时间的log产生时回调 :param log_line:最近一条的log 内容 :param tag:启动的方式,是normal的启动,还是自定义方式的启动:fullydrawnlaunch #如果监控到到fully drawn这样的log,则优先统计这种log,它表示了到起始界面自定义界面的启动时间 :return:void ''' # logger.debug(log_line) # add begin by liurui # 08-28 10:57:30.229 18882 19137 D IC5: CLogProducer == > code = 0, uuid = 4FE71E350379C64611CCD905938C10CA, eventType = performance, eventName = am_activity_launch_timeme, \ # log_time = 2019-08-28 10:57:30.229, contextInfo = {"tag": "am_activity_launch_time", "start_time": "2019-08-28 10:57:16", # "activity_name_original": "com.android.settings\/.FallbackHome", # "activity_name": "com.android.settings#com.android.settings.FallbackHome", # "this_time": "916", "total_time": "916", "start_type": "code_start", # "gmt_create": "2019-08-28 10:57:16.742", "uploadtime": "2019-08-28 10:57:30.173", # "boottime": "2019-08-28 10:57:18.502", "firstupload": "2019-08-28 10:57:25.733"} ltag = "" if ("am_activity_launch_time" in log_line or "am_activity_fully_drawn_time" in log_line): # 最近增加的一条如果是启动时间相关的log,那么回调所有注册的_handle if "am_activity_launch_time" in log_line: ltag = "normal launch" elif "am_activity_fully_drawn_time" in log_line: ltag = "fullydrawn launch" logger.debug("launchtime log:" + log_line) if ltag: content = [] timestamp = time.time() content.append(TimeUtils.formatTimeStamp(timestamp)) temp_list = log_line.split()[-1].replace("[", "").replace( "]", "").split(',')[2:5] for i in range(len(temp_list)): content.append(temp_list[i]) content.append(ltag) logger.debug("Launch Info: " + str(content)) if len(content) == 5: content = self.trim_value(content) if content: self.update_launch_list(content, timestamp)
def _get_power_save(self, power_data, timestamp): if isinstance(power_data, dict): self.power_filename = power_data['power_file'] logger.debug("dataworker power_filename: " + str(self.power_filename)) else: try: ''' 0 1 2 3 4 power_data: ("datetime","level","voltage(V)","tempreture(C)","current(mA)") example: [collection_time, device_power_info.level, device_power_info.voltage, device_power_info.temp, device_power_info.current] ''' power_data[0] = timestamp dic = { "time": power_data[0] * 1000, "level": power_data[1], "vol": power_data[2], "temp": power_data[3], "current": power_data[4] } self.perf_data['power'].append(dic) with open(self.power_filename, 'a+') as writer: logger.debug( "write power data in dataworker。。。。。。 timestamp:" + str(power_data[0])) if isinstance(power_data[0], float): power_data[0] = TimeUtils.formatTimeStamp( power_data[0]) tmp_dic = copy.deepcopy(dic) tmp_dic["time"] = power_data[0] logger.debug(tmp_dic) writer_p = csv.writer(writer, lineterminator='\n') # logger.debug("------------------ dataworker power data: " + str(power_data)) writer_p.writerow(power_data) except Exception as e: logger.error('power save error') s = traceback.format_exc() logger.debug(s)
def _get_cpu_save(self, cpu_data, timestamp): if isinstance(cpu_data, dict): self.cpu_filename = cpu_data['cpu_file'] logger.debug("cpu_filename: " + str(self.cpu_filename)) else: try: ''' 0 1 2 3 4 5 6 7 8 9 cpu_data: ("datetime", " cpu_rate%", "user%", "system%", "all_jiffies","packagename", "pid", "uid", "pck_jiffies", "pid_cpu%") 对应的值是:[collection_time, cpu_info.cpu_rate, cpu_info.user_rate, cpu_info.system_rate, cpu_info.cpu_jiffs, cpu_pck_info.pckagename, cpu_pck_info.pid, cpu_pck_info.uid,cpu_pck_info.p_cpu_jiffs, cpu_pck_info.p_cpu_rate], ''' cpu_data[0] = timestamp dic = { "time": cpu_data[0] * 1000, "total": cpu_data[1], "cpu_jiffies": cpu_data[4], "user": cpu_data[2], "sys": cpu_data[3], "pck_jiffies": cpu_data[8], "pid_cpu": cpu_data[9] } self.perf_data['cpu'].append(dic) with open(self.cpu_filename, 'a+') as writer: logger.debug( "write cpu data in dataworker mem timestamp: " + str(cpu_data[0])) cpu_data[0] = TimeUtils.formatTimeStamp(cpu_data[0]) tmp_dic = copy.deepcopy(dic) tmp_dic["time"] = cpu_data[0] logger.debug(tmp_dic) writer_p = csv.writer(writer, lineterminator='\n') # logger.debug("------------------ dataworker cpudate: " + str(cpu_data)) writer_p.writerow(cpu_data) except Exception as e: logger.error('cpu save error') s = traceback.format_exc() logger.error(s)
def _get_mem_save(self, mem_data, timestamp): if isinstance(mem_data, dict): self.mem_filename = mem_data['mem_file'] logger.debug("mem_filename: " + str(self.mem_filename)) else: try: ''' 0 1 2 3 4 5 6 mem_data: ("datatime", "total_ram(KB)", "free_ram(KB)", "pckagename", "pid", "pid_pss(KB)", "pid_alloc_heap(KB)") 对应的值是:[formatTimeStamp(collection_time), cpu_info.cpu_rate, cpu_info.user_rate, cpu_info.system_rate, cpu_info.cpu_jiffs, cpu_pck_info.pckagename, cpu_pck_info.pid, cpu_pck_info.uid,cpu_pck_info.p_cpu_jiffs, cpu_pck_info.p_cpu_rate] ''' mem_data[0] = timestamp dic = { "time": mem_data[0] * 1000, "total": mem_data[1], "free": mem_data[2], "pss": mem_data[5], "heap": mem_data[6] } self.perf_data['mem'].append(dic) with open(self.mem_filename, 'a+') as writer: logger.debug( "write mem data in dataworker。。。。。。 mem timestamp: " + str(mem_data[0])) if isinstance(mem_data[0], float): mem_data[0] = TimeUtils.formatTimeStamp(mem_data[0]) tmp_dic = copy.deepcopy(dic) tmp_dic["time"] = mem_data[0] logger.debug(tmp_dic) writer_p = csv.writer(writer, lineterminator='\n') # logger.debug("------------------ dataworker memdata: " + str(mem_data)) writer_p.writerow(mem_data) except Exception as e: logger.error('mem save error') s = traceback.format_exc() logger.debug(s)
def is_overtime_days(self, filepath, days=7): result = self.run_shell_cmd('ls -l %s' % filepath) if not result: return False result = result.replace('\r\r\n', '\n') if 'No such file or directory' in result: logger.error('文件(夹) %s 不存在' % filepath) return False re_time = re.compile(r'\S*\s+(\d+-\d+-\d+\s+\d+:\d+)\s+\S+') match = re_time.search(result) if match: last_modify_time = match.group(1) logger.debug(last_modify_time) last_modify_timestamp = TimeUtils.getTimeStamp(last_modify_time, "%Y-%m-%d %H:%M") # logger.debug(last_modify_timestamp) if last_modify_timestamp < (time.time() - days * 24 * 60 * 60): logger.debug(filepath + " is overtime days:" + str(days)) return True else: logger.debug(filepath + " is not overtime days:" + str(days)) return False logger.debug(filepath + " not have match time formatter") return False
def handle_exception(self, log_line): ''' 这个方法在每次有log时回调 :param log_line:最近一条的log 内容 异常日志写一个文件 :return:void ''' for tag in self.exception_log_list: if tag in log_line: logger.debug("exception Info: " + log_line) tmp_file = os.path.join(RuntimeData.package_save_path, 'exception.log') with open(tmp_file, 'a+') as f: f.write(log_line + '\n') # 这个路径 空格会有影响 process_stack_log_file = os.path.join( RuntimeData.package_save_path, 'process_stack_%s_%s.log' % (self.package, TimeUtils.getCurrentTimeUnderline())) # 如果进程挂了,pid会变 ,抓变后进程pid的堆栈没有意义 # self.logmonitor.device.adb.get_process_stack(self.package,process_stack_log_file) if RuntimeData.old_pid: self.device.adb.get_process_stack_from_pid( RuntimeData.old_pid, process_stack_log_file)
def run(self, time_out=None): self.clear_heapdump() # objgraph.show_growth() # 对设备连接情况的检查 if not self.serialnum: # androiddevice 没传 serialnum,默认执行adb shell logger.info( "serialnum in config file is null,default get connected phone") is_device_connect = False for i in range(0, 5): if self.device.adb.is_connected(self.serialnum): is_device_connect = True break else: logger.error("device not found:" + self.serialnum) time.sleep(2) if not is_device_connect: logger.error("after 5 times check,device not found:" + self.serialnum) return # 对是否安装被测app的检查 只在最开始检查一次 if not self.device.adb.is_app_installed(self.packages[0]): logger.error("test app not installed:" + self.packages[0]) return try: #初始化数据处理的类,将没有消息队列传递过去,以便获取数据,并处理 # datahandle = DataWorker(self.get_queue_dic()) # 将queue传进去,与datahandle那个线程交互 self.add_monitor( CpuMonitor(self.serialnum, self.packages, self.frequency, self.timeout)) self.add_monitor( MemMonitor(self.serialnum, self.packages, self.frequency, self.timeout)) self.add_monitor( TrafficMonitor(self.serialnum, self.packages[0], self.frequency, self.timeout)) # 软件方式 获取电量不准,已用硬件方案测试功耗 # self.add_monitor(PowerMonitor(self.serialnum, self.frequency,self.timeout)) self.add_monitor( FPSMonitor(self.serialnum, self.packages[0], self.frequency, self.timeout)) # 6.0以下能采集到fd数据,7.0以上没权限 if self.device.adb.get_sdk_version() <= 23: self.add_monitor( FdMonitor(self.serialnum, self.packages[0], self.frequency, self.timeout)) self.add_monitor( ThreadNumMonitor(self.serialnum, self.packages[0], self.frequency, self.timeout)) if self.config_dic["monkey"] == "true": self.add_monitor(Monkey(self.serialnum, self.packages[0])) if self.config_dic["main_activity"] and self.config_dic[ "activity_list"]: self.add_monitor( DeviceMonitor(self.serialnum, self.packages[0], self.frequency, self.config_dic["main_activity"], self.config_dic["activity_list"], RuntimeData.exit_event)) if len(self.monitors): start_time = TimeUtils.getCurrentTimeUnderline() RuntimeData.start_time = start_time if self.config_dic["save_path"]: RuntimeData.package_save_path = os.path.join( self.config_dic["save_path"], self.packages[0], start_time) else: RuntimeData.package_save_path = os.path.join( RuntimeData.top_dir, 'results', self.packages[0], start_time) FileUtils.makedir(RuntimeData.package_save_path) self.save_device_info() for monitor in self.monitors: #启动所有的monitors try: monitor.start(start_time) except Exception as e: logger.error(e) # logcat的代码可能会引起死锁,拎出来单独处理logcat try: self.logcat_monitor = LogcatMonitor( self.serialnum, self.packages[0]) # 如果有异常日志标志,才启动这个模块 if self.exceptionlog_list: self.logcat_monitor.set_exception_list( self.exceptionlog_list) self.logcat_monitor.add_log_handle( self.logcat_monitor.handle_exception) time.sleep(1) self.logcat_monitor.start(start_time) except Exception as e: logger.error(e) timeout = time_out if time_out != None else self.config_dic[ 'timeout'] endtime = time.time() + timeout while (time.time() < endtime): #吊着主线程防止线程中断 # 时间到或测试过程中检测到异常 if self.check_exit_signal_quit(): logger.error("app " + str(self.packages[0]) + " exit signal, quit!") break time.sleep(self.frequency) logger.debug("time is up,finish!!!") self.stop() # try: # datahandle.stop() # time.sleep(self.frequency*2) # # 延迟一点时间结束上报,已让数据上报完 # # report.stop() # except: # logger.debug("report or datahandle stop exception") # finally: # logger.info("time is up, end") # os._exit(0) except KeyboardInterrupt: #捕获键盘异常的事件,例如ctrl c logger.debug(" catch keyboardInterrupt, goodbye!!!") # 收尾工作 self.stop() os._exit(0) except Exception as e: logger.error("Exception in run") logger.error(e)
def _calculator_thread(self, start_time): '''处理surfaceflinger数据 ''' fps_file = os.path.join(RuntimeData.package_save_path, 'fps.csv') if self.use_legacy_method: fps_title = ['datetime', 'fps'] else: fps_title = ['datetime', "activity window", 'fps', 'jank'] try: with open(fps_file, 'a+') as df: csv.writer(df, lineterminator='\n').writerow(fps_title) if self.fps_queue: fps_file_dic = {'fps_file': fps_file} self.fps_queue.put(fps_file_dic) except RuntimeError as e: logger.exception(e) while True: try: data = self.data_queue.get() if isinstance(data, str) and data == 'Stop': break before = time.time() if self.use_legacy_method: td = data['timestamp'] - self.surface_before['timestamp'] seconds = td.seconds + td.microseconds / 1e6 frame_count = (data['page_flip_count'] - self.surface_before['page_flip_count']) fps = int(round(frame_count / seconds)) if fps > 60: fps = 60 self.surface_before = data logger.debug('FPS:%2s' % fps) tmp_list = [TimeUtils.getCurrentTimeUnderline(), fps] try: with open(fps_file, 'a+') as f: # tmp_list[0] = TimeUtils.formatTimeStamp(tmp_list[0]) csv.writer(f, lineterminator='\n').writerow(tmp_list) except RuntimeError as e: logger.exception(e) else: refresh_period = data[0] timestamps = data[1] collect_time = data[2] fps, jank = self._calculate_results( refresh_period, timestamps) logger.debug('FPS:%2s Jank:%s' % (fps, jank)) fps_list = [collect_time, self.focus_window, fps, jank] if self.fps_queue: self.fps_queue.put(fps_list) if not self.fps_queue: #为了让单个脚本运行时保存数据 try: with open(fps_file, 'a+') as f: tmp_list = copy.deepcopy(fps_list) tmp_list[0] = TimeUtils.formatTimeStamp( tmp_list[0]) csv.writer( f, lineterminator='\n').writerow(tmp_list) except RuntimeError as e: logger.exception(e) time_consume = time.time() - before delta_inter = self.frequency - time_consume if delta_inter > 0: time.sleep(delta_inter) except: logger.error( "an exception hanpend in fps _calculator_thread ,reason unkown!" ) s = traceback.format_exc() logger.debug(s) if self.fps_queue: self.fps_queue.task_done()
def save(self): pass def parse(self, file_path): '''解析 :param str file_path: 要解析数据文件的路径 ''' pass def get_fps_collector(self): '''获得fps收集器,收集器里保存着time fps jank的列表 :return: fps收集器 :rtype: SurfaceStatsCollector ''' return self.fpscollector if __name__ == '__main__': # tulanduo android8.0 api level 27 monitor = FPSMonitor('TC79SSDMO7HEY5Z9', "com.alibaba.ailabs.genie.smartapp", 1) # mate 9 android8.0 # monitor = FPSMonitor('MKJNW18226007860',"com.sankuai.meituan",2) # android8.0 Google Pixel 2 # monitor = FPSMonitor('HT7B81A05143',package_name = "com.alibaba.ailibs.genie.contacts",1) monitor.start(TimeUtils.getCurrentTimeUnderline()) time.sleep(600) monitor.stop()
class ThreadNumMonitor(object): def __init__(self, device_id, packagename, interval=1.0, timeout=24 * 60 * 60, thread_queue=None): self.device = AndroidDevice(device_id) if not packagename: packagename = self.device.adb.get_foreground_process() self.thread_package_collector = ThreadNumPackageCollector( self.device, packagename, interval, timeout, thread_queue) def start(self, start_time): self.start_time = start_time self.thread_package_collector.start(start_time) def stop(self): self.thread_package_collector.stop() def save(self): pass if __name__ == "__main__": monitor = ThreadNumMonitor("", "com.yunos.tv.alitvasr", 3) monitor.start(TimeUtils.getCurrentTime()) time.sleep(20) monitor.stop() # monitor.save()
def _logcat_thread_func(self, save_dir, process_list, params=""): '''获取logcat线程 ''' self.append_log_line_num = 0 self.file_log_line_num = 0 self.log_file_create_time = None logs = [] logger.debug("logcat_thread_func") log_is_none = 0 while self._logcat_running: try: log = self._log_pipe.stdout.readline().strip() if not isinstance(log, str): try: log = str(log,"utf8") except Exception as e: log = repr(log) logger.error('str error:'+log) logger.error(e) if log: log_is_none = 0 # logger.debug(log) logs.append(log) # if self._log_pipe.poll() != None: # logger.debug('process:%s have exited' % self._log_pipe.pid) # if self._logcat_running : # self._log_pipe = self.run_shell_cmd('logcat ' + params, sync=False) # else : # break for _handle in self._logcat_handle: try: _handle(log) except Exception as e: logger.error("an exception happen in logcat handle log , reason unkown!, e:") logger.error(e) self.append_log_line_num = self.append_log_line_num + 1 self.file_log_line_num = self.file_log_line_num + 1 # if self.append_log_line_num > 1000: if self.append_log_line_num > 100: if not self.log_file_create_time: self.log_file_create_time = TimeUtils.getCurrentTimeUnderline() logcat_file = os.path.join(save_dir, 'logcat_%s.log' % self.log_file_create_time) self.append_log_line_num = 0 self.save(logcat_file, logs) logs = [] # 新建文件 if self.file_log_line_num > 600000: # if self.file_log_line_num > 200: self.file_log_line_num = 0 self.log_file_create_time = TimeUtils.getCurrentTimeUnderline() logcat_file = os.path.join(save_dir, 'logcat_%s.log' % self.log_file_create_time) self.save(logcat_file, logs) logs = [] else: log_is_none = log_is_none + 1 if log_is_none % 1000 == 0: logger.info("log is none") self._log_pipe = self.run_shell_cmd('logcat -v threadtime ' + params, sync=False) except: logger.error("an exception hanpend in logcat thread, reason unkown!") s = traceback.format_exc() logger.debug(s)
def dump_native_heap(self, package, save_path): native_heap_file = "/data/local/tmp/%s_native_heap_%s.txt" % (package, TimeUtils.getCurrentTimeUnderline()) self.run_shell_cmd("am dumpheap -n %s %s" % (package, native_heap_file))
def dumpheap(self, package, save_path): heapfile = "/data/local/tmp/%s_dumpheap_%s.hprof" % (package, TimeUtils.getCurrentTimeUnderline()) self.run_shell_cmd("am dumpheap %s %s" % (package, heapfile)) time.sleep(10) self.pull_file(heapfile,save_path)