def _monkey_thread_func(self,save_dir): '''获取monkey线程,保存monkey日志,monkey Crash日志暂不处理,后续有需要再处理 ''' self.append_log_line_num = 0 self.file_log_line_num = 0 self.log_file_create_time = None log_is_none = 0 logs = [] logger.debug("monkey_thread_func") if RuntimeData.start_time is None: RuntimeData.start_time = TimeUtils.getCurrentTime() while self.running: try: log = self._log_pipe.stdout.readline().strip() if not isinstance(log, str): # 先编码为unicode try: log = str(log, "utf8") except Exception as e: log = repr(log) logger.error('str error:' + log) logger.error(e) if log: logs.append(log) self.append_log_line_num = self.append_log_line_num + 1 self.file_log_line_num = self.file_log_line_num + 1 # if self.append_log_line_num > 1000: if self.append_log_line_num > 100: if not self.log_file_create_time: self.log_file_create_time = TimeUtils.getCurrentTimeUnderline() log_file = os.path.join(save_dir, 'monkey_%s.log' % self.log_file_create_time) self.append_log_line_num = 0 # 降低音量,避免音量过大,导致语音指令失败 self.device.adb.run_shell_cmd("input keyevent 25") self.save(log_file, logs) logs = [] # 新建文件 if self.file_log_line_num > 600000: # if self.file_log_line_num > 200: self.file_log_line_num = 0 self.log_file_create_time = TimeUtils.getCurrentTimeUnderline() log_file = os.path.join(save_dir, 'monkey_%s.log' % self.log_file_create_time) self.save(log_file, logs) logs = [] else: log_is_none = log_is_none + 1 if log_is_none % 1000 == 0: logger.info("log is none") if not self.device.adb.is_process_running("com.android.commands.monkey") and self.running: self.device.adb.kill_process("com.android.commands.monkey") self._log_pipe = self.device.adb.run_shell_cmd(self.monkey_cmd, sync=False) except: logger.error("an exception hanpend in monkey thread, reason unkown!") s = traceback.format_exc() logger.debug(s)
def __init__(self, csv_dir, packages=[]): os.chdir(csv_dir) # 需要画曲线的csv文件名 self.summary_csf_file = { "cpuinfo.csv": { "table_name": "pid_cpu", "x_axis": "datatime", "y_axis": "%", "values": ["pid_cpu%", "total_pid_cpu%"] }, "meminfo.csv": { "table_name": "pid_pss", "x_axis": "datatime", "y_axis": "mem(MB)", "values": ["pid_pss(MB)", "total_pss(MB)"] }, "pid_change.csv": { "table_name": "pid", "x_axis": "datatime", "y_axis": "pid_num", "values": ["pid"] }, } self.packages = packages if len(self.packages) > 0: for package in self.packages: pss_detail_dic = { "table_name": "pss_detail", "x_axis": "datatime", "y_axis": "mem(MB)", "values": ["pss", "java_heap", "native_heap", "system"] } # 文件名太长会导致写excel失败 self.summary_csf_file["pss_%s.csv" % package.split(".")[-1].replace( ":", "_")] = pss_detail_dic logger.debug(self.packages) logger.debug(self.summary_csf_file) logger.info('create report for %s' % csv_dir) file_names = self.filter_file_names(csv_dir) logger.debug('%s' % file_names) if file_names: book_name = 'summary_%s.xlsx' % TimeUtils.getCurrentTimeUnderline() excel = Excel(book_name) for file_name in file_names: logger.debug('get csv %s to excel' % file_name) values = self.summary_csf_file[file_name] excel.csv_to_xlsx(file_name, values["table_name"], values["x_axis"], values["y_axis"], values["values"]) logger.info('wait to save %s' % book_name) excel.save()
def run(self, time_out=None): self.clear_heapdump() # objgraph.show_growth() # 对设备连接情况的检查 if not self.serialnum: # androiddevice 没传 serialnum,默认执行adb shell logger.info( "serialnum in config file is null,default get connected phone") is_device_connect = False for i in range(0, 5): if self.device.adb.is_connected(self.serialnum): is_device_connect = True break else: logger.error("device not found:" + self.serialnum) time.sleep(2) if not is_device_connect: logger.error("after 5 times check,device not found:" + self.serialnum) return # 对是否安装被测app的检查 只在最开始检查一次 if not self.device.adb.is_app_installed(self.packages[0]): logger.error("test app not installed:" + self.packages[0]) return try: #初始化数据处理的类,将没有消息队列传递过去,以便获取数据,并处理 # datahandle = DataWorker(self.get_queue_dic()) # 将queue传进去,与datahandle那个线程交互 self.add_monitor( CpuMonitor(self.serialnum, self.packages, self.frequency, self.timeout)) self.add_monitor( MemMonitor(self.serialnum, self.packages, self.frequency, self.timeout)) self.add_monitor( TrafficMonitor(self.serialnum, self.packages[0], self.frequency, self.timeout)) # 软件方式 获取电量不准,已用硬件方案测试功耗 # self.add_monitor(PowerMonitor(self.serialnum, self.frequency,self.timeout)) self.add_monitor( FPSMonitor(self.serialnum, self.packages[0], self.frequency, self.timeout)) # 6.0以下能采集到fd数据,7.0以上没权限 if self.device.adb.get_sdk_version() <= 23: self.add_monitor( FdMonitor(self.serialnum, self.packages[0], self.frequency, self.timeout)) self.add_monitor( ThreadNumMonitor(self.serialnum, self.packages[0], self.frequency, self.timeout)) if self.config_dic["monkey"] == "true": self.add_monitor(Monkey(self.serialnum, self.packages[0])) if self.config_dic["main_activity"] and self.config_dic[ "activity_list"]: self.add_monitor( DeviceMonitor(self.serialnum, self.packages[0], self.frequency, self.config_dic["main_activity"], self.config_dic["activity_list"], RuntimeData.exit_event)) if len(self.monitors): start_time = TimeUtils.getCurrentTimeUnderline() RuntimeData.start_time = start_time if self.config_dic["save_path"]: RuntimeData.package_save_path = os.path.join( self.config_dic["save_path"], self.packages[0], start_time) else: RuntimeData.package_save_path = os.path.join( RuntimeData.top_dir, 'results', self.packages[0], start_time) FileUtils.makedir(RuntimeData.package_save_path) self.save_device_info() for monitor in self.monitors: #启动所有的monitors try: monitor.start(start_time) except Exception as e: logger.error(e) # logcat的代码可能会引起死锁,拎出来单独处理logcat try: self.logcat_monitor = LogcatMonitor( self.serialnum, self.packages[0]) # 如果有异常日志标志,才启动这个模块 if self.exceptionlog_list: self.logcat_monitor.set_exception_list( self.exceptionlog_list) self.logcat_monitor.add_log_handle( self.logcat_monitor.handle_exception) time.sleep(1) self.logcat_monitor.start(start_time) except Exception as e: logger.error(e) timeout = time_out if time_out != None else self.config_dic[ 'timeout'] endtime = time.time() + timeout while (time.time() < endtime): #吊着主线程防止线程中断 # 时间到或测试过程中检测到异常 if self.check_exit_signal_quit(): logger.error("app " + str(self.packages[0]) + " exit signal, quit!") break time.sleep(self.frequency) logger.debug("time is up,finish!!!") self.stop() # try: # datahandle.stop() # time.sleep(self.frequency*2) # # 延迟一点时间结束上报,已让数据上报完 # # report.stop() # except: # logger.debug("report or datahandle stop exception") # finally: # logger.info("time is up, end") # os._exit(0) except KeyboardInterrupt: #捕获键盘异常的事件,例如ctrl c logger.debug(" catch keyboardInterrupt, goodbye!!!") # 收尾工作 self.stop() os._exit(0) except Exception as e: logger.error("Exception in run") logger.error(e)
def _logcat_thread_func(self, save_dir, process_list, params=""): '''获取logcat线程 ''' self.append_log_line_num = 0 self.file_log_line_num = 0 self.log_file_create_time = None logs = [] logger.debug("logcat_thread_func") log_is_none = 0 while self._logcat_running: try: log = self._log_pipe.stdout.readline().strip() if not isinstance(log, str): try: log = str(log,"utf8") except Exception as e: log = repr(log) logger.error('str error:'+log) logger.error(e) if log: log_is_none = 0 # logger.debug(log) logs.append(log) # if self._log_pipe.poll() != None: # logger.debug('process:%s have exited' % self._log_pipe.pid) # if self._logcat_running : # self._log_pipe = self.run_shell_cmd('logcat ' + params, sync=False) # else : # break for _handle in self._logcat_handle: try: _handle(log) except Exception as e: logger.error("an exception happen in logcat handle log , reason unkown!, e:") logger.error(e) self.append_log_line_num = self.append_log_line_num + 1 self.file_log_line_num = self.file_log_line_num + 1 # if self.append_log_line_num > 1000: if self.append_log_line_num > 100: if not self.log_file_create_time: self.log_file_create_time = TimeUtils.getCurrentTimeUnderline() logcat_file = os.path.join(save_dir, 'logcat_%s.log' % self.log_file_create_time) self.append_log_line_num = 0 self.save(logcat_file, logs) logs = [] # 新建文件 if self.file_log_line_num > 600000: # if self.file_log_line_num > 200: self.file_log_line_num = 0 self.log_file_create_time = TimeUtils.getCurrentTimeUnderline() logcat_file = os.path.join(save_dir, 'logcat_%s.log' % self.log_file_create_time) self.save(logcat_file, logs) logs = [] else: log_is_none = log_is_none + 1 if log_is_none % 1000 == 0: logger.info("log is none") self._log_pipe = self.run_shell_cmd('logcat -v threadtime ' + params, sync=False) except: logger.error("an exception hanpend in logcat thread, reason unkown!") s = traceback.format_exc() logger.debug(s)
def _run_cmd_once(self, cmd, *argv, **kwds): '''执行一次adb命令:cmd :param str cmd: 命令字符串 :param list argv: 可变参数 :param dict kwds: 可选关键字参数 (超时/异步) :return: 执行adb命令的子进程或执行的结果 :rtype: Popen or str ''' import locale import codecs if self._device_id: cmdlet = [self._adb_path, '-s', self._device_id, cmd] else: cmdlet = [self._adb_path, cmd] for i in range(len(argv)): arg = argv[i] if not isinstance(argv[i], str): arg = arg.decode('utf8') cmdlet.append(arg) # logger.debug('ADB cmd:' + " ".join(cmdlet)) # logger.debug(cmdlet) cmdStr = " ".join(cmdlet) logger.debug(cmdStr) process = None # windows上 不要传cmdStr 目录有空格,会报错 # if ADB.os_name == "Windows": # process = subprocess.Popen(cmdlet, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE,shell=True) # # mac上传list会报错:Android Debug Bridge version # else: # windows ["adb devices"] 提示没有命令 ,改为str执行 process = subprocess.Popen(cmdStr, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) if "sync" in kwds and kwds['sync'] == False: # 异步执行命令,不等待结果,返回该子进程对象 return process before = time.time() timeout = 10 if "timeout" in kwds: timeout = kwds['timeout'] if timeout != None and timeout > 0: # timeout = None 或者小于等于0时,一直等待执行结果 threading.Thread(None, self._timer, (process, timeout)) (out, error) = process.communicate() # 执行错误 mac out无输出 error有输出 返回值非0 # 执行错误 windows out有输出 error没有输出,返回值0 if process.poll() != 0: # 返回码为非0,表示命令未执行成功返回 # logger.logError("adb执行出错或超时,出错命令是:\n%s"%cmdlet) if error and len(error) != 0: logger.debug("adb error info:\n%s" % error) if "no devices/emulators found" in str(out) or "no devices/emulators found" in str(error): logger.error("no devices/emulators found,please reconnect phone,make sure adb shell normal") return "" # 退出整个进程 if "killing" in str(out) or "killing" in str(error): logger.error( "adb 5037 port is occupied,please stop the process occupied 5037 port,make sure adb devices normal") return "" if "device not found" in str(out) or "device not found" in str(error): logger.error("device not found,please reconnect phone,make sure adb devices normal") self.before_connect = False self.after_connect = False return "" if "offline" in str(out) or "offline" in str(error): logger.error("device offline,please reconnect phone,make sure adb devices normal") return "" if "more than one" in str(out) or "more than one" in str(error): logger.error("more than one device,please input device serialnum!") # sys.exit(1) if "Android Debug Bridge version" in str(out) or "Android Debug Bridge version" in str(error): logger.error("adb cmd error!:" + out) # sys.exit(1) if str(out, "utf-8") == '': # logger.debug("out is empty ,use error") out = error self.after_connect = True after = time.time() time_consume = after - before logger.info(cmdStr + " time consume: " + str(time_consume)) if not isinstance(out, str): try: out = str(out, "utf8") except Exception as e: out = repr(out) return out.strip()
def _collect_memory_thread(self, start_time): end_time = time.time() + self._timeout mem_list_titile = ["datatime", "total_ram(MB)", "free_ram(MB)"] pid_list_titile = ["datatime"] pss_detail_titile = [ "datatime", "package", "pid", "pss", "java_heap", "native_heap", "system" ] for i in range(0, len(self.packages)): mem_list_titile.extend(["package", "pid", "pid_pss(MB)"]) pid_list_titile.extend(["package", "pid"]) if len(self.packages) > 1: mem_list_titile.append("total_pss(MB)") mem_file = os.path.join(RuntimeData.package_save_path, 'meminfo.csv') pid_file = os.path.join(RuntimeData.package_save_path, 'pid_change.csv') for package in self.packages: pss_detail_file = os.path.join( RuntimeData.package_save_path, 'pss_%s.csv' % package.split(".")[-1].replace(":", "_")) with open(pss_detail_file, 'a+') as df: csv.writer(df, lineterminator='\n').writerow(pss_detail_titile) try: with open(mem_file, 'a+') as df: csv.writer(df, lineterminator='\n').writerow(mem_list_titile) if self.mem_queue: mem_file_dic = {'mem_file': mem_file} self.mem_queue.put(mem_file_dic) with open(pid_file, 'a+') as df: csv.writer(df, lineterminator='\n').writerow(pid_list_titile) except RuntimeError as e: logger.error(e) starttime_stamp = TimeUtils.getTimeStamp(start_time, "%Y_%m_%d_%H_%M_%S") old_package_pid_pss_list = [] dumpsys_mem_times = 0 # D系统上会报错 System server has no access to file context # hprof_path = "/sdcard/hprof" hprof_path = "/data/local/tmp" self.device.adb.run_shell_cmd("mkdir " + hprof_path) # sdcard 卡目录下dump需要打开这个开关 self.device.adb.run_shell_cmd("setenforce 0") first_dump = True while not self._stop_event.is_set() and time.time() < end_time: try: before = time.time() logger.debug( "-----------into _collect_mem_thread loop, thread is : " + str(threading.current_thread().name)) collection_time = time.time() # # 获取主进程的详细信息 for package in self.packages: mem_pck_snapshot = self._dumpsys_process_meminfo(package) if 0 == mem_pck_snapshot.totalPSS: logger.error("package total pss is 0:%s" % package) continue pss_detail_file = os.path.join( RuntimeData.package_save_path, 'pss_%s.csv' % package.split(".")[-1].replace(":", "_")) pss_detail_list = [ TimeUtils.formatTimeStamp(collection_time), package, mem_pck_snapshot.pid, mem_pck_snapshot.totalPSS, mem_pck_snapshot.javaHeap, mem_pck_snapshot.nativeHeap, mem_pck_snapshot.system ] with open(pss_detail_file, 'a+') as pss_writer: writer_p = csv.writer(pss_writer, lineterminator='\n') writer_p.writerow(pss_detail_list) # 写到pss_detail表格中 # 手机每5分钟 dumpheap一次 if (before - starttime_stamp) > 300 or first_dump: # 先清理hprof文件 filelist = self.device.adb.list_dir(hprof_path) if filelist: for file in filelist: for package in self.packages: if package in file: self.device.adb.delete_file(hprof_path + "/" + file) # if (before - starttime_stamp) % 60 < self._interval and "D" in self.device.adb.get_system_version(): for package in self.packages: self.device.adb.dumpheap(package, RuntimeData.package_save_path) starttime_stamp = before # self.device.adb.run_shell_cmd("kill -10 %s"%str(mem_pck_snapshot.pid)) # dumpsys meminfo 耗时长,可能会导致system server cpu占用变高,降低采集频率 dumpsys_mem_times = dumpsys_mem_times + 1 # 10倍率frequency dumpsys meminfo一次 if dumpsys_mem_times % 10 == 0 or first_dump: mem_device_snapshot = self._dumpsys_meminfo() # 如果没有采集到dumpsys meminfo的信息,正常情况totalmem不可能为0 if mem_device_snapshot == None or not mem_device_snapshot.package_pid_pss_list or mem_device_snapshot.totalmem == 0: logger.error("mem_device_snapshot is none") # 如果获取不到结果,继续延长采集间隔 dumpsys_mem_times = dumpsys_mem_times - 1 continue first_dump = False logger.debug("current time: " + TimeUtils.getCurrentTime() + ", processname: " + ",total pss:" + str(mem_device_snapshot.total_pss)) logger.debug("collection time in meminfo is : " + TimeUtils.getCurrentTime()) gather_list = [ TimeUtils.formatTimeStamp(collection_time), mem_device_snapshot.totalmem, mem_device_snapshot.freemem ] pid_list = [TimeUtils.formatTimeStamp(collection_time)] pid_change = False for i in range(0, len(self.packages)): if len(mem_device_snapshot.package_pid_pss_list ) == len(self.packages): gather_list.extend([ mem_device_snapshot.package_pid_pss_list[i] ["package"], mem_device_snapshot.package_pid_pss_list[i] ["pid"], mem_device_snapshot.package_pid_pss_list[i] ["pss"] ]) if not old_package_pid_pss_list: old_package_pid_pss_list = mem_device_snapshot.package_pid_pss_list pid_change = True else: for i in range(0, len(self.packages)): package = mem_device_snapshot.package_pid_pss_list[ i]["package"] if mem_device_snapshot.package_pid_pss_list[i]["pid"] and \ old_package_pid_pss_list[i]["pid"]!=mem_device_snapshot.package_pid_pss_list[i]["pid"]: pid_change = True # 确保上次pid也有 if old_package_pid_pss_list[i]["pid"]: if package and package in RuntimeData.config_dic[ "pid_change_focus_package"]: # 确保有tombstones文件才提单 self.device.adb.pull_file( "/data/vendor/tombstones", RuntimeData.package_save_path) if pid_change: old_package_pid_pss_list = mem_device_snapshot.package_pid_pss_list for i in range(0, len(self.packages)): if len(old_package_pid_pss_list) == len( self.packages): pid_list.extend([ old_package_pid_pss_list[i]["package"], old_package_pid_pss_list[i]["pid"] ]) try: with open(pid_file, 'a+') as pid_writer: writer_p = csv.writer(pid_writer, lineterminator='\n') writer_p.writerow(pid_list) logger.debug("write to file:" + pid_file) logger.debug(pid_list) except RuntimeError as e: logger.error(e) if len(self.packages) > 1: gather_list.append(mem_device_snapshot.total_pss) if self.mem_queue: gather_list[0] = collection_time self.mem_queue.put(gather_list) if not self.mem_queue: #为了本地单个文件运行 try: with open(mem_file, 'a+') as mem_writer: writer_p = csv.writer(mem_writer, lineterminator='\n') writer_p.writerow(gather_list) logger.debug("write to file:" + mem_file) logger.debug(gather_list) except RuntimeError as e: logger.error(e) after = time.time() time_consume = after - before delta_inter = self._interval - time_consume logger.info("time consume for meminfos: " + str(time_consume)) if delta_inter > 0: time.sleep(delta_inter) except: logger.error( "an exception hanpend in meminfo thread, reason unkown!") s = traceback.format_exc() logger.debug(s) if self.mem_queue: self.mem_queue.task_done() logger.debug("stop event is set or timeout")
def _handle_data_thread(self): ''' 该方法在独立线程中运行,是一个消费者,负责处理所有的生产者搜集的数据 :return: ''' time.sleep(1) while not self._stop_event.is_set(): try: # 消息队列会一直阻塞直到 取到数据为止,如果生产者的线程发生了异常,则需要生产者线程会往对应的queue发出一个task_done以结束队列 # power 数据处理 self.perf_data = { "task_id": "", "activity": [], 'launch_time': [], 'cpu': [], "mem": [], 'traffic': [], "fluency": [], 'power': [], "fd": [], "thread": [] } try: power_data = self.power_queue.get() if isinstance(power_data, list): self.timestamp = power_data[0] logger.debug("dataworker logger timestamp: " + str(self.timestamp)) logger.info("now collecting data") self._get_power_save(power_data, self.timestamp) except queue.Empty: logger.debug("dataworker power queue Empty") # traffic数据的处理 try: traffic_data = self.traffic_queue.get(timeout=2) # logger.debug(traffic_data) self._get_traffic_save(traffic_data, self.timestamp) except queue.Empty: logger.debug("dataworker traffic queue Empty") #fps数据的处理 try: fps_data = self.fps_queue.get(timeout=2) self._get_fps_save(fps_data, self.timestamp) except queue.Empty: logger.debug("dataworker fps queue Empty") #cpu数据处理 try: cpu_data = self.cpu_queue.get(timeout=2) # logger.debug(cpu_data) self._get_cpu_save(cpu_data, self.timestamp) except queue.Empty: logger.debug("dataworker cpu queue Empty") #mem 数据处理 try: mem_data = self.mem_queue.get(timeout=2) # logger.debug(mem_data) self._get_mem_save(mem_data, self.timestamp) except queue.Empty: logger.debug("dataworker mem queue Empty") try: fd_data = self.fd_queue.get(timeout=2) # logger.debug("fd_data") # logger.debug(fd_data) self._get_fd_save(fd_data, self.timestamp) except queue.Empty: logger.debug("dataworker fd_data queue Empty") try: thread_data = self.thread_queue.get(timeout=2) # logger.debug("thread_data") # logger.debug(thread_data) self._get_thread_save(thread_data, self.timestamp) except queue.Empty: logger.debug("dataworker thread queue Empty") try: activity_data = self.activity_queue.get(timeout=2) self._get_activity_save(activity_data, self.timestamp) except queue.Empty: logger.debug("dataworker activity queue Empty") except Exception as e: logger.error(e) s = traceback.format_exc() logger.debug(s) # 将堆栈信息打印到log中 #logger.info("dataworker interval: " + str(self.interval)) # 封装后最后统一放入 perf_queue中 logger.debug("perf_str put in queue:" + json.dumps(self.perf_data)) # 没有启动上报线程,避免无限增长 # perf_queue.put(self.perf_data) time.sleep(self.interval)