def start(self, interval, start_time): self.interval = interval self.starttime = start_time self.dataworker_thread = threading.Thread( target=self._handle_data_thread) self.dataworker_thread.start() logger.debug("DataWorker started...")
def killOccupy5037Process(): if ADB.get_os_name() == "Windows": sub = subprocess.Popen('netstat -ano|findstr \"5037\"', stdout=subprocess.PIPE, shell=True) ret = sub.stdout.read() sub.wait() if not ret: logger.debug("netstat is empty") return lines = ret.splitlines() for line in lines: if "LISTENING" in line: logger.debug(line) pid = line.split()[-1] sub = subprocess.Popen('tasklist |findstr %s' % pid, stdout=subprocess.PIPE, shell=True) ret = sub.stdout.read() sub.wait() process = ret.split()[0] logger.debug("pid:%s ,process:%s occupy 5037 port" % (pid, process)) # DDMS会用到adb 杀了adb会导致 IDE调试或控制台可能不正常,后面需要改环境变量 subprocess.Popen("taskkill /T /F /PID %s" % pid, stdout=subprocess.PIPE, shell=True) logger.debug("kill process %s" % process) break else: logger.debug("don't have process occupy 5037")
def _collect_thread_num_thread(self, start_time): end_time = time.time() + self._timeout thread_list_titile = ("datatime", "packagename", "pid", "thread_num") thread_num_file = os.path.join(RuntimeData.package_save_path, 'thread_num.csv') try: with open(thread_num_file, 'a+') as df: csv.writer(df, lineterminator='\n').writerow(thread_list_titile) if self.thread_queue: thread_file_dic = {'thread_file': thread_num_file} self.thread_queue.put(thread_file_dic) except RuntimeError as e: logger.error(e) while not self._stop_event.is_set() and time.time() < end_time: try: before = time.time() logger.debug( "-----------into _collect_thread_num_thread loop, thread is : " + str(threading.current_thread().name)) # 获取pakagename的thread num信息 thread_pck_info = self.get_process_thread_num(self.packagename) logger.debug(thread_pck_info) current_time = TimeUtils.getCurrentTime() if not thread_pck_info: continue else: logger.debug("current time: " + current_time + ", processname: " + thread_pck_info[1] + ", pid: " + str(thread_pck_info[2]) + " thread num: " + str(thread_pck_info[3])) if self.thread_queue: self.thread_queue.put(thread_pck_info) if not self.thread_queue: #为了本地单个文件运行 try: with open(thread_num_file, 'a+') as thread_writer: writer_p = csv.writer(thread_writer, lineterminator='\n') thread_pck_info[0] = current_time writer_p.writerow(thread_pck_info) except RuntimeError as e: logger.error(e) after = time.time() time_consume = after - before delta_inter = self._interval - time_consume logger.debug("time_consume for thread num infos: " + str(time_consume)) if delta_inter > 0: time.sleep(delta_inter) except: logger.error( "an exception hanpend in thread num thread, reason unkown!" ) s = traceback.format_exc() logger.debug(s) if self.thread_queue: self.thread_queue.task_done()
def check_config_option(self, config_dic, parse, section, option): if parse.has_option(section, option): try: config_dic[option] = parse.get(section, option) if option == 'frequency': config_dic[option] = (int)(parse.get(section, option)) if option == 'timeout':#timeout 的单位是分钟 config_dic[option] = (int)(parse.get(section, option))*60 if option in ["exceptionlog" ,"phone_log_path","space_size_check_path","package","pid_change_focus_package", "watcher_users","main_activity","activity_list"]: if option == "activity_list" or option == "main_activity": config_dic[option] = parse.get(section, option).strip().replace("\n","").split(";") else: config_dic[option] = parse.get(section, option).split(";") except:#配置项中数值发生错误 if option != 'serialnum': logger.debug("config option error:"+option) self._config_error() else: config_dic[option] = '' else:#配置项没有配置 if option not in ['serialnum',"main_activity","activity_list","pid_change_focus_package","shell_file"]: logger.debug("config option error:" + option) self._config_error() else: config_dic[option] = '' return config_dic
def update_launch_list(self, content, timestamp): # if self.packagename in content[1]: self.launch_list.append(content) tmp_file = os.path.join(RuntimeData.package_save_path, 'launch_logcat.csv') perf_data = { "task_id": "", 'launch_time': [], 'cpu': [], "mem": [], 'traffic': [], "fluency": [], 'power': [], } dic = { "time": timestamp, "act_name": content[1], "this_time": content[2], "total_time": content[3], "launch_type": content[4] } perf_data['launch_time'].append(dic) # perf_queue.put(perf_data) with open(tmp_file, "a+") as f: csvwriter = csv.writer(f, lineterminator='\n') #这种方式可以去除csv的空行 logger.debug("save launchtime data to csv: " + str(self.launch_list)) csvwriter.writerows(self.launch_list) del self.launch_list[:]
def _parse(self): sp_lines = self.source.split('\n') for line in sp_lines: if self.uid and self.uid in line: # logger.debug(" target uid : "+str(self.uid)) tart_list = line.split() tag = tart_list[2] # logger.debug(" tag is: " +tag) if tag == '0x0': #tag即acct_tag_hex这一列,默认是0,表示与这个uid关联的流量,有时候用户需要在自己的uid内添加一个其他 # tag表示这个模块中的子模块的流量,就可以通过setThreadTag # logger.debug(" tart_list: " + str(tart_list)) self.rx_uid_bytes += int( tart_list[5]) #不区分网络的类型,直接算总和,wifi和mobile, lo数据的总和 # logger.debug(self.rx_uid_bytes) self.rx_uid_packets += int(tart_list[6]) self.tx_uid_bytes += int(tart_list[7]) self.tx_uid_packets += int(tart_list[8]) self.total_uid_bytes = self.tx_uid_bytes + self.rx_uid_bytes self.total_uid_packets = self.tx_uid_packets + self.rx_uid_packets if (tart_list[1] == 'lo'): #对应着iface这列,表示本地流量 self.lo_uid_bytes += int(tart_list[5]) + int( tart_list[7]) # logger.debug(" lo_uid_bytes: " + str(self.lo_uid_bytes)) if (int(tart_list[4]) == 0): #统计后台流量 self.bg_bytes += int(tart_list[5]) + int(tart_list[7]) # logger.debug(" backgroud data : " +str(self.bg_bytes)) elif (int(tart_list[4]) == 1): #统计前台流量 self.fg_bytes += int(tart_list[5]) + int(tart_list[7]) # logger.debug(" fg data: " +str(self.fg_bytes)) logger.debug(" total uid bytes : " + str(self.total_uid_bytes))
def list_dir_between_time(self, dir_path, start_time, end_time): '''列取目录下 起止时间点之间的文件 start_time end_time 时间戳 返回文件绝对路径 列表 ''' # ls - l # -rwxrwx--- 1 root root 19897899 2018-12-27 18:02 com.alibaba.ailabs.ar.fireeye2_dumpheap_2018_12_27_18_02_52.hprof result = self.run_shell_cmd('ls -l %s' % dir_path) if not result: return "" result = result.replace('\r\r\n', '\n') if 'No such file or directory' in result: logger.error('文件(夹) %s 不存在' % dir_path) file_list = [] re_time = re.compile(r'\S*\s+(\d+-\d+-\d+\s+\d+:\d+)\s+\S+') for line in result.split('\n'): items = line.split() match = re_time.search(line) if match: last_modify_time = match.group(1) logger.debug(last_modify_time) last_modify_timestamp = TimeUtils.getTimeStamp(last_modify_time, "%Y-%m-%d %H:%M") # logger.debug(last_modify_timestamp) if start_time < last_modify_timestamp and last_modify_timestamp < end_time: logger.debug("append file:" + items[-1]) file_list.append('%s/%s' % (dir_path, items[-1])) return file_list
def get_data_from_threadstart(self, traffic_snapshot): # 获取从当前线程开始的流量值 traffic_snapshot.total_uid_bytes = traffic_snapshot.total_uid_bytes - self.traffic_init_dic[ 'total'] if (traffic_snapshot.total_uid_bytes - self.traffic_init_dic['total']) >= 0 else 0 traffic_snapshot.total_uid_packets = traffic_snapshot.total_uid_packets - self.traffic_init_dic[ 'total_packets'] if ( traffic_snapshot.total_uid_packets - self.traffic_init_dic['total_packets']) >= 0 else 0 traffic_snapshot.rx_uid_bytes = traffic_snapshot.rx_uid_bytes - self.traffic_init_dic[ 'rx'] if (traffic_snapshot.rx_uid_bytes - self.traffic_init_dic['rx']) >= 0 else 0 traffic_snapshot.rx_uid_packets = traffic_snapshot.rx_uid_packets - self.traffic_init_dic[ 'rx_packets'] if (traffic_snapshot.rx_uid_packets - self.traffic_init_dic['rx_packets']) >= 0 else 0 traffic_snapshot.tx_uid_bytes = traffic_snapshot.tx_uid_bytes - self.traffic_init_dic[ 'tx'] if (traffic_snapshot.tx_uid_bytes - self.traffic_init_dic['tx']) >= 0 else 0 traffic_snapshot.tx_uid_packets = traffic_snapshot.tx_uid_packets - self.traffic_init_dic[ 'tx_packets'] if (traffic_snapshot.tx_uid_packets - self.traffic_init_dic['tx_packets']) >= 0 else 0 traffic_snapshot.fg_bytes = traffic_snapshot.fg_bytes - self.traffic_init_dic[ 'fg'] if (traffic_snapshot.fg_bytes - self.traffic_init_dic['fg']) >= 0 else 0 traffic_snapshot.bg_bytes = traffic_snapshot.bg_bytes - self.traffic_init_dic[ 'bg'] if (traffic_snapshot.bg_bytes - self.traffic_init_dic['bg']) >= 0 else 0 traffic_snapshot.lo_uid_bytes = traffic_snapshot.lo_uid_bytes - self.traffic_init_dic[ 'lo'] if (traffic_snapshot.lo_uid_bytes - self.traffic_init_dic['lo']) >= 0 else 0 logger.debug(traffic_snapshot) return traffic_snapshot
def get_focus_activity(self): ''' 通过dumpsys window windows获取activity名称 window名? ''' activity_name = '' activity_line = '' activity_line_split = '' dumpsys_result = self.run_shell_cmd('dumpsys window windows') dumpsys_result_list = dumpsys_result.split('\n') for line in dumpsys_result_list: if line.find('mCurrentFocus') != -1: activity_line = line.strip() # Android # Android 8.0 mCurrentFocus的输出行 # mCurrentFocus=Window{2f4cb8b u0 com.google.android.apps.photos/com.google.android.apps.photos.home.HomeActivity} if activity_line: activity_line_split = activity_line.split(' ') else: return activity_name logger.debug('dumpsys window windows命令activity_line_split结果: %s' % activity_line_split) if len(activity_line_split) > 1: if activity_line_split[1] == 'u0': activity_name = activity_line_split[2].rstrip('}') else: activity_name = activity_line_split[1] return activity_name
def _dumpsys_process_meminfo(self, process): ''' dump 进程详细内存 耗时 1s以内 :param process: :return: ''' time_old = time.time() out = self.device.adb.run_shell_cmd('dumpsys meminfo %s' % process) # self.num = self.num + 1 # if self.num % 10 == 0: #避免:在windows 无法创建文件名,不能有冒号: process_rename = process.replace(":", "_") meminfo_file = os.path.join(RuntimeData.package_save_path, 'dumpsys_meminfo_%s.txt' % process_rename) with open(meminfo_file, "a+") as writer: writer.write(TimeUtils.getCurrentTime() + " dumpsys meminfo package info:\n") if out: writer.write(out + "\n\n") # self.num = 0 passedtime = time.time() - time_old #测试meminfo这个命令的耗时,执行的时长在400多ms logger.debug("dumpsys meminfo package time consume:" + str(passedtime)) out.replace('\r', '') return MemInfoPackage(dump=out)
def _parse(self): sp_lines = self.source.split('\n') for line in sp_lines: # wlan0: 1241508864 840739 0 0 0 0 0 7 7149177 73416 0 6 0 0 0 0 # 获取其中 接受流量1241508864 发送流量7149177 if "wlan0:" in line: items = line.split() self.wifi_rx = int(items[1]) self.wifi_tx = int(items[9]) self.wifi_total = self.wifi_rx + self.wifi_tx logger.debug("wifi_rx : " + items[1] + " wifi_tx : " + items[9] + " total wifi:" + str(self.wifi_total)) # 移动 3 4 5G 流量 # rmnet0: 362133448 298441 0 0 0 0 0 0 10641124 91012 0 0 0 0 0 0 if "rmnet0:" in line: items = line.split() self.mobile_rx = int(items[1]) self.mobile_tx = int(items[9]) self.mobile_total = self.wifi_rx + self.wifi_tx logger.debug("mobile_rx : " + items[1] + " mobile_tx : " + items[9] + " total mobile:" + str(self.mobile_total)) self.rx = self.wifi_rx + self.mobile_rx self.tx = self.wifi_tx + self.mobile_tx self.total = self.wifi_total + self.mobile_total
def start(self, start_time): ''' 启动一个cpu监控器,监控cpu信息 :return: ''' self.cpu_collector.start(start_time) logger.debug("INFO: CpuMonitor has started...")
def _get_activity_save(self, activity_data, timestamp): if self.first_time: activity_title = ("datetime", "current_activity") self.first_time = False self.activity_file = os.path.join(RuntimeData.package_save_path, 'current_activity.csv') try: with open(self.activity_file, 'a+') as af: csv.writer(af, lineterminator='\n').writerow(activity_title) except Exception as e: logger.error("file not found: " + str(self.activity_file)) else: try: activity_data[0] = timestamp dic = { "time": activity_data[0] * 1000, "name": activity_data[1] } self.perf_data['activity'].append(dic) with open(self.activity_file, 'a+') as writer: if isinstance(activity_data[0], float): activity_data[0] = TimeUtils.formatTimeStamp( activity_data[0]) tmp_dic = copy.deepcopy(dic) tmp_dic["time"] = activity_data[0] logger.debug(tmp_dic) writer_p = csv.writer(writer, lineterminator='\n') writer_p.writerow(activity_data) except Exception as e: logger.error("activity save error ") s = traceback.format_exc() logger.debug(s)
def start(self,start_time): if not RuntimeData.package_save_path: RuntimeData.package_save_path = os.path.join(os.path.abspath(os.path.join(os.getcwd(), "../..")),'results',self.device.adb._device_id,start_time) if not os.path.exists(RuntimeData.package_save_path): os.makedirs(RuntimeData.package_save_path) self.start_time = start_time self.power_collector.start(start_time) logger.debug("INFO: PowerMonitor has started...")
def start(self,start_time): ''' 启动一个搜集器来启动一个新的线程搜集cpu信息 :return: ''' self.collect_device_cpu_thread = threading.Thread(target=self._collect_cpu_thread, args=(start_time,)) self.collect_device_cpu_thread.start() logger.debug("INFO: CpuCollector started...")
def stop(self): logger.debug("INFO: TrafficCollecor stop...") if (self.collect_traffic_thread.isAlive()): self._stop_event.set() self.collect_traffic_thread.join(timeout=1) self.collect_traffic_thread = None if self.traffic_queue: self.traffic_queue.task_done()
def stop_logcat(self): '''停止logcat进程 ''' self._logcat_running = False logger.debug("stop logcat") if hasattr(self, '_log_pipe'): if self._log_pipe.poll() == None: # 判断logcat进程是否存在 self._log_pipe.terminate()
def recover(): if ADB.checkAdbNormal(): logger.debug("adb is normal") return else: logger.error("adb is not normal") ADB.kill_server() ADB.start_server()
def filter_file_names(self, device): csv_files = [] logger.debug(device) for f in os.listdir(device): if os.path.isfile(os.path.join(device, f)) and os.path.basename( f) in self.summary_csf_file.keys(): logger.debug(os.path.join(device, f)) csv_files.append(f) return csv_files
def _cat_current(self): current = 0 # cat /sys/class/power_supply/Battery/current_now Android9 上没权限 reg = self.device.adb.run_shell_cmd('cat /sys/class/power_supply/battery/current_now') if isinstance(reg, str) and "No such file or directory"==reg: logger.debug("can't get current from file /sys/class/power_supply/battery/current_now") elif reg: current = reg return current
def stop(self): logger.debug("INFO: ThreadNumPackageCollector stop... ") if (self.collect_thread_num_thread.isAlive()): self._stop_event.set() self.collect_thread_num_thread.join(timeout=1) self.collect_thread_num_thread = None #结束的时候,发送一个任务完成的信号,以结束队列 if self.thread_queue: self.thread_queue.task_done()
def _monkey_thread_func(self,save_dir): '''获取monkey线程,保存monkey日志,monkey Crash日志暂不处理,后续有需要再处理 ''' self.append_log_line_num = 0 self.file_log_line_num = 0 self.log_file_create_time = None log_is_none = 0 logs = [] logger.debug("monkey_thread_func") if RuntimeData.start_time is None: RuntimeData.start_time = TimeUtils.getCurrentTime() while self.running: try: log = self._log_pipe.stdout.readline().strip() if not isinstance(log, str): # 先编码为unicode try: log = str(log, "utf8") except Exception as e: log = repr(log) logger.error('str error:' + log) logger.error(e) if log: logs.append(log) self.append_log_line_num = self.append_log_line_num + 1 self.file_log_line_num = self.file_log_line_num + 1 # if self.append_log_line_num > 1000: if self.append_log_line_num > 100: if not self.log_file_create_time: self.log_file_create_time = TimeUtils.getCurrentTimeUnderline() log_file = os.path.join(save_dir, 'monkey_%s.log' % self.log_file_create_time) self.append_log_line_num = 0 # 降低音量,避免音量过大,导致语音指令失败 self.device.adb.run_shell_cmd("input keyevent 25") self.save(log_file, logs) logs = [] # 新建文件 if self.file_log_line_num > 600000: # if self.file_log_line_num > 200: self.file_log_line_num = 0 self.log_file_create_time = TimeUtils.getCurrentTimeUnderline() log_file = os.path.join(save_dir, 'monkey_%s.log' % self.log_file_create_time) self.save(log_file, logs) logs = [] else: log_is_none = log_is_none + 1 if log_is_none % 1000 == 0: logger.info("log is none") if not self.device.adb.is_process_running("com.android.commands.monkey") and self.running: self.device.adb.kill_process("com.android.commands.monkey") self._log_pipe = self.device.adb.run_shell_cmd(self.monkey_cmd, sync=False) except: logger.error("an exception hanpend in monkey thread, reason unkown!") s = traceback.format_exc() logger.debug(s)
def start(self, start_time): if not RuntimeData.package_save_path: RuntimeData.package_save_path = os.path.join( os.path.abspath(os.path.join(os.getcwd(), "../..")), 'results', self.package, start_time) if not os.path.exists(RuntimeData.package_save_path): os.makedirs(RuntimeData.package_save_path) self.start_time = start_time self.traffic_colloctor.start(start_time) logger.debug("INFO: TrafficMonitor has started...")
def stop(self): '''结束logcat日志监控器 ''' logger.debug("logcat monitor: stop...") self.remove_log_handle(self.launchtime.handle_launchtime) # 删除回调 logger.debug("logcat monitor: stopped") if self.exception_log_list: self.remove_log_handle(self.handle_exception) self.device.adb.stop_logcat() self.running = False
def get_pckinfo_from_ps(self, packagename): ''' 从ps中获取应用的信息:pid,uid,packagename :param packagename: 目标包名 :return: 返回目标包名的列表信息 ''' # get list_process 有重复代码 results = self.run_shell_cmd('ps -A') lines = results.replace('\r', '').splitlines() # logger.debug(lines) busybox = False if lines[0].startswith('PID'): busybox = True result_list = [] for line in lines: # logger.debug(line) if packagename in line: items = line.split() if not busybox: if len(items) < 9: err_msg = "ps命令返回格式错误:\n%s" % line if len(items) == 8: target_pkg = items[7] # 只保存与packagename完全相同的进程,对于以包名开头的其他进程将不会被保存,例如packagename:com.xx,com.xx:play不会保存 if packagename == target_pkg: result_list.append( {'pid': int(items[1]), 'uid': items[0], 'proc_name': items[-1], 'status': items[-2]}) else: logger.error(err_msg) else: target_pck = items[-1] # 只保存与packagename完全相同的进程,对于以包名开头的其他进程将不会被保存,例如packagename:com.xx,com.xx:play不会保存 if packagename == target_pck: result_list.append( {'pid': int(items[1]), 'uid': items[0], 'proc_name': items[-1], 'status': items[-2]}) else: # busybox = True idx = 4 cmd = items[idx] if len(cmd) == 1: # 有时候发现此处会有“N” idx += 1 cmd = items[idx] idx += 1 if cmd[0] == '{' and cmd[-1] == '}': cmd = items[idx] target_pkg = items[-1] if packagename == target_pkg: result_list.append( {'pid': int(items[0]), 'uid': items[1], 'proc_name': cmd, 'status': items[-2]}) if len(result_list) > 0: logger.debug(" get pckinfo from ps: " + str(result_list)) return result_list
def parse_data_from_config(self): ''' 从配置文件中解析出需要的信息,包名,时间间隔,设备的序列号等 :return:配置文件中读出来的数值的字典 ''' config_dic = {} configpath = os.path.join(RuntimeData.top_dir, "config.conf") logger.debug("configpath:%s" % configpath) if not os.path.isfile(configpath): logger.error("the config file didn't exist: " + configpath) raise RuntimeError("the config file didn't exist: " + configpath) # 避免windows会用系统默认的gbk打开 with open(configpath, encoding="utf-8") as f: content = f.read() # Window下用记事本打开配置文件并修改保存后,编码为UNICODE或UTF-8的文件的文件头 # 会被相应的加上\xff\xfe(\xff\xfe)或\xef\xbb\xbf,然后再传递给ConfigParser解析的时候会出错 # ,因此解析之前,先替换掉 content = re.sub(r"\xfe\xff", "", content) content = re.sub(r"\xff\xfe", "", content) content = re.sub(r"\xef\xbb\xbf", "", content) open(configpath, 'w', encoding="utf-8").write(content) paser = ConfigParser() paser.read(configpath, encoding="utf-8") config_dic = self.check_config_option(config_dic, paser, "Common", "package") config_dic = self.check_config_option(config_dic, paser, "Common", "pid_change_focus_package") config_dic = self.check_config_option(config_dic, paser, "Common", "frequency") config_dic = self.check_config_option(config_dic, paser, "Common", "dumpheap_freq") config_dic = self.check_config_option(config_dic, paser, "Common", "timeout") config_dic = self.check_config_option(config_dic, paser, "Common", "serialnum") config_dic = self.check_config_option(config_dic, paser, "Common", "mailbox") config_dic = self.check_config_option(config_dic, paser, "Common", "exceptionlog") config_dic = self.check_config_option(config_dic, paser, "Common", "save_path") config_dic = self.check_config_option(config_dic, paser, "Common", "phone_log_path") # 读取monkey配置 config_dic = self.check_config_option(config_dic, paser, "Common", "monkey") config_dic = self.check_config_option(config_dic, paser, "Common", "main_activity") config_dic = self.check_config_option(config_dic, paser, "Common", "activity_list") logger.debug(config_dic) return config_dic
def get_process_thread_num(self, process): pid = self.device.adb.get_pid_from_pck(self.packagename) out = self.device.adb.run_shell_cmd('ls -lt /proc/%s/task' % pid) collection_time = time.time() logger.debug("collection time in thread_num info is : " + str(collection_time)) if out: # logger.debug("thread num out:"+out) thread_num = len(out.split("\n")) return [collection_time, self.packagename, pid, thread_num] else: return []