def generate_results(task, data): """ 创建所有相关的文件 :param task: 任务信息,主要是路径 :param data: 经过解析,最终得到的数据 """ # 目录结构: # 第一级: # 文件:overview.html, overview_data.js, logo.png # 目录: exceptions, signals # 第二级: # exceptions: # exceptions.html, exceptions.css, logo.png # signals: # signals.html, signals.css, logo.png num_exception = len(data[__M_JAVA__]) num_signal = len(data[__M_NATIVE__]) num_anr = len(data[__M_ANR__]) result_path = task.log_path + "/Here" # 创建对应的目录 try: os.makedirs(result_path, exist_ok=True) except OSError, e: tool.log( LOG_TAG, "create folder { %s } failed, reason: { %s }" % (result_path, e.message)) return False
def __receiver(self, receiver): ''' receive data from other process :param receiver: a system pipe which can be used for receiving data from child processes ''' tool.log("start __receiver") while self.running: data = os.read(receiver, 256) # get path and percent from data like "path: /home/path/xxx, percent: 0.1341" m = re.match(r"^path: (/.+), percent: ([01]\.\d+)?$", data) if m: path = m.group(1) percent = float(m.group(2)) #print "pid = %d" % multiprocessing.current_process().pid, "path = %s" % path, percent else: # TODO 由于没有同步机制, 会出现还没写入完成就被读走的情况,仅作为一种大概的反馈 # __receiver.error : not formatted data : -30 (copy).log, percent: 0.209564000682 tool.log("__receiver.error", "not formatted data : %s" % data) continue # calculate current progress self._progress_dict_[path] = percent # float progress = float(sum(self._progress_dict_.values())) / len( self.file_path_list) # int progress = int(progress * 100) print progress # send progress to task manager if self.task_listener: self.task_listener.on_task_progress_changed( self.task, progress)
def UpdateTaskState(self, task): if task.state >= Task.__STATE_DONE__: tool.log("UpdateTaskState.error", "Wrong State %d" % task.state) raise Exception, "Wrong State %d" % task.state index = self.ulc.FindItem(-1, task.log_path) if index != wx.NOT_FOUND: path_item = self.ulc.GetItem(index, 0) path_item.SetImage(task.state) path_item.SetPyData(task) self.ulc.SetItem(path_item) state_item = self.ulc.GetItem(index, 1) # try: if task.state == Task.__STATE_PROCESSING__: state_item.SetText('') gauge = wx.Gauge(self.ulc, -1, size=(200, 20), style=wx.GA_HORIZONTAL | wx.GA_SMOOTH) state_item.SetWindow(gauge) elif task.state == Task.__STATE_PAUSED__: state_item.DeleteWindow() state_item.SetText(LANG.task_state_paused) elif task.state == Task.__STATE_GENERATING__: state_item.DeleteWindow() state_item.SetText(LANG.task_state_generating) # except Exception as e: # tool.log("UpdateTaskState.error", e.message) self.ulc.SetItem(state_item) if self.ulc.GetItemCount() > 1: self.ulc.SortItems() self.ulc.Refresh()
def getLoad(self): #return 10 if not self.load: # 1 需要去除文件夹中不需要解析的文件,再求合 # 2 self.load = min(load_cal, files_count) self.files = file_list = tool.getParseableFileList(self.log_path) tool.log("getLoad", self.files) self.load = tool.getFilesSizeMB(file_list) / Task.__LOAD_UNIT__ + 1 self.load = min(self.load, len(file_list)) return self.load
def UpdateTaskProgress(self, task, progress): if task.state != Task.__STATE_PROCESSING__: tool.log("UpdateTaskProgress.error", "Wrong State %d" % task.state) return index = self.ulc.FindItem(-1, task.log_path) if index != wx.NOT_FOUND: state_item = self.ulc.GetItem(index, 1) gauge = state_item.GetWindow() gauge.SetValue(min(max(0, int(progress)), 100))
def get_load_sum(self): """ 获取任务总量 """ load_sum = 0 for t in self._queue: load_sum += t.getLoad() tool.log(LOG_TAG, "load_sum = %d" % load_sum) return load_sum
def __run__(self, log_path, modules): """ start a process and do job :param log_path: the log to parse :param modules: the modules we care :return: the parse result we get """ #print multiprocessing.current_process().pid tool.log("__start_one_parser", log_path) parser = Parser(log_path, modules) parser.set_pipe_sender(self.wpipe) return parser.parse()
def generate_fatal_exception(folder, data): """ 创建Fatal Exception的html文件 :param folder 指定的结果存放目录 :param data: 存储着fatal exception的数据 """ exception_folder = folder + "/exceptions" try: os.mkdir(exception_folder) except OSError, e: tool.log( LOG_TAG, "create folder { %s } failed, reason: { %s }" % (exception_folder, e.message)) return False
def create_overview_data(folder, data, result_links): """ 在js文件夹下创建overview_data.js :param folder: 目录 :param data: :return: """ data_folder = folder + "/js" try: os.mkdir(data_folder) except OSError, e: tool.log( LOG_TAG, "create folder { %s } failed, reason: { %s }" % (data_folder, e.message)) return False
def _handle_tasks(self): tool.log(LOG_TAG, "_handle_tasks start") while self._running: # 1秒一次的轮询 time.sleep(1) # 如果等待队列中没有任务, 则继续 if self._waitingQueue.size() <= 0: continue # 当处理队列中有任务的时候,计算当前的总工作量,用以确定TaskManager是否还能承受 if self._processingQueue.size() > 0: # 当前任务量 current_load = self._processingQueue.get_load_sum() # 下一个任务的任务量 future_load = self._waitingQueue.get_header_load() # 如果两者相加没超过总任务量, 则运行 if current_load + future_load <= TaskManager._MAX_PROCESSING_: # 将等待队列的任务放入处理队列,并启动进程池执行任务 t = self._waitingQueue.get() self._processingQueue.put(t) pm = ParserManager(t) if self._task_listener: pm.set_task_listener(self._task_listener) pm.execute() tool.log(LOG_TAG, "_handle_tasks: start ParserManager 1") else: tool.log(LOG_TAG, "_handle_tasks: please hold on") else: # 如果当前处理队列中没有任务, 直接从等待队列中拿出一个任务放到处理队列 task = self._waitingQueue.get() # 计算任务量 task.getLoad() self._processingQueue.put(task) if self._task_listener: self._task_listener.on_task_state_changed(task) tool.log(LOG_TAG, "tagggggggggggggggg") tool.log(LOG_TAG, "state = %d" % task.state) # 启动任务 pm = ParserManager(task) pm.set_task_listener(self._task_listener) pm.execute()
def AddTask(self, task): # 先插入item,并形成item,然后排序 if task.state != Task.__STATE_WAITING__: tool.log("AddTask.error", "Only task in waiting can ne added") return index = self.ulc.GetItemCount() self.ulc.InsertImageStringItem(index, task.log_path, task.state) path_item = self.ulc.GetItem(index, 0) path_item.SetPyData(task) self.ulc.SetItem(path_item) state_item = self.ulc.GetItem(index, 1) if task.state == Task.__STATE_WAITING__: state_item.SetText(LANG.task_state_waiting) self.ulc.SetItem(state_item) self.ulc.SortItems() self.ulc.Refresh()
def select(sql, args=()): if sql is None or sql == '': __notifySelected(None) __tool.log("select.error", "sql is null") return None tryCreateTables() db = sqlite3.connect(__getDbPath()) cursor = db.cursor() try: cursor.execute(sql, args) ret = cursor.fetchall() __notifySelected(ret) return ret except Exception as e: __notifySelected(None) __tool.log("select.error", e.message) return None finally: db.close()
def delete(sql, args=()): if sql is None or sql == '': __notifyDeleted(False, "sql is null") __tool.log("delete.error", "sql is null") return False, tryCreateTables() db = sqlite3.connect(__getDbPath()) cursor = db.cursor() try: cursor.execute(sql, args) db.commit() __notifyDeleted(True) return True, except Exception as e: __notifyDeleted(False, e.message) __tool.log("delete.error", e.message) return False, e.message #return False finally: db.close()
def insert(sql, args=()): if sql is None or sql == '': __notifyInsert(False, "sql is null") __tool.log("select.error", "sql is null") return False, tryCreateTables() db = sqlite3.connect(__getDbPath()) cursor = db.cursor() try: cursor.execute( #'''INSERT INTO history (name, state, log_path, src_path, create_time) VALUES (?,?,?,?,?)''' #,(task.name, task.state, task.log_path, task.src_path, task.create_time) sql, args ) db.commit() __notifyInsert(True) __tool.log("sqlite3.insert success", sql + str(args)) return True, except sqlite3.IntegrityError as e: __notifyInsert(False, e.message) __tool.log("sqlite3.IntegrityError", e.message) return False, e.message #return False finally: db.close()
def generate_results(task, data): """ 创建所有相关的文件 :param task: 任务信息,主要是路径 :param data: 经过解析,最终得到的数据 """ # 目录结构: # 第一级: # 文件:overview.html # 目录: exceptions, signals, js # 第二级: # exceptions: # exceptions.html, exceptions.css # signals: # signals.html, signals.css # js: # overview_data.js, exporting.js, highcharts.js num_exception = len(data[__M_JAVA__]) num_signal = len(data[__M_NATIVE__]) num_anr = len(data[__M_ANR__]) p, f = os.path.split(task.log_path) result_path = p + "/%s.alar" % f # 创建对应的目录 try: if tool.checkDirExists(result_path): shutil.rmtree(result_path) os.mkdir(result_path) except OSError, e: tool.log( LOG_TAG, "create folder { %s } failed, reason: { %s }" % (result_path, e.message)) return False
def update(sql, args=()): if sql is None or sql == '': __notifyUpdated(False, "sql is null") __tool.log("update", "sqlite is null") return False, tryCreateTables() db = sqlite3.connect(__getDbPath()) cursor = db.cursor() try: print args cursor.execute(sql, args) db.commit() __notifyUpdated(True) __tool.log("sqlite3.update", sql + str(args) + " success") return True, except sqlite3.IntegrityError as e: __notifyUpdated(False, e.message) __tool.log("update.IntegrityError", e.message) return False, e.message #return False finally: db.close()
def __notifySelected(data): try: for c in __SELECTED_LISTENER: c(data) except Exception as e: __tool.log("__notifySelected.error", e.message)
def __notifyInsert(result, msg=''): try: for c in __INSERT_LISTENERS: c(result, msg) except Exception as e: __tool.log("__notifyInsert.error", e.message)
def addInsertedListener(callback): if callback not in __INSERT_LISTENERS: __INSERT_LISTENERS.append(callback) __tool.log("add callback" + str(callback)) else: __tool.log("callback exists")
def __notifyUpdated(result, msg=''): try: for c in __UPDATED_LISTENER: c(result, msg) except Exception as e: __tool.log("__notifyUpdated.error", e.message)
def addUpdatedListener(callback): if callback not in __UPDATED_LISTENER: __tool.log("addUpdatedListener", "add callback" + str(callback)) __UPDATED_LISTENER.append(callback) else: __tool.log("addUpdatedListener", "call back existed")
def __notifyDeleted(result, msg=''): try: for c in __DELETE_LISTENER: c(result, msg) except Exception as e: __tool.log("__notifyDeleted.error", e.message)
def addDeletedListener(callback): if callback not in __DELETE_LISTENER: __DELETE_LISTENER.append(callback) __tool.log("sqlite.addDeletedListener " + str(callback)) else: __tool.log("sqlite.addDeletedListener callback exists")
def rollbackline(self, offset): try: super(LineReader, self).seek(offset, 1) self.line_num -= 1 except IOError as ioe: tool.log("rollbackline", "IOError: " + ioe.strerror)
def addSelectedListener(callback): if callback not in __SELECTED_LISTENER: __SELECTED_LISTENER.append(callback) __tool.log("sqlite.addSelectedListener " + str(callback)) else: __tool.log("sqlite.addSelectedListener callback exists")
__tool.log("addUpdatedListener", "add callback" + str(callback)) __UPDATED_LISTENER.append(callback) else: __tool.log("addUpdatedListener", "call back existed") def __notifyUpdated(result, msg=''): try: for c in __UPDATED_LISTENER: c(result, msg) except Exception as e: __tool.log("__notifyUpdated.error", e.message) # --------------Test-------------------------------------------------------------------------- if __name__ == "__main__": __tool.log(__getDbPath()) tryCreateTables() def a(r,m): print "__notifyUpdated a ", r, m # def b(r): # print "addSelectedListener b ", r #addInsertedListener(a) #addInsertedListener(b) # addSelectedListener(a) # addSelectedListener(b) # addSelectedListener(b) # addDeletedListener(a) # addDeletedListener(a) addUpdatedListener(a)