def generate_results(task, data):
"""
创建所有相关的文件
:param task: 任务信息，主要是路径
:param data: 经过解析，最终得到的数据
"""
# 目录结构：
# 第一级：
# 文件：overview.html, overview_data.js, logo.png
# 目录： exceptions, signals
# 第二级：
# exceptions：
# exceptions.html, exceptions.css, logo.png
# signals:
# signals.html, signals.css, logo.png
num_exception = len(data[__M_JAVA__])
num_signal = len(data[__M_NATIVE__])
num_anr = len(data[__M_ANR__])
result_path = task.log_path + "/Here"
# 创建对应的目录
try:
os.makedirs(result_path, exist_ok=True)
except OSError, e:
tool.log(
LOG_TAG, "create folder { %s } failed, reason: { %s }" %
(result_path, e.message))
return False
def __receiver(self, receiver):
'''
receive data from other process
:param receiver: a system pipe which can be used for receiving data from child processes
'''
tool.log("start __receiver")
while self.running:
data = os.read(receiver, 256)
# get path and percent from data like "path: /home/path/xxx, percent: 0.1341"
m = re.match(r"^path: (/.+), percent: ([01]\.\d+)?$", data)
if m:
path = m.group(1)
percent = float(m.group(2))
#print "pid = %d" % multiprocessing.current_process().pid, "path = %s" % path, percent
else:
# TODO 由于没有同步机制， 会出现还没写入完成就被读走的情况，仅作为一种大概的反馈
# __receiver.error : not formatted data : -30 (copy).log, percent: 0.209564000682
tool.log("__receiver.error", "not formatted data : %s" % data)
continue
# calculate current progress
self._progress_dict_[path] = percent
# float
progress = float(sum(self._progress_dict_.values())) / len(
self.file_path_list)
# int
progress = int(progress * 100)
print progress
# send progress to task manager
if self.task_listener:
self.task_listener.on_task_progress_changed(
self.task, progress)
def UpdateTaskState(self, task):
if task.state >= Task.__STATE_DONE__:
tool.log("UpdateTaskState.error", "Wrong State %d" % task.state)
raise Exception, "Wrong State %d" % task.state
index = self.ulc.FindItem(-1, task.log_path)
if index != wx.NOT_FOUND:
path_item = self.ulc.GetItem(index, 0)
path_item.SetImage(task.state)
path_item.SetPyData(task)
self.ulc.SetItem(path_item)
state_item = self.ulc.GetItem(index, 1)
# try:
if task.state == Task.__STATE_PROCESSING__:
state_item.SetText('')
gauge = wx.Gauge(self.ulc,
-1,
size=(200, 20),
style=wx.GA_HORIZONTAL | wx.GA_SMOOTH)
state_item.SetWindow(gauge)
elif task.state == Task.__STATE_PAUSED__:
state_item.DeleteWindow()
state_item.SetText(LANG.task_state_paused)
elif task.state == Task.__STATE_GENERATING__:
state_item.DeleteWindow()
state_item.SetText(LANG.task_state_generating)
# except Exception as e:
# tool.log("UpdateTaskState.error", e.message)
self.ulc.SetItem(state_item)
if self.ulc.GetItemCount() > 1:
self.ulc.SortItems()
self.ulc.Refresh()
def getLoad(self):
#return 10
if not self.load:
# 1 需要去除文件夹中不需要解析的文件，再求合
# 2 self.load = min(load_cal, files_count)
self.files = file_list = tool.getParseableFileList(self.log_path)
tool.log("getLoad", self.files)
self.load = tool.getFilesSizeMB(file_list) / Task.__LOAD_UNIT__ + 1
self.load = min(self.load, len(file_list))
return self.load
def UpdateTaskProgress(self, task, progress):
if task.state != Task.__STATE_PROCESSING__:
tool.log("UpdateTaskProgress.error", "Wrong State %d" % task.state)
return
index = self.ulc.FindItem(-1, task.log_path)
if index != wx.NOT_FOUND:
state_item = self.ulc.GetItem(index, 1)
gauge = state_item.GetWindow()
gauge.SetValue(min(max(0, int(progress)), 100))
def get_load_sum(self):
"""
获取任务总量
"""
load_sum = 0
for t in self._queue:
load_sum += t.getLoad()
tool.log(LOG_TAG, "load_sum = %d" % load_sum)
return load_sum
def __run__(self, log_path, modules):
"""
start a process and do job
:param log_path: the log to parse
:param modules: the modules we care
:return: the parse result we get
"""
#print multiprocessing.current_process().pid
tool.log("__start_one_parser", log_path)
parser = Parser(log_path, modules)
parser.set_pipe_sender(self.wpipe)
return parser.parse()
def generate_fatal_exception(folder, data):
"""
创建Fatal Exception的html文件
:param folder 指定的结果存放目录
:param data: 存储着fatal exception的数据
"""
exception_folder = folder + "/exceptions"
try:
os.mkdir(exception_folder)
except OSError, e:
tool.log(
LOG_TAG, "create folder { %s } failed, reason: { %s }" %
(exception_folder, e.message))
return False
def create_overview_data(folder, data, result_links):
"""
在js文件夹下创建overview_data.js
:param folder: 目录
:param data:
:return:
"""
data_folder = folder + "/js"
try:
os.mkdir(data_folder)
except OSError, e:
tool.log(
LOG_TAG, "create folder { %s } failed, reason: { %s }" %
(data_folder, e.message))
return False
def _handle_tasks(self):
tool.log(LOG_TAG, "_handle_tasks start")
while self._running:
# 1秒一次的轮询
time.sleep(1)
# 如果等待队列中没有任务， 则继续
if self._waitingQueue.size() <= 0:
continue
# 当处理队列中有任务的时候，计算当前的总工作量，用以确定TaskManager是否还能承受
if self._processingQueue.size() > 0:
# 当前任务量
current_load = self._processingQueue.get_load_sum()
# 下一个任务的任务量
future_load = self._waitingQueue.get_header_load()
# 如果两者相加没超过总任务量， 则运行
if current_load + future_load <= TaskManager._MAX_PROCESSING_:
# 将等待队列的任务放入处理队列，并启动进程池执行任务
t = self._waitingQueue.get()
self._processingQueue.put(t)
pm = ParserManager(t)
if self._task_listener:
pm.set_task_listener(self._task_listener)
pm.execute()
tool.log(LOG_TAG, "_handle_tasks: start ParserManager 1")
else:
tool.log(LOG_TAG, "_handle_tasks: please hold on")
else:
# 如果当前处理队列中没有任务， 直接从等待队列中拿出一个任务放到处理队列
task = self._waitingQueue.get()
# 计算任务量
task.getLoad()
self._processingQueue.put(task)
if self._task_listener:
self._task_listener.on_task_state_changed(task)
tool.log(LOG_TAG, "tagggggggggggggggg")
tool.log(LOG_TAG, "state = %d" % task.state)
# 启动任务
pm = ParserManager(task)
pm.set_task_listener(self._task_listener)
pm.execute()
def AddTask(self, task):
# 先插入item，并形成item，然后排序
if task.state != Task.__STATE_WAITING__:
tool.log("AddTask.error", "Only task in waiting can ne added")
return
index = self.ulc.GetItemCount()
self.ulc.InsertImageStringItem(index, task.log_path, task.state)
path_item = self.ulc.GetItem(index, 0)
path_item.SetPyData(task)
self.ulc.SetItem(path_item)
state_item = self.ulc.GetItem(index, 1)
if task.state == Task.__STATE_WAITING__:
state_item.SetText(LANG.task_state_waiting)
self.ulc.SetItem(state_item)
self.ulc.SortItems()
self.ulc.Refresh()
def select(sql, args=()):
if sql is None or sql == '':
__notifySelected(None)
__tool.log("select.error", "sql is null")
return None
tryCreateTables()
db = sqlite3.connect(__getDbPath())
cursor = db.cursor()
try:
cursor.execute(sql, args)
ret = cursor.fetchall()
__notifySelected(ret)
return ret
except Exception as e:
__notifySelected(None)
__tool.log("select.error", e.message)
return None
finally:
db.close()
def delete(sql, args=()):
if sql is None or sql == '':
__notifyDeleted(False, "sql is null")
__tool.log("delete.error", "sql is null")
return False,
tryCreateTables()
db = sqlite3.connect(__getDbPath())
cursor = db.cursor()
try:
cursor.execute(sql, args)
db.commit()
__notifyDeleted(True)
return True,
except Exception as e:
__notifyDeleted(False, e.message)
__tool.log("delete.error", e.message)
return False, e.message
#return False
finally:
db.close()
def insert(sql, args=()):
if sql is None or sql == '':
__notifyInsert(False, "sql is null")
__tool.log("select.error", "sql is null")
return False,
tryCreateTables()
db = sqlite3.connect(__getDbPath())
cursor = db.cursor()
try:
cursor.execute(
#'''INSERT INTO history (name, state, log_path, src_path, create_time) VALUES (?,?,?,?,?)'''
#,(task.name, task.state, task.log_path, task.src_path, task.create_time)
sql, args
)
db.commit()
__notifyInsert(True)
__tool.log("sqlite3.insert success", sql + str(args))
return True,
except sqlite3.IntegrityError as e:
__notifyInsert(False, e.message)
__tool.log("sqlite3.IntegrityError", e.message)
return False, e.message
#return False
finally:
db.close()
def generate_results(task, data):
"""
创建所有相关的文件
:param task: 任务信息，主要是路径
:param data: 经过解析，最终得到的数据
"""
# 目录结构：
# 第一级：
# 文件：overview.html
# 目录： exceptions, signals, js
# 第二级：
# exceptions：
# exceptions.html, exceptions.css
# signals:
# signals.html, signals.css
# js:
# overview_data.js, exporting.js, highcharts.js
num_exception = len(data[__M_JAVA__])
num_signal = len(data[__M_NATIVE__])
num_anr = len(data[__M_ANR__])
p, f = os.path.split(task.log_path)
result_path = p + "/%s.alar" % f
# 创建对应的目录
try:
if tool.checkDirExists(result_path):
shutil.rmtree(result_path)
os.mkdir(result_path)
except OSError, e:
tool.log(
LOG_TAG, "create folder { %s } failed, reason: { %s }" %
(result_path, e.message))
return False
def update(sql, args=()):
if sql is None or sql == '':
__notifyUpdated(False, "sql is null")
__tool.log("update", "sqlite is null")
return False,
tryCreateTables()
db = sqlite3.connect(__getDbPath())
cursor = db.cursor()
try:
print args
cursor.execute(sql, args)
db.commit()
__notifyUpdated(True)
__tool.log("sqlite3.update", sql + str(args) + " success")
return True,
except sqlite3.IntegrityError as e:
__notifyUpdated(False, e.message)
__tool.log("update.IntegrityError", e.message)
return False, e.message
#return False
finally:
db.close()
def __notifySelected(data):
try:
for c in __SELECTED_LISTENER:
c(data)
except Exception as e:
__tool.log("__notifySelected.error", e.message)
def __notifyInsert(result, msg=''):
try:
for c in __INSERT_LISTENERS:
c(result, msg)
except Exception as e:
__tool.log("__notifyInsert.error", e.message)
def addInsertedListener(callback):
if callback not in __INSERT_LISTENERS:
__INSERT_LISTENERS.append(callback)
__tool.log("add callback" + str(callback))
else:
__tool.log("callback exists")
def __notifyUpdated(result, msg=''):
try:
for c in __UPDATED_LISTENER:
c(result, msg)
except Exception as e:
__tool.log("__notifyUpdated.error", e.message)
def addUpdatedListener(callback):
if callback not in __UPDATED_LISTENER:
__tool.log("addUpdatedListener", "add callback" + str(callback))
__UPDATED_LISTENER.append(callback)
else:
__tool.log("addUpdatedListener", "call back existed")
def __notifyDeleted(result, msg=''):
try:
for c in __DELETE_LISTENER:
c(result, msg)
except Exception as e:
__tool.log("__notifyDeleted.error", e.message)
def addDeletedListener(callback):
if callback not in __DELETE_LISTENER:
__DELETE_LISTENER.append(callback)
__tool.log("sqlite.addDeletedListener " + str(callback))
else:
__tool.log("sqlite.addDeletedListener callback exists")
def rollbackline(self, offset):
try:
super(LineReader, self).seek(offset, 1)
self.line_num -= 1
except IOError as ioe:
tool.log("rollbackline", "IOError: " + ioe.strerror)
def addSelectedListener(callback):
if callback not in __SELECTED_LISTENER:
__SELECTED_LISTENER.append(callback)
__tool.log("sqlite.addSelectedListener " + str(callback))
else:
__tool.log("sqlite.addSelectedListener callback exists")
__tool.log("addUpdatedListener", "add callback" + str(callback))
__UPDATED_LISTENER.append(callback)
else:
__tool.log("addUpdatedListener", "call back existed")
def __notifyUpdated(result, msg=''):
try:
for c in __UPDATED_LISTENER:
c(result, msg)
except Exception as e:
__tool.log("__notifyUpdated.error", e.message)
# --------------Test--------------------------------------------------------------------------
if __name__ == "__main__":
__tool.log(__getDbPath())
tryCreateTables()
def a(r,m):
print "__notifyUpdated a ", r, m
# def b(r):
# print "addSelectedListener b ", r
#addInsertedListener(a)
#addInsertedListener(b)
# addSelectedListener(a)
# addSelectedListener(b)
# addSelectedListener(b)
# addDeletedListener(a)
# addDeletedListener(a)
addUpdatedListener(a)
