def do_executeMethod(self, fs, sb, date_from_ms, date_to_ms):
for f in fs:
lines = file(f).readlines()
print datetime.datetime.now().isoformat().replace("T", " "), f, "readline ok"
if not analyse_qbrm_log.is_effective_log_file(lines, date_from_ms, date_to_ms):
print "ignore ", f
continue
for line in lines:
line = ui_public.get_line(line, log_file_encode)
log_infos = line.split("|", 5)
if len(log_infos) < 6:
continue
epoch = (long)(log_infos[0])
if date_from_ms > 0 and epoch < date_from_ms:
continue
if date_to_ms > 0 and epoch > date_to_ms:
continue
self.valid_row_count += 1
class_method = log_infos[4] + "." + log_infos[5]
execute_time = long(log_infos[1])
return_size = 0
if len(log_infos[2]) > 0:
return_size = long(log_infos[2])
if self.log_dic.__contains__(class_method):
self.log_dic[class_method][1].append(execute_time)
self.log_dic[class_method][2].append(return_size)
else:
self.log_dic[class_method] = [class_method, [execute_time], [return_size]]
print datetime.datetime.now().isoformat().replace("T", " "), f, "merge ok"
def find_uuids(self, fs, date_from_ms, date_to_ms):
uuids = {}
for f in fs:
lines = file(f).readlines()
print datetime.datetime.now().isoformat().replace('T', ' '), f, 'readline ok'
if not analyse_qbrm_log.is_effective_log_file(lines, date_from_ms, date_to_ms):
print 'ignore ', f
continue
for line in lines:
line = ui_public.get_line(line, log_file_encode)
log_infos = line.split('|', 5)
if len(log_infos) < 6:
continue
epoch = (long)(log_infos[0])
if date_from_ms > 0 and epoch < date_from_ms:
continue
if date_to_ms > 0 and epoch > date_to_ms:
continue
request_url = log_infos[4]
if request_url.find('?') < 0 and len(log_infos[5]) > 0:
request_url += '?'
request_url += log_infos[5]
if request_url == self.url:
uuids[log_infos[3][0:log_infos[3].find('.')]] = log_infos[1][0:log_infos[1].find(',')]
if len(uuids) > max_row:
break
if len(uuids) > max_row:
break
return uuids
def statisticErrorSql(self, fs, sb, date_from_ms, date_to_ms):
sb.append('\n<h4>执行错误的SQL:</h4>')
error_sql_dic = {}
for f in fs:
lines = file(f).readlines()
print datetime.datetime.now().isoformat().replace(
'T', ' '), f, 'readline ok'
if not analyse_qbrm_log.is_effective_log_file(
lines, date_from_ms, date_to_ms):
print 'ignore ', f
continue
for line in lines:
line = ui_public.get_line(line, log_file_encode)
sql_log = line.split('|', 6)
if len(sql_log) < 7 or sql_log[3] != 's':
continue
epoch = (long)(sql_log[0])
if date_from_ms > 0 and epoch < date_from_ms:
continue
if date_to_ms > 0 and epoch > date_to_ms:
continue
self.valid_row_count += 1
key = sql_log[5] + sql_log[6]
if error_sql_dic.__contains__(key):
error_sql_dic[key][2] = error_sql_dic[key][2] + 1
else:
error_sql_dic[key] = [sql_log[5], sql_log[6], 1]
print datetime.datetime.now().isoformat().replace(
'T', ' '), f, 'merge ok'
for error_sql in error_sql_dic.items():
sb.append('\n' + str(error_sql[1][2]) + '次, ' + error_sql[1][0] +
' ' + error_sql[1][1])
def statisticErrorSql(self, fs, sb, date_from_ms, date_to_ms):
sb.append( '\n<h4>执行错误的SQL:</h4>')
error_sql_dic = {}
for f in fs:
lines = file(f).readlines()
print datetime.datetime.now().isoformat().replace('T', ' '), f, 'readline ok'
if not analyse_qbrm_log.is_effective_log_file(lines, date_from_ms, date_to_ms):
print 'ignore ', f
continue
for line in lines:
line = ui_public.get_line(line, log_file_encode)
sql_log = line.split('|', 6)
if len(sql_log) < 7 or sql_log[3] != 's':
continue
epoch = (long)(sql_log[0])
if date_from_ms > 0 and epoch < date_from_ms:
continue
if date_to_ms > 0 and epoch > date_to_ms:
continue
self.valid_row_count += 1
key = sql_log[5] + sql_log[6]
if error_sql_dic.__contains__(key):
error_sql_dic[key][2] = error_sql_dic[key][2] + 1
else:
error_sql_dic[key] = [sql_log[5], sql_log[6], 1]
print datetime.datetime.now().isoformat().replace('T', ' '), f, 'merge ok'
for error_sql in error_sql_dic.items():
sb.append('\n' + str(error_sql[1][2]) + '次, ' + error_sql[1][0] + ' ' + error_sql[1][1])
def find_uuids(self, fs, date_from_ms, date_to_ms):
uuids = {}
for f in fs:
lines = file(f).readlines()
print datetime.datetime.now().isoformat().replace('T', ' '), f, 'readline ok'
if not analyse_qbrm_log.is_effective_log_file(lines, date_from_ms, date_to_ms):
print 'ignore ', f
continue
for line in lines:
line = ui_public.get_line(line, log_file_encode)
log_infos = line.split('|', 5)
if len(log_infos) < 6:
continue
epoch = (long)(log_infos[0])
if date_from_ms > 0 and epoch < date_from_ms:
continue
if date_to_ms > 0 and epoch > date_to_ms:
continue
request_url = log_infos[4]
if request_url.find('?') < 0 and len(log_infos[5]) > 0:
request_url += '?'
request_url += log_infos[5]
if request_url == self.url:
uuids[log_infos[3][0:log_infos[3].find('.')]] = log_infos[1][0:log_infos[1].find(',')]
if len(uuids) > max_row:
break
if len(uuids) > max_row:
break
return uuids
def do_executeMethod(self, fs, sb, date_from_ms, date_to_ms):
for f in fs:
lines = file(f).readlines()
print datetime.datetime.now().isoformat().replace(
'T', ' '), f, 'readline ok'
if not analyse_qbrm_log.is_effective_log_file(
lines, date_from_ms, date_to_ms):
print 'ignore ', f
continue
for line in lines:
line = ui_public.get_line(line, log_file_encode)
log_infos = line.split('|', 5)
if len(log_infos) < 6:
continue
epoch = (long)(log_infos[0])
if date_from_ms > 0 and epoch < date_from_ms:
continue
if date_to_ms > 0 and epoch > date_to_ms:
continue
self.valid_row_count += 1
class_method = log_infos[4] + '.' + log_infos[5]
execute_time = long(log_infos[1])
return_size = 0
if len(log_infos[2]) > 0:
return_size = long(log_infos[2])
if self.log_dic.__contains__(class_method):
self.log_dic[class_method][1].append(execute_time)
self.log_dic[class_method][2].append(return_size)
else:
self.log_dic[class_method] = [
class_method, [execute_time], [return_size]
]
print datetime.datetime.now().isoformat().replace(
'T', ' '), f, 'merge ok'
def do_executeRequest(self, fs, sb, date_from_ms, date_to_ms, match_user,
match_url):
for f in fs:
lines = file(f).readlines()
print datetime.datetime.now().isoformat().replace(
'T', ' '), f, 'readline ok'
if not analyse_qbrm_log.is_effective_log_file(
lines, date_from_ms, date_to_ms):
print 'ignore ', f
continue
for line in lines:
line = ui_public.get_line(line, log_file_encode)
log_infos = line.split('|', 5)
if len(log_infos) < 6:
continue
if match_user != None and match_user != "":
uuid_thread_user = log_infos[3].split('.', 3)
if len(uuid_thread_user) < 3 or not (re.match(
match_user, uuid_thread_user[2])):
continue
epoch = (long)(log_infos[0])
if date_from_ms > 0 and epoch < date_from_ms:
continue
if date_to_ms > 0 and epoch > date_to_ms:
continue
self.valid_row_count += 1
request_url = log_infos[4]
total_time = (long)(log_infos[1][0:log_infos[1].index(',')])
sql_time = (long)(log_infos[1][log_infos[1].index(',') + 1:])
#sql count
sql_counts = log_infos[2]
sql_count_single = sql_counts[0:sql_counts.index(',')]
sql_counts = sql_counts[sql_counts.index(',') + 1:]
sql_count_batch_sum = sql_counts[sql_counts.index(',') + 1:]
total_sql_count = long(sql_count_single) + long(
sql_count_batch_sum)
if self.log_dic.__contains__(request_url):
self.log_dic[request_url][1].append(total_time)
self.log_dic[request_url][2].append(sql_time)
self.log_dic[request_url][3].append(total_sql_count)
else:
request_uri = request_url
if request_url.find('?') < 0 and len(log_infos[5]) > 0:
request_uri += '?'
request_uri += log_infos[5]
self.log_dic[request_url] = [
request_uri, [total_time], [sql_time],
[total_sql_count]
]
print datetime.datetime.now().isoformat().replace(
'T', ' '), f, 'merge ok'
def find_methods(self, fs, date_from_ms, date_to_ms, uuids):
methods = []
for f in fs:
lines = file(f).readlines()
print datetime.datetime.now().isoformat().replace('T', ' '), f, 'readline ok'
if not analyse_qbrm_log.is_effective_log_file(lines, date_from_ms, date_to_ms):
print 'ignore ', f
continue
for line in lines:
line = ui_public.get_line(line, log_file_encode)
for uuid_key in uuids.keys():
if line.find(uuid_key) > -1:
methods.append(line)
return methods
def find_methods(self, fs, date_from_ms, date_to_ms, uuids):
methods = []
for f in fs:
lines = file(f).readlines()
print datetime.datetime.now().isoformat().replace('T', ' '), f, 'readline ok'
if not analyse_qbrm_log.is_effective_log_file(lines, date_from_ms, date_to_ms):
print 'ignore ', f
continue
for line in lines:
line = ui_public.get_line(line, log_file_encode)
for uuid_key in uuids.keys():
if line.find(uuid_key) > -1:
methods.append(line)
return methods
def statisticSqlExecuteTime(self, fs, sb, date_from_ms, date_to_ms):
for f in fs:
sys.setrecursionlimit(sys.maxint)
#1 begin read()读入，全部匹配
#2 begin 分行载入，逐条匹配
#3 begin 全部载入，字符串分析
lines = file(f).readlines()
print datetime.datetime.now().isoformat().replace(
'T', ' '), f, 'readline ok'
if not analyse_qbrm_log.is_effective_log_file(
lines, date_from_ms, date_to_ms):
print 'ignore ', f
continue
for line in lines:
line = ui_public.get_line(line, log_file_encode)
sql_log = line.split('|', 5)
if len(sql_log) < 6 or sql_log[3] != 's':
continue
epoch = (long)(sql_log[0])
if date_from_ms > 0 and epoch < date_from_ms:
continue
if date_to_ms > 0 and epoch > date_to_ms:
continue
self.valid_row_count += 1
strsql_merge = self.get_abstract_sql(sql_log[4], sql_log[5])
if self.sql_log_dic.__contains__(strsql_merge):
sql_times = self.sql_log_dic[strsql_merge][1]
sql_times.append(int(sql_log[1]))
else:
statement_type = ''
if len(sql_log[4]) == 0:
statement_type = 's'
self.sql_log_dic[strsql_merge] = [
sql_log[5], [int(sql_log[1])], statement_type
]
#end
print datetime.datetime.now().isoformat().replace(
'T', ' '), f, 'merge ok'
self.sort_dict(sb, max_row_count, self.sql_log_dic)
print datetime.datetime.now().isoformat().replace('T',
' '), fs, 'sort ok'
def do_executeRequest(self, fs, sb, date_from_ms, date_to_ms, match_user, match_url):
for f in fs:
lines = file(f).readlines()
print datetime.datetime.now().isoformat().replace('T', ' '), f, 'readline ok'
if not analyse_qbrm_log.is_effective_log_file(lines, date_from_ms, date_to_ms):
print 'ignore ', f
continue
for line in lines:
line = ui_public.get_line(line, log_file_encode)
log_infos = line.split('|', 5)
if len(log_infos) < 6:
continue
if match_user != None and match_user != "":
uuid_thread_user = log_infos[3].split('.', 3)
if len(uuid_thread_user) < 3 or not(re.match(match_user, uuid_thread_user[2])):
continue
epoch = (long)(log_infos[0])
if date_from_ms > 0 and epoch < date_from_ms:
continue
if date_to_ms > 0 and epoch > date_to_ms:
continue
self.valid_row_count += 1
request_url = log_infos[4]
total_time = (long)(log_infos[1][0:log_infos[1].index(',')])
sql_time = (long)(log_infos[1][log_infos[1].index(',') + 1:])
#sql count
sql_counts = log_infos[2]
sql_count_single = sql_counts[0 : sql_counts.index(',')]
sql_counts = sql_counts[sql_counts.index(',')+1 : ]
sql_count_batch_sum = sql_counts[sql_counts.index(',')+1 : ]
total_sql_count = long(sql_count_single) + long(sql_count_batch_sum)
if self.log_dic.__contains__(request_url):
self.log_dic[request_url][1].append(total_time)
self.log_dic[request_url][2].append(sql_time)
self.log_dic[request_url][3].append(total_sql_count)
else:
request_uri = request_url
if request_url.find('?') < 0 and len(log_infos[5]) > 0:
request_uri += '?'
request_uri += log_infos[5]
self.log_dic[request_url] = [request_uri, [total_time], [sql_time], [total_sql_count]]
print datetime.datetime.now().isoformat().replace('T', ' '), f, 'merge ok'
def statisticSqlExecuteTime(self, fs, sb, date_from_ms, date_to_ms):
for f in fs:
sys.setrecursionlimit(sys.maxint)
#1 begin read()读入，全部匹配
#2 begin 分行载入，逐条匹配
#3 begin 全部载入，字符串分析
lines = file(f).readlines()
print datetime.datetime.now().isoformat().replace('T', ' '), f, 'readline ok'
if not analyse_qbrm_log.is_effective_log_file(lines, date_from_ms, date_to_ms):
print 'ignore ', f
continue
for line in lines:
line = ui_public.get_line(line, log_file_encode)
sql_log = line.split('|', 5)
if len(sql_log) < 6 or sql_log[3] != 's':
continue
epoch = (long)(sql_log[0])
if date_from_ms > 0 and epoch < date_from_ms:
continue
if date_to_ms > 0 and epoch > date_to_ms:
continue
self.valid_row_count += 1
strsql_merge = self.get_abstract_sql(sql_log[4], sql_log[5])
if self.sql_log_dic.__contains__(strsql_merge):
sql_times = self.sql_log_dic[strsql_merge][1]
sql_times.append(int(sql_log[1]))
else:
statement_type = ''
if len(sql_log[4]) == 0:
statement_type = 's'
self.sql_log_dic[strsql_merge] = [sql_log[5], [int(sql_log[1])], statement_type]
#end
print datetime.datetime.now().isoformat().replace('T', ' '), f, 'merge ok'
self.sort_dict(sb, max_row_count, self.sql_log_dic)
print datetime.datetime.now().isoformat().replace('T', ' '), fs, 'sort ok'