示例#1
0
 def do_executeMethod(self, fs, sb, date_from_ms, date_to_ms):
     for f in fs:
         lines = file(f).readlines()
         print datetime.datetime.now().isoformat().replace("T", " "), f, "readline ok"
         if not analyse_qbrm_log.is_effective_log_file(lines, date_from_ms, date_to_ms):
             print "ignore ", f
             continue
         for line in lines:
             line = ui_public.get_line(line, log_file_encode)
             log_infos = line.split("|", 5)
             if len(log_infos) < 6:
                 continue
             epoch = (long)(log_infos[0])
             if date_from_ms > 0 and epoch < date_from_ms:
                 continue
             if date_to_ms > 0 and epoch > date_to_ms:
                 continue
             self.valid_row_count += 1
             class_method = log_infos[4] + "." + log_infos[5]
             execute_time = long(log_infos[1])
             return_size = 0
             if len(log_infos[2]) > 0:
                 return_size = long(log_infos[2])
             if self.log_dic.__contains__(class_method):
                 self.log_dic[class_method][1].append(execute_time)
                 self.log_dic[class_method][2].append(return_size)
             else:
                 self.log_dic[class_method] = [class_method, [execute_time], [return_size]]
         print datetime.datetime.now().isoformat().replace("T", " "), f, "merge ok"
示例#2
0
 def find_uuids(self, fs, date_from_ms, date_to_ms):
     uuids = {}
     for f in fs:
         lines = file(f).readlines()
         print datetime.datetime.now().isoformat().replace('T', ' '), f, 'readline ok'
         if not analyse_qbrm_log.is_effective_log_file(lines, date_from_ms, date_to_ms):
             print 'ignore ', f
             continue
         for line in lines:
             line = ui_public.get_line(line, log_file_encode)
             log_infos = line.split('|', 5)
             if len(log_infos) < 6:
                 continue
             epoch = (long)(log_infos[0])
             if date_from_ms > 0 and epoch < date_from_ms:
                 continue
             if date_to_ms > 0 and epoch > date_to_ms:
                 continue
             request_url = log_infos[4]
             if request_url.find('?') < 0 and len(log_infos[5]) > 0:
                 request_url += '?'
             request_url += log_infos[5]
             if request_url == self.url:
                 uuids[log_infos[3][0:log_infos[3].find('.')]] = log_infos[1][0:log_infos[1].find(',')]
             if len(uuids) > max_row:
                 break
         if len(uuids) > max_row:
             break
     return uuids
示例#3
0
 def statisticErrorSql(self, fs, sb, date_from_ms, date_to_ms):
     sb.append('\n<h4>执行错误的SQL:</h4>')
     error_sql_dic = {}
     for f in fs:
         lines = file(f).readlines()
         print datetime.datetime.now().isoformat().replace(
             'T', ' '), f, 'readline ok'
         if not analyse_qbrm_log.is_effective_log_file(
                 lines, date_from_ms, date_to_ms):
             print 'ignore ', f
             continue
         for line in lines:
             line = ui_public.get_line(line, log_file_encode)
             sql_log = line.split('|', 6)
             if len(sql_log) < 7 or sql_log[3] != 's':
                 continue
             epoch = (long)(sql_log[0])
             if date_from_ms > 0 and epoch < date_from_ms:
                 continue
             if date_to_ms > 0 and epoch > date_to_ms:
                 continue
             self.valid_row_count += 1
             key = sql_log[5] + sql_log[6]
             if error_sql_dic.__contains__(key):
                 error_sql_dic[key][2] = error_sql_dic[key][2] + 1
             else:
                 error_sql_dic[key] = [sql_log[5], sql_log[6], 1]
         print datetime.datetime.now().isoformat().replace(
             'T', ' '), f, 'merge ok'
     for error_sql in error_sql_dic.items():
         sb.append('\n' + str(error_sql[1][2]) + '次, ' + error_sql[1][0] +
                   ' ' + error_sql[1][1])
示例#4
0
 def statisticErrorSql(self, fs, sb, date_from_ms, date_to_ms):
     sb.append( '\n<h4>执行错误的SQL:</h4>')
     error_sql_dic = {}
     for f in fs:
         lines = file(f).readlines()
         print datetime.datetime.now().isoformat().replace('T', ' '), f, 'readline ok'
         if not analyse_qbrm_log.is_effective_log_file(lines, date_from_ms, date_to_ms):
             print 'ignore ', f
             continue
         for line in lines:
             line = ui_public.get_line(line, log_file_encode)
             sql_log = line.split('|', 6)
             if len(sql_log) < 7 or sql_log[3] != 's':
                 continue
             epoch = (long)(sql_log[0])
             if date_from_ms > 0 and epoch < date_from_ms:
                 continue
             if date_to_ms > 0 and epoch > date_to_ms:
                 continue
             self.valid_row_count += 1
             key = sql_log[5] + sql_log[6]
             if error_sql_dic.__contains__(key):
                 error_sql_dic[key][2] = error_sql_dic[key][2] + 1
             else:
                 error_sql_dic[key] = [sql_log[5], sql_log[6], 1]
         print datetime.datetime.now().isoformat().replace('T', ' '), f, 'merge ok'
     for error_sql in error_sql_dic.items():
         sb.append('\n' + str(error_sql[1][2]) + '次, ' + error_sql[1][0] + ' ' + error_sql[1][1])
示例#5
0
 def find_uuids(self, fs, date_from_ms, date_to_ms):
     uuids = {}
     for f in fs:
         lines = file(f).readlines()
         print datetime.datetime.now().isoformat().replace('T', ' '), f, 'readline ok'
         if not analyse_qbrm_log.is_effective_log_file(lines, date_from_ms, date_to_ms):
             print 'ignore ', f
             continue
         for line in lines:
             line = ui_public.get_line(line, log_file_encode)
             log_infos = line.split('|', 5)
             if len(log_infos) < 6:
                 continue
             epoch = (long)(log_infos[0])
             if date_from_ms > 0 and epoch < date_from_ms:
                 continue
             if date_to_ms > 0 and epoch > date_to_ms:
                 continue
             request_url = log_infos[4]
             if request_url.find('?') < 0 and len(log_infos[5]) > 0:
                 request_url += '?'
             request_url += log_infos[5]
             if request_url == self.url:
                 uuids[log_infos[3][0:log_infos[3].find('.')]] = log_infos[1][0:log_infos[1].find(',')]
             if len(uuids) > max_row:
                 break
         if len(uuids) > max_row:
             break
     return uuids
 def do_executeMethod(self, fs, sb, date_from_ms, date_to_ms):
     for f in fs:
         lines = file(f).readlines()
         print datetime.datetime.now().isoformat().replace(
             'T', ' '), f, 'readline ok'
         if not analyse_qbrm_log.is_effective_log_file(
                 lines, date_from_ms, date_to_ms):
             print 'ignore ', f
             continue
         for line in lines:
             line = ui_public.get_line(line, log_file_encode)
             log_infos = line.split('|', 5)
             if len(log_infos) < 6:
                 continue
             epoch = (long)(log_infos[0])
             if date_from_ms > 0 and epoch < date_from_ms:
                 continue
             if date_to_ms > 0 and epoch > date_to_ms:
                 continue
             self.valid_row_count += 1
             class_method = log_infos[4] + '.' + log_infos[5]
             execute_time = long(log_infos[1])
             return_size = 0
             if len(log_infos[2]) > 0:
                 return_size = long(log_infos[2])
             if self.log_dic.__contains__(class_method):
                 self.log_dic[class_method][1].append(execute_time)
                 self.log_dic[class_method][2].append(return_size)
             else:
                 self.log_dic[class_method] = [
                     class_method, [execute_time], [return_size]
                 ]
         print datetime.datetime.now().isoformat().replace(
             'T', ' '), f, 'merge ok'
示例#7
0
 def do_executeRequest(self, fs, sb, date_from_ms, date_to_ms, match_user,
                       match_url):
     for f in fs:
         lines = file(f).readlines()
         print datetime.datetime.now().isoformat().replace(
             'T', ' '), f, 'readline ok'
         if not analyse_qbrm_log.is_effective_log_file(
                 lines, date_from_ms, date_to_ms):
             print 'ignore ', f
             continue
         for line in lines:
             line = ui_public.get_line(line, log_file_encode)
             log_infos = line.split('|', 5)
             if len(log_infos) < 6:
                 continue
             if match_user != None and match_user != "":
                 uuid_thread_user = log_infos[3].split('.', 3)
                 if len(uuid_thread_user) < 3 or not (re.match(
                         match_user, uuid_thread_user[2])):
                     continue
             epoch = (long)(log_infos[0])
             if date_from_ms > 0 and epoch < date_from_ms:
                 continue
             if date_to_ms > 0 and epoch > date_to_ms:
                 continue
             self.valid_row_count += 1
             request_url = log_infos[4]
             total_time = (long)(log_infos[1][0:log_infos[1].index(',')])
             sql_time = (long)(log_infos[1][log_infos[1].index(',') + 1:])
             #sql count
             sql_counts = log_infos[2]
             sql_count_single = sql_counts[0:sql_counts.index(',')]
             sql_counts = sql_counts[sql_counts.index(',') + 1:]
             sql_count_batch_sum = sql_counts[sql_counts.index(',') + 1:]
             total_sql_count = long(sql_count_single) + long(
                 sql_count_batch_sum)
             if self.log_dic.__contains__(request_url):
                 self.log_dic[request_url][1].append(total_time)
                 self.log_dic[request_url][2].append(sql_time)
                 self.log_dic[request_url][3].append(total_sql_count)
             else:
                 request_uri = request_url
                 if request_url.find('?') < 0 and len(log_infos[5]) > 0:
                     request_uri += '?'
                 request_uri += log_infos[5]
                 self.log_dic[request_url] = [
                     request_uri, [total_time], [sql_time],
                     [total_sql_count]
                 ]
         print datetime.datetime.now().isoformat().replace(
             'T', ' '), f, 'merge ok'
示例#8
0
 def find_methods(self, fs, date_from_ms, date_to_ms, uuids):
     methods = []
     for f in fs:
         lines = file(f).readlines()
         print datetime.datetime.now().isoformat().replace('T', ' '), f, 'readline ok'
         if not analyse_qbrm_log.is_effective_log_file(lines, date_from_ms, date_to_ms):
             print 'ignore ', f
             continue
         for line in lines:
             line = ui_public.get_line(line, log_file_encode)
             for uuid_key in uuids.keys():
                 if line.find(uuid_key) > -1:
                     methods.append(line)
     return methods
示例#9
0
 def find_methods(self, fs, date_from_ms, date_to_ms, uuids):
     methods = []
     for f in fs:
         lines = file(f).readlines()
         print datetime.datetime.now().isoformat().replace('T', ' '), f, 'readline ok'
         if not analyse_qbrm_log.is_effective_log_file(lines, date_from_ms, date_to_ms):
             print 'ignore ', f
             continue
         for line in lines:
             line = ui_public.get_line(line, log_file_encode)
             for uuid_key in uuids.keys():
                 if line.find(uuid_key) > -1:
                     methods.append(line)
     return methods
示例#10
0
    def statisticSqlExecuteTime(self, fs, sb, date_from_ms, date_to_ms):
        for f in fs:
            sys.setrecursionlimit(sys.maxint)
            #1 begin read()读入,全部匹配

            #2 begin 分行载入,逐条匹配

            #3 begin 全部载入,字符串分析
            lines = file(f).readlines()
            print datetime.datetime.now().isoformat().replace(
                'T', ' '), f, 'readline ok'
            if not analyse_qbrm_log.is_effective_log_file(
                    lines, date_from_ms, date_to_ms):
                print 'ignore ', f
                continue
            for line in lines:
                line = ui_public.get_line(line, log_file_encode)
                sql_log = line.split('|', 5)
                if len(sql_log) < 6 or sql_log[3] != 's':
                    continue
                epoch = (long)(sql_log[0])
                if date_from_ms > 0 and epoch < date_from_ms:
                    continue
                if date_to_ms > 0 and epoch > date_to_ms:
                    continue
                self.valid_row_count += 1
                strsql_merge = self.get_abstract_sql(sql_log[4], sql_log[5])
                if self.sql_log_dic.__contains__(strsql_merge):
                    sql_times = self.sql_log_dic[strsql_merge][1]
                    sql_times.append(int(sql_log[1]))
                else:
                    statement_type = ''
                    if len(sql_log[4]) == 0:
                        statement_type = 's'
                    self.sql_log_dic[strsql_merge] = [
                        sql_log[5], [int(sql_log[1])], statement_type
                    ]
            #end

            print datetime.datetime.now().isoformat().replace(
                'T', ' '), f, 'merge ok'
        self.sort_dict(sb, max_row_count, self.sql_log_dic)
        print datetime.datetime.now().isoformat().replace('T',
                                                          ' '), fs, 'sort ok'
示例#11
0
 def do_executeRequest(self, fs, sb, date_from_ms, date_to_ms, match_user, match_url):
     for f in fs:
         lines = file(f).readlines()
         print datetime.datetime.now().isoformat().replace('T', ' '), f, 'readline ok'
         if not analyse_qbrm_log.is_effective_log_file(lines, date_from_ms, date_to_ms):
             print 'ignore ', f
             continue
         for line in lines:
             line = ui_public.get_line(line, log_file_encode)
             log_infos = line.split('|', 5)
             if len(log_infos) < 6:
                 continue
             if match_user != None and match_user != "":
                 uuid_thread_user = log_infos[3].split('.', 3)
                 if len(uuid_thread_user) < 3  or not(re.match(match_user, uuid_thread_user[2])):
                     continue
             epoch = (long)(log_infos[0])
             if date_from_ms > 0 and epoch < date_from_ms:
                 continue
             if date_to_ms > 0 and epoch > date_to_ms:
                 continue
             self.valid_row_count += 1
             request_url = log_infos[4]            
             total_time = (long)(log_infos[1][0:log_infos[1].index(',')])
             sql_time = (long)(log_infos[1][log_infos[1].index(',') + 1:])
             #sql count
             sql_counts = log_infos[2]
             sql_count_single = sql_counts[0 : sql_counts.index(',')]
             sql_counts = sql_counts[sql_counts.index(',')+1 : ]
             sql_count_batch_sum = sql_counts[sql_counts.index(',')+1 : ]
             total_sql_count = long(sql_count_single) + long(sql_count_batch_sum)
             if self.log_dic.__contains__(request_url):
                 self.log_dic[request_url][1].append(total_time)
                 self.log_dic[request_url][2].append(sql_time)
                 self.log_dic[request_url][3].append(total_sql_count)
             else:
                 request_uri = request_url
                 if request_url.find('?') < 0 and len(log_infos[5]) > 0:
                     request_uri += '?'
                 request_uri += log_infos[5]
                 self.log_dic[request_url] = [request_uri, [total_time], [sql_time], [total_sql_count]]
         print datetime.datetime.now().isoformat().replace('T', ' '), f, 'merge ok'
示例#12
0
    def statisticSqlExecuteTime(self, fs, sb, date_from_ms, date_to_ms):
        for f in fs:
            sys.setrecursionlimit(sys.maxint)
            #1 begin read()读入,全部匹配            

            #2 begin 分行载入,逐条匹配
            
            #3 begin 全部载入,字符串分析
            lines = file(f).readlines()
            print datetime.datetime.now().isoformat().replace('T', ' '), f, 'readline ok'
            if not analyse_qbrm_log.is_effective_log_file(lines, date_from_ms, date_to_ms):
                print 'ignore ', f
                continue
            for line in lines:
                line = ui_public.get_line(line, log_file_encode)
                sql_log = line.split('|', 5)
                if len(sql_log) < 6 or sql_log[3] != 's':
                    continue
                epoch = (long)(sql_log[0])
                if date_from_ms > 0 and epoch < date_from_ms:
                    continue
                if date_to_ms > 0 and epoch > date_to_ms:
                    continue
                self.valid_row_count += 1
                strsql_merge = self.get_abstract_sql(sql_log[4], sql_log[5])
                if self.sql_log_dic.__contains__(strsql_merge):
                    sql_times = self.sql_log_dic[strsql_merge][1]
                    sql_times.append(int(sql_log[1]))
                else:
                    statement_type = ''
                    if len(sql_log[4]) == 0:
                        statement_type = 's'
                    self.sql_log_dic[strsql_merge] = [sql_log[5], [int(sql_log[1])], statement_type]
            #end
            
            print datetime.datetime.now().isoformat().replace('T', ' '), f, 'merge ok'
        self.sort_dict(sb, max_row_count, self.sql_log_dic)
        print datetime.datetime.now().isoformat().replace('T', ' '), fs, 'sort ok'