def __del__(self): logging.debug("SIPSMsgParser __del__") self.in_shutdown = True if (self.in_cs_msg): self.submit_cs_message() # go through all caches and submit unprocessed messages as-is # clients/requests cache for clientid in self.d_cs_clients_msgs: for refid in self.d_cs_clients_msgs[clientid]: for typeid in self.d_cs_clients_msgs[clientid][refid]: self.d_cs_msg = self.d_cs_clients_msgs[clientid][refid][ typeid] self.d_cs_msg[ 'is_failed'] = -1 # we dont know, no information in log self.submit_cs_message() #external auth messages for endpoint in self.d_cs_exta_intops: for typeid in self.d_cs_exta_intops[endpoint]: self.d_cs_msg = self.d_cs_exta_intops[endpoint][typeid] self.d_cs_msg[ 'is_failed'] = -1 # we dont know, no information in log self.submit_cs_message() LogParser.__del__(self) return
def seachHtmlFile(self,curpath=None,filename=None,methodlist=None,devices=None): listcontent=[] _failCount=0 _sucessCount=0 failCase=[] content=None dictcontent={} if not os.path.exists(curpath): print "%s路径不存在"%curpath for root,dirs,files in os.walk(curpath,True): if -1!=root.find(filename): print root for item in files: path=os.path.join(root,item) if -1!=path.find(filename): if os.path.splitext(item)[1]=='.html': testcase=os.path.split(path)[0].split('\\')[-1] listcontent=self.parseHtml(path) log=LogParser() _str=log.parser('%s\\%s\\log.html'%(curpath,testcase)) if _str=='crash': _failCount=_failCount+1 dictcontent['%s'%testcase]='crash' else: if len(listcontent)>0: _failCount=_failCount+1 content=listcontent[0] dictcontent['%s'%testcase]=content else: content='Success' dictcontent['%s'%testcase]=content _sucessCount=_sucessCount+1 self.headHtml('%s\sumHtml.html'%curpath) for device in devices: total=0 _fail=0 _success=0 for key in dictcontent.keys(): if device in key: total=total+1 if dictcontent[key]!='Success': _fail=_fail+1 failCase.append(key) else: _success=_success+1 self.titleHtml('%s\sumHtml.html'%curpath,device) self.getSummaryHtml(curpath,'%s\sumHtml.html'%curpath,total,_success,_fail,failCase,device) for device in devices: dic={} for key in dictcontent.keys(): if device in key: dic['%s'%key]=dictcontent[key] if dictcontent[key]!='Success': self.failDetailHtml(curpath,'%s\\%s_FailHtml.html'%(curpath,device),dic,device) self.titleHtml('%s\sumHtml.html'%curpath,device) self.getDetailHtml(curpath,'%s\sumHtml.html'%curpath,dic,device) self.endHtml('%s\sumHtml.html'%curpath)
def ParsePQATAndPrintTable(self, logName): metrics = ["HPWL", "TNS", "WNS"] parser = LogParser(logName, TableType.PQAT, self.cfgParser) table = parser.ParsePQAT(metrics) table = MakeTableInPercents(table) pqatName = r"%s.csv" % (os.path.basename(logName)) PQATFileName = os.path.join(os.path.dirname(logName), pqatName) PrintTableToFile(PQATFileName, table, metrics)
def __init__(self, submitter, tags={}): logging.debug("SIPSMsgParser __init__") LogParser.__init__(self, submitter, tags) # buffer self.sip_msg = '' # dictionary for SIP msg self.d_sip_msg = {} # bool we are in sip msg self.in_sip_msg = 0
def __init__(self,submitter,tags={}): logging.debug("TLibMsgParser __init__") LogParser.__init__(self, submitter,tags) # buffer self.tlib_msg = '' # dictionary for SIP msg self.d_tlib_msg = {} # bool we are in sip msg self.in_tlib_msg = 0
def __init__(self, submitter, tags={}): logging.debug("TLibMsgParser __init__") LogParser.__init__(self, submitter, tags) # buffer self.tlib_msg = '' # dictionary for SIP msg self.d_tlib_msg = {} # bool we are in sip msg self.in_tlib_msg = 0 self.has_ConnID = False self.has_ThisDN = False
def PrintAndPlotPFST(logName, metrics, stages): parser = LogParser(logName) table = parser.ParsePFST(metrics, stages) if table == []: print("Error: table is empty") return table = MakeTableInPercents(table) PrintTableToFile(r"%s.csv" % logName, table, metrics, stages) del table[0] #don"t use values of 0 iteration [xValues, yValues] = ExtractXYFromTable(table) PlotChartForBenchmark(logName, xValues, "TNS", yValues, "HPWL")
def failDetailHtml(self,curpath,filename,dictcontent,device): log=LogParser() f=open(filename,'w') str='<html>\ <head>\ <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />\ <title>Test</title>\ <style>img{float:left;margin:5px;}</style>' str+='</head>\ <body align=center>' str+='<center><h1>失败测试报告</h1>' str+= '<table border=1 borderColor=#EAEAEA width=100% style=border-collapse:collapse>\ <tr>\ <th width=350>用例名称</th>\ <th>结果</th>\ </tr>' for key in dictcontent.keys(): ## _str=log.parser('%s\\%s\\log.html'%(curpath,key)) case=key[0:len(key)-len(device)-1] str += '<tr>\ <th><a href="./%s/report.html">%s</a></th>'%(key,case) if dictcontent[key]!="Success": if dictcontent[key]=="crash": str+='<th style=color:#FF0033 id="e">%s:<a href="./%s/log.html">System.out</a></th></tr>'%(dictcontent[key],key) else: str+='<th style=color:#FF0033 id="e">Exception:%s</th></tr>'%(dictcontent[key]) else: str+='<th id="e">%s</th></tr>'%(dictcontent[key])
def __init__(self, submitter, tags={}): logging.debug("CSMsgParser __init__") LogParser.__init__(self, submitter, tags) # buffer self.cs_msg = '' # dictionary for current CS msg self.d_cs_msg = {} # dictonary for previous CS message TODO: do we need a full stack ? self.d_cs_msg_stack = [] # bool we are in CS msg self.in_cs_msg = 0 # we are handling termination (cleanup caches) self.in_shutdown = False # dictionary for messages per clients that are currently pending future processing self.d_cs_clients_msgs = {} # aux dictionary for ext auth module requests by internal reqid_exta (linked to d_cs_clients_msgs self.d_cs_exta_msgs = {} # dictonary for ext auth operations messages by ext auth endpoint self.d_cs_exta_intops = {}
def openDirectory(self): folder = str( QFileDialog.getExistingDirectory(self, "Select Directory", "./")) if not folder: return logmerger = LogMerger(folder) logparser = LogParser(folder) self.events = logparser.getEvents() self.logs = logparser.getLogs() # for event in self.events: # print(event.getEventName()) # print(event.getEventSymbol()) # print(event.getEventLogs()) # Event Checkbox init self.eventCheckboxInit() # device MAC Label init self.ui.deviceMACLabel.setText(logparser.getDeviceMAC())
def getDetailHtml(self,curpath,filename,dictcontent,device): log=LogParser() f=open(filename,'a') str = '<table border=1 borderColor=#EAEAEA width=100% style=border-collapse:collapse>\ <tr>\ <th width=350>用例名称</th>\ <th>结果</th>\ </tr>' for key in dictcontent.keys(): ## _str=log.parser('%s\\%s\\log.html'%(curpath,key)) case=key[0:len(key)-len(device)-1] str += '<tr>\ <th><a href="./%s/report.html">%s</a></th>'%(key,case) if dictcontent[key]!="Success": if dictcontent[key]=="crash": str+='<th style=color:#FF0033 id="e">%s:<a href="./%s/log.html">System.out</a></th></tr>'%(dictcontent[key],key) else: str+='<th style=color:#FF0033 id="e">Exception:%s</th></tr>'%(dictcontent[key]) else: str+='<th id="e">%s</th></tr>'%(dictcontent[key])
from utilsCommon import sys_oper from utilsCommon import sys_base from utilsCommon import Config_agent from auditOper import auditOper import os import pyinotify import threading import time import ctypes import sys from datetime import datetime import commands import hashlib from lib_dec import dec_lib logparser = LogParser() config = ConfigInfor() operpaser = OperParser() stateInfo = StateInfo() class OnIOHandler(pyinotify.ProcessEvent): def process_IN_MODIFY(self, event): global g_EventListSet, mutex if event.maskname == 'IN_MODIFY': if mutex.acquire(5): g_EventListSet.add(event.pathname) mutex.release() class inotify_log(threading.Thread):
def __init__(self,submitter,tags={}): LogParser.__init__(self, submitter,tags) # dictionary for SIP msg self.d_std_msg = {}
def ParseLog(self, logName): parser = LogParser(logName, TableType.PFST, self.cfgParser) return parser.ParsePFST(self.metrics, self.stages)
def seachHtmlFile(self,curpath=None,filename=None,methodlist=None,devices=None): listcontent=[] _failCount=0 _sucessCount=0 failCase=[] content=None dictcontent={} if not os.path.exists(curpath): print "%s路径不存在"%curpath for root,dirs,files in os.walk(curpath,True): if -1!=root.find(filename): print root for item in files: path=os.path.join(root,item) if -1!=path.find(filename): if os.path.splitext(item)[1]=='.html': testcase=os.path.split(path)[0].split('\\')[-1] listcontent=self.parseHtml(path) log=LogParser() _str=log.parser('%s\\%s\\log.html'%(curpath,testcase)) if _str=='crash': _failCount=_failCount+1 dictcontent['%s'%testcase]='crash' else: if len(listcontent)>0: _failCount=_failCount+1 content=listcontent[0] dictcontent['%s'%testcase]=content else: content='Success' dictcontent['%s'%testcase]=content _sucessCount=_sucessCount+1 self.headHtml('%s\sumHtml.html'%curpath) for device in devices: total=0 _fail=0 _success=0 for key in dictcontent.keys(): if device in key: total=total+1 if dictcontent[key]!='Success': _fail=_fail+1 failCase.append(key) else: _success=_success+1 self.titleHtml('%s\sumHtml.html'%curpath,device) self.getSummaryHtml(curpath,'%s\sumHtml.html'%curpath,total,_success,_fail,failCase,device) for device in devices: dic={} dicf={} for key in dictcontent.keys(): if device in key: dic['%s'%key]=dictcontent[key] if dictcontent[key]!='Success': dicf['%s'%key]=dictcontent[key] self.failDetailHtml(curpath,'%s\\%s_FailHtml.html'%(curpath,device),dicf,device) self.titleHtml('%s\sumHtml.html'%curpath,device) self.getDetailHtml(curpath,'%s\sumHtml.html'%curpath,dic,device) self.endHtml('%s\sumHtml.html'%curpath)
def __del__(self): logging.debug("SIPSMsgParser __del__") if (self.in_sip_msg): self.submit_sip_message() LogParser.__del__(self) return
from InputFileValidator import InputFileValidator from LogParser import LogParser import argparse if __name__ == '__main__': parser = argparse.ArgumentParser(description='This is a Python version of grep function.') parser.add_argument('name_of_file', type=str, help='an name of the file to check') parser.add_argument('string_to_search', type=str, help='full phrase to search in the file') args = parser.parse_args() name_of_file = args.name_of_file string_to_search = args.string_to_search file_validator = InputFileValidator(name_of_file) if file_validator.validate(): log_parser = LogParser(file_validator.get_list_of_files(), string_to_search) print(log_parser.parselog())
def __del__(self): logging.debug("TLibSMsgParser __del__") if (self.in_tlib_msg): self.submit_tlib_message() LogParser.__del__(self) return
def __del__(self): logging.debug("SIPSMsgParser __del__") if(self.in_sip_msg): self.submit_sip_message() LogParser.__del__(self) return
import matplotlib.pyplot as plt import matplotlib.animation as animation from matplotlib import style from LogParser import LogParser style.use('fivethirtyeight') fig = plt.figure() ax1 = fig.add_subplot(1, 1, 1) old_f = [] lp = LogParser('config.ini') def animate(i): global old_f with open('history.log', 'r') as f: new_f = f.readlines() if new_f != old_f: old_f = new_f[:] lp.add_log(old_f) lp.watson_report_cumulative() # Isolate x axis x = lp.axes['x'] ax1.clear() for name, things in lp.axes.items(): if name == 'x': continue print(lp.axes) plt.plot(x, things, label=name) plt.legend(loc=2, prop={'size': 7})
import os import matplotlib.pyplot as plt from LogParser import LogParser from InputFileValidator import InputFileValidator InputFileValidator.validate(LogParser.get_parser()) data = LogParser.parse_log(LogParser.get_parser(), "PrChecker.Downs")[1:] data_to_strings = [] iterator = 0 x_buffer = [] y_buffer = [] for buffer in data: part = buffer.split() for atom in part: if atom.isdigit(): data_to_strings.append(atom) for string in data_to_strings: if iterator == 0: x_buffer.append(string) iterator += 1 elif iterator == 1: y_buffer.append(string) iterator += 1 else: iterator = 0 LogParser.save_result( "./results/results", LogParser.parse_log(LogParser.get_parser(), "PrChecker.Downs"))
def __del__(self): logging.debug("TLibSMsgParser __del__") if(self.in_tlib_msg): self.submit_tlib_message() LogParser.__del__(self) return
from MsgWrap import OperMsgParser from LogParser import LogParser from utilsCommon import pf_oper from utilsCommon import pf_base from utilsCommon import PrntLog from utilsCommon import judge_ip_localhost from utilsCommon import get_netcard import os import crypt import random, string import threading import traceback opermsgpaser = OperMsgParser() logparser = LogParser() gCreateAccountrSet = set() class OperParser(object): def init_gCreateAccountrSet(self): ret = os.path.exists('AddUserList.conf') if ret: global gCreateAccountrSet file = open('AddUserList.conf', 'r') for line in file.readlines(): gCreateAccountrSet.add(line.rstrip('\n')) file.close() def OperParserMsg(self, str): try:
def main(arguments: Optional[List[str]] = None) -> None: def parse_ignored_ips(x: str) -> List[ipaddress.IPv4Network]: return [ ipaddress.ip_network(address, strict=False) for address in x.split(',') ] parser = argparse.ArgumentParser(description='Process log files.') group = parser.add_mutually_exclusive_group(required=True) group.add_argument('--realtime', '--interactive', '-i', '-r', action='store_true', help='Watch a single log file in realtime') group.add_argument( '--batch', '-b', action='store_true', help='Print a report on one or more completed log files') group.add_argument('--slug-summary', action='store_true', dest='show_slugs', help="Show the slugs that have been used in a log file") group.add_argument('--xxfake', action='store_true', help=argparse.SUPPRESS, dest='fake') # For testing only group2 = parser.add_mutually_exclusive_group() group2.add_argument('--by-ip', action='store_true', dest='by_ip', help='Sorts batched logs by host ip') group2.add_argument('--by-time', action='store_false', dest='by_ip', help='Sorts batched logs by session start time') parser.add_argument('--html', action='store_true', dest='uses_html', help='Generate html output rather than text output') parser.add_argument('--api-host-url', default=DEFAULT_FIELDS_PREFIX, metavar='URL', dest='api_host_url', help='base url to access the information') parser.add_argument('--reverse-dns', '--dns', action='store_true', dest='uses_reverse_dns', help='Attempt to resolve the real host name') parser.add_argument( '--ignore-ip', '-x', default=[], action="append", metavar='cidrlist', dest='ignore_ip', type=parse_ignored_ips, help='list of ips to ignore. May be specified multiple times') parser.add_argument( '--session-timeout', default=60, type=int, metavar="minutes", dest='session_timeout_minutes', help='a session ends after this period (minutes) of inactivity') parser.add_argument('--output', '-o', type=argparse.FileType('w'), default=sys.stdout, dest='output', help="output file. default is stdout") # TODO(fy): Temporary hack for when I don't have internet access parser.add_argument('--xxlocal', action="store_true", dest="uses_local", help=argparse.SUPPRESS) parser.add_argument('log_files', nargs=argparse.REMAINDER, help='log files') args = parser.parse_args(arguments) slugs = Slug.ToInfoMap( LOCAL_SLUGS_PREFIX if args.uses_local else args.api_host_url) # args.ignored_ip comes out as a list of lists, and it needs to be flattened. ignored_ips = [ip for arg_list in args.ignore_ip for ip in arg_list] session_info_generator = SessionInfoGenerator(slugs, ignored_ips) log_parser = LogParser(session_info_generator, **vars(args)) if args.realtime: if len(args.log_files) != 1: raise Exception("Must specify exactly one file for batch mode.") log_entries_realtime = LogReader.read_logs_from_tailed_file( args.log_files[0]) log_parser.run_realtime(log_entries_realtime) else: if len(args.log_files) < 1: raise Exception("Must specify at least one log file.") log_entries_list = LogReader.read_logs(args.log_files) if args.batch: log_parser.run_batch(log_entries_list) elif args.show_slugs: log_parser.show_slugs(log_entries_list) elif args.fake: log_entries_list.sort(key=operator.attrgetter('time')) log_parser.run_realtime(iter(log_entries_list))
when argument "port" is not included, the default port would be 3306\n But other arguments "host", "user", "passwd", "db" are required.''', action='append') parser.add_argument('-s', help='the OS where the chat log came from. "android" and "iOS" is available.') parser.add_argument('-v', '--version', action='version', version='%(prog)s 0.11') startOptions = parser.parse_args() dbConfig = {} if startOptions.db: for db_arg in startOptions.e: leftside, rightside = parseExpression(db_arg) dbConfig[leftside] = rightside if 'PORT' not in dbConfig.keys(): dbConfig['PORT'] = 3306 for necessary in ['HOST', 'USER', 'PASSWD', 'DB']: if necessary not in dbConfig.keys(): print('[Error]Invalid Argument:: "%s" variable not in Database Config' % necessary) exit(-1) parser = LogParser(system=startOptions.s if startOptions.s else 'android', topKw=100) data = parser.process(startOptions.source) if startOptions.db: saveToDB(data, dbConfig) else: print(json.dumps(data))
def test_parse(self): LogParser("./test_log.txt", ".", "@ERROR_GROUP") with open("./test_log.txt_err_timestamps.txt") as f: lines = f.readlines() self.assertEquals(len(lines), 1) self.assertEquals(lines[0], "1502443757149")
def __init__(self, submitter, tags={}): LogParser.__init__(self, submitter, tags) # dictionary for SIP msg self.d_std_msg = {}