def cfg_oainfo(self): """ 针对 MicroFocus ITOM OA 中的 agent.log_<日期> 的文件 """ # 初始化数据和相关控制参数 logdata = {} sqldata = [] log_num = 0 ########################################################################### # 判断 ovconfget 命令的开始 sta_ovconf = False # 判断 ovconfget 命令的结束 end_ovconf_mid = False # 第一次匹配 ****** 时做一次记录 end_ovconf = False # ovconfget 命令数据 agt_ovconf = 'Null' ########################################################################### # 判断 ovdeploy -inv -inclbdl -includeupdates 命令的开始 sta_ovdeploy = False # 判断 ovdeploy -inv -inclbdl -includeupdates 命令的结束 end_ovdeploy_mid = False # 第一次匹配 ****** 时做一次记录 end_ovdeploy = False # ovdeploy -inv -inclbdl -includeupdates 命令数据 agt_ovdeploy = 'Null' ########################################################################### # 与软件/系统相关的参数 os_name = 'Null' os_type = 'Null' os_version = 'Null' os_hostname = 'Null' agt_version = 'Null' agt_core_id = 'Null' os_machine = 'Null' # 尝试开始读取文件 try: with open(self.filepath, mode='r', encoding='utf-8', errors='replace') as f: for line in f: try: log_num += 1 if len(re.findall('Cmd executed.*ovconfget', line, re.IGNORECASE)) > 0 and sta_ovconf == False: sta_ovconf = True end_ovconf = False agt_ovconf = '' elif len(re.findall('\*{40}', line)) > 0 and sta_ovconf == True and end_ovconf == False: if end_ovconf_mid == False: end_ovconf_mid = True else: end_ovconf = True elif len(re.findall('\*{40}', line)) == 0 and sta_ovconf == True and end_ovconf == False: agt_ovconf = agt_ovconf + line if len(re.findall('agtversion=', line)) > 0: # OA 的版本 agt_version = line.strip().split('=')[-1].strip() elif len(re.findall('osname=', line)) > 0: # 系统发行版的名字 os_name = line.strip().split('=')[-1].strip() elif len(re.findall('ostype=', line)) > 0: # 系统类型 os_type = line.strip().split('=')[-1].strip() elif len(re.findall('osversion=', line)) > 0: # 系统内核版本 os_version = line.strip().split('=')[-1].strip() elif len(re.findall('CORE_ID=', line)) > 0: # OA 的 core id agt_core_id = line.strip().split('=')[-1].strip() elif len(re.findall('OPC_NODENAME=', line)) > 0: # 主机名 os_hostname = line.strip().split('=')[-1].strip() elif len(re.findall('ovdeploy -inv -inclbdl -includeupdates', line)) > 0 and sta_ovdeploy == False: sta_ovdeploy = True end_ovdeploy = False agt_ovdeploy = '' elif len(re.findall('\*{40}', line)) > 0 and sta_ovdeploy == True and end_ovdeploy ==False: if end_ovdeploy_mid == False: end_ovdeploy_mid = True else: end_ovdeploy = True elif len(re.findall('\*{40}', line)) == 0 and sta_ovdeploy == True and end_ovdeploy == False: agt_ovdeploy = agt_ovdeploy + line elif len(re.findall('Machine: ', line)) > 0: os_machine = line.strip().split(':')[-1].strip() except Exception as e: logSQLCreate.warning( "file:{}\nline:{}\nSource:{}\nException:{}".format(self.filepath, str(log_num), line, e)) # 整理获取的值 logdata['os_name'] = os_name.strip() logdata['os_type'] = os_type.strip() logdata['os_hostname'] = os_hostname.strip() logdata['os_version'] = os_version.strip() logdata['os_machine'] = os_machine.strip() logdata['agt_version'] = agt_version.strip() logdata['agt_core_id'] = agt_core_id.strip() logdata['agt_ovconf'] = agt_ovconf.strip() logdata['agt_ovdeploy'] = agt_ovdeploy.strip() for k, v in logdata.items(): try: sql_insert = 'INSERT INTO cfg_OAInfo (attribute, value) VALUES ("{}", "{}");'.format(k, v) sqldata.append(sql_insert) except Exception as e: logSQLCreate.warning("Can't generate SQL INSERT INTO statement! - {}".format(e)) self.SQLData = ({'db_name': self.db_name, 'db_type': self.db_type, 'db_table': 'cfg_OAInfo', 'db_data': sqldata, }) # 生成一个对应的 .lck 文件, 在数据写入完成后, 再删除 .lck 文件 datafilepath = r'./temp/{}'.format(self.file_id) open('{}.lck'.format(datafilepath), 'w').close() with open(datafilepath, 'wb') as f: pickle.dump(self.SQLData, f) os.remove('{}.lck'.format(datafilepath)) except Exception as reason: logSQLCreate.error('logfile read error:{}'.format(reason))
def log_trace(self): """ 针对 MicroFocus ITOM MP for Microsoft SQL Server 中的 trace 文件 """ # 初始化数据和相关控制参数 logdata = [] sqldata = [] isnoblk = True log_num = 0 isstart = False # 尝试开始读取文件 try: with open(self.filepath, mode='r', encoding='utf-8', errors='replace') as f: for line in f: try: log_num += 1 isnoblk = True for blkrule in BlackRule: if len(re.findall(blkrule, line, re.IGNORECASE)) > 0: isnoblk = False # 判断日志的开头是否是事件的开始, 如果不是则忽略 if isstart == False: if len(re.findall('\d{4}-\d+-\d+ \d+:\d+:\d+.\d{3}|\d{4}-\d+-\d+T\d+:\d+:\d+.\d{3}', line)) > 0: isstart = True # 如果改行既不在黑名单, 并且也已经确定 isstart 为 True, 则开始日志匹配流程 if isnoblk and isstart: line = line.strip() # 判断该行日志是否符合格式 if len(re.findall('\d{4}-\d+-\d+ \d+:\d+:\d+.\d{3}', line)) > 0: if re.findall('fail|error|deny|refused|exception', line, re.IGNORECASE): log_level = 'ERROR' else: log_level = 'INFO' log_time = sql_write.sqlite_to_datetime(line.split(' ', 2)[0] + ' ' + line.split(' ', 2)[1]) log_comp = line.split(' ', 2)[-1].split(': ', 1)[0].strip() log_detail = line.split(' ', 2)[-1].split(':', 1)[1].strip() logdata.append({'logfile':self.filepath, 'logline':log_num, 'loglevel':log_level, 'logtime':log_time, 'logcomp':log_comp, 'logdetail':log_detail}) elif len(re.findall('\d{4}-\d+-\d+T\d+:\d+:\d+.\d{3}', line)) > 0: if re.findall('fail|error|deny|refused|exception', line, re.IGNORECASE): log_level = 'ERROR' else: log_level = 'INFO' log_time = sql_write.sqlite_to_datetime(line.split(' ', 2)[0]) log_comp = line.split(' ', 1)[-1].split(': ', 1)[0].strip() log_detail = line.split(' ', 1)[-1].split(':', 1)[1].strip() logdata.append({'logfile':self.filepath, 'logline':log_num, 'loglevel':log_level, 'logtime':log_time, 'logcomp':log_comp, 'logdetail':log_detail}) else: logdata[-1]['logdetail'] = logdata[-1]['logdetail'] + '\n' + line # 抹掉日志中剩下的 '/n' logdata[-1]['logdetail'] = logdata[-1]['logdetail'].strip() except Exception as e: logSQLCreate.warning("file:{}\nline:{}\nSource:{}\nException:{}".format(self.filepath,str(log_num),line,e)) for data in logdata: if data.get('logdetail') == '': data['logdetail'] = 'Null' try: data['logdetail'] = sql_string.sqlite_to_string(data.get('logdetail')) sql_insert = 'INSERT INTO log_trace (logfile, logline, loglevel, logtime, logcomp, logdetail) VALUES ("{}","{}","{}","{}","{}","{}");'.format(data.get('logfile'), str(data.get('logline')),data.get('loglevel'),data.get('logtime'),data.get('logcomp'),data.get('logdetail')) sqldata.append(sql_insert) except Exception as e: logSQLCreate.warning("Can't generate SQL INSERT INTO statement! - {}".format(e)) self.SQLData = ({'db_name':self.db_name, 'db_type':self.db_type, 'db_table':'log_trace', 'db_data':sqldata,}) # 生成一个对应的 .lck 文件, 在数据写入完成后, 再删除 .lck 文件 datafilepath = r'./temp/{}'.format(self.file_id) open('{}.lck'.format(datafilepath), 'w').close() with open(datafilepath, 'wb') as f: pickle.dump(self.SQLData, f) os.remove('{}.lck'.format(datafilepath)) except Exception as reason: logSQLCreate.error('logfile read error:{}'.format(reason))
def log_traces(self): """ 针对 MicroFocus ITOM OA 中的 trace 文件 (需要先解码) """ # 初始化数据和相关控制参数 logdata = [] sqldata = [] isnoblk = True log_num = 0 isstart = False # 尝试开始读取文件 try: with open(self.filepath, mode='r', encoding='utf-8', errors='replace') as f: for line in f: try: log_num += 1 isnoblk = True for blkrule in BlackRule: if len(re.findall(blkrule, line, re.IGNORECASE)) > 0: isnoblk = False # 判断日志的开头是否是事件的开始, 如果不是则忽略 if isstart == False: if re.findall('"Info",|"Warn",|"Error",', line): isstart = True # 如果改行既不在黑名单, 并且也已经确定 isstart 为 True, 则开始日志匹配流程 if isnoblk and isstart: line = line.strip() # 判断该行日志是否符合格式 if len(line.split(',', 11)) >= 11: line_data = line.split(',',11) log_level = line_data[0].strip()[1:-1] log_time = sql_write.sqlite_to_datetime(line_data[2].strip()) log_Machine = line_data[3].strip() log_comp = line_data[4].strip() if re.findall(',,', line): log_pid = line.replace('"', "'").split(',', 10)[-3].strip() else: log_pid = line_data[-3].strip().replace('"', "'") if re.findall(',,', line): log_tid = line.replace('"', "'").split(',', 10)[-2].strip() else: log_tid = line_data[-2].strip().replace('"', "'") log_tic_count = line_data[1].strip().replace('"', "'") if re.findall(',,', line): log_msg = line.replace('"', "'").split(',', 10)[-1].strip() else: log_msg = line_data[-1].strip().replace('"', "'") logdata.append({'logfile': self.filepath, 'logline': log_num, 'loglevel': log_level, 'logtime': log_time, 'machine': log_Machine, 'logcomp': log_comp, 'pid': log_pid, 'tid': log_tid, 'tic_count': log_tic_count, 'logdetail': log_msg}) else: logdata[-1]['logdetail'] = logdata[-1]['logdetail'] + '\n' + line # 抹掉日志中剩下的 '/n' logdata[-1]['logdetail'] = logdata[-1]['logdetail'].strip() except Exception as e: logSQLCreate.warning( "file:{}\nline:{}\nSource:{}\nException:{}".format(self.filepath, str(log_num), line, e)) for data in logdata: try: data['logdetail'] = sql_string.sqlite_to_string(data.get('logdetail')) sql_insert = 'INSERT INTO log_Trace (logfile, logline, loglevel, logtime, logcomp, logdetail, machine, pid, tid, tic_count) ' \ 'VALUES ("{}","{}","{}","{}","{}","{}","{}","{}","{}","{}");'.format( data.get('logfile'), str(data.get('logline')), data.get('loglevel'), data.get('logtime'), data.get('logcomp'), data.get('logdetail'), data.get('machine'), data.get('pid'), data.get('tid'), data.get('tic_count')) sqldata.append(sql_insert) except Exception as e: logSQLCreate.warning("Can't generate SQL INSERT INTO statement! - {}".format(e)) self.SQLData = ({'db_name': self.db_name, 'db_type': self.db_type, 'db_table': 'log_Trace', 'db_data': sqldata, }) # 生成一个对应的 .lck 文件, 在数据写入完成后, 再删除 .lck 文件 datafilepath = r'./temp/{}'.format(self.file_id) open('{}.lck'.format(datafilepath), 'w').close() with open(datafilepath, 'wb') as f: pickle.dump(self.SQLData, f) os.remove('{}.lck'.format(datafilepath)) except Exception as reason: logSQLCreate.error('logfile read error:{}'.format(reason))