def main(): """ read from syslog file into XALT db. """ sA = [] sA.append("CommandLine:") for v in sys.argv: sA.append('"' + v + '"') XALT_Stack.push(" ".join(sA)) args = CmdLineOptions().execute() xalt = XALTdb(args.confFn) syslogFile = args.syslog icnt = 0 t1 = time.time() try: rmapT = Rmap(args.rmapD).reverseMapT() except Exception as e: print(e, file=sys.stderr) print("Failed to read reverseMap file -> exiting") sys.exit(1) lnkCnt = 0 pkgCnt = 0 runCnt = 0 badCnt = 0 count = 0 recordT = {} fnA = [args.leftover, syslogFile] parseSyslog = ParseSyslog(args.leftover) #----------------------------- # Figure out size in bytes. fnSz = 0 for fn in fnA: if (not os.path.isfile(fn)): continue fnSz += os.path.getsize(fn) #---------------------------------------------------------- # Count the number and sum the run_time for all scalar jobs filter = Filter(100) pbar = ProgressBar(maxVal=fnSz, fd=sys.stdout) for fn in fnA: if (not os.path.isfile(fn)): continue old = (fn == args.leftover) lineNo = 0 f = open(fn, 'r') for line in f: lineNo += 1 count += len(line) pbar.update(count) if (not ("XALT_LOGGING" in line)): continue try: t, done = parseSyslog.parse(line, args.syshost, old) except Exception as e: #print(e, file=sys.stderr) #print("lineNo:",lineNo,"file:",fn,"line:",line, file=sys.stderr) #print("Now continuing processing!", file=sys.stderr) continue if (not done or t['kind'] != "run"): continue ################################## # If the json conversion fails, # then ignore record and keep going value = False try: value = json.loads(t['value']) filter.register(value) except Exception as e: #print("fn:",fn,"line:",lineNo,"value:",t['value'],file=sys.stderr) continue f.close() pbar.fini() filter.report_stats() badsyslog = 0 count = 0 parseSyslog = ParseSyslog(args.leftover) pbar = ProgressBar(maxVal=max(fnSz, 1), fd=sys.stdout) for fn in fnA: if (not os.path.isfile(fn)): continue old = (fn == args.leftover) f = open(fn, 'r') for line in f: count += len(line) pbar.update(count) if (not ("XALT_LOGGING" in line)): continue try: t, done = parseSyslog.parse(line, args.syshost, old) except Exception as e: badsyslog += 1 continue if (not done): continue ################################## # If the json conversion fails, # then ignore record and keep going try: value = json.loads(t['value']) except Exception as e: continue try: XALT_Stack.push("XALT_LOGGING: " + t['kind'] + " " + t['syshost']) if (t['kind'] == "link"): XALT_Stack.push("link_to_db()") xalt.link_to_db(rmapT, value) XALT_Stack.pop() lnkCnt += 1 elif (t['kind'] == "run"): if (filter.apply(value)): XALT_Stack.push("run_to_db()") xalt.run_to_db(rmapT, value) XALT_Stack.pop() runCnt += 1 elif (t['kind'] == "pkg"): XALT_Stack.push("pkg_to_db()") xalt.pkg_to_db(t['syshost'], value) XALT_Stack.pop() pkgCnt += 1 else: print("Error in xalt_syslog_to_db", file=sys.stderr) XALT_Stack.pop() except Exception as e: print(e, file=sys.stderr) badCnt += 1 f.close() pbar.fini() t2 = time.time() rt = t2 - t1 if (args.timer): print("Time: ", time.strftime("%T", time.gmtime(rt))) print("total processed : ", count, ", num links: ", lnkCnt, ", num runs: ", runCnt, ", pkgCnt: ", pkgCnt, ", badCnt: ", badCnt, ", badsyslog: ", badsyslog) # if there is anything left in recordT file write it out to the leftover file. parseSyslog.writeRecordT()
def main(): """ read from syslog file into XALT db. """ sA = [] sA.append("CommandLine:") for v in sys.argv: sA.append('"' + v + '"') XALT_Stack.push(" ".join(sA)) args = CmdLineOptions().execute() xalt = XALTdb(args.confFn) syslogFile = args.syslog icnt = 0 t1 = time.time() try: rmapT = Rmap(args.rmapD).reverseMapT() except Exception as e: print(e, file=sys.stderr) print("Failed to read reverseMap file -> exiting") print(traceback.format_exc()) sys.exit(1) u2acctT = {} if (args.u2acct): fp = open(args.u2acct, "r") u2acctT = json.loads(fp.read()) fp.close() lnkCnt = 0 pkgCnt = 0 runCnt = 0 badCnt = 0 count = 0 recordT = {} timeRecord = TimeRecord() fnA = [args.leftover, syslogFile] parseSyslog = ParseSyslog(args.leftover) #----------------------------- # Figure out size in bytes. fnSz = 0 for fn in fnA: if (not os.path.isfile(fn)): continue fnSz += os.path.getsize(fn) badsyslog = 0 count = 0 parseSyslog = ParseSyslog(args.leftover) pbar = ProgressBar(maxVal=max(fnSz, 1), fd=sys.stdout) random.seed() for fn in fnA: if (not os.path.isfile(fn)): continue old = (fn == args.leftover) f = open(fn, 'r') for line in f: count += len(line) pbar.update(count) if (not ("XALT_LOGGING" in line)): continue try: t, done = parseSyslog.parse(line, args.syshost, old) except Exception as e: badsyslog += 1 continue if (not done): continue ################################## # If the json conversion fails, # then ignore record and keep going try: value = json.loads(t['value']) except Exception as e: continue try: XALT_Stack.push("XALT_LOGGING: " + t['kind'] + " " + t['syshost']) if (t['kind'] == "link"): XALT_Stack.push("link_to_db()") xalt.link_to_db(rmapT, value) XALT_Stack.pop() lnkCnt += 1 elif (t['kind'] == "run"): XALT_Stack.push("run_to_db()") runTime = value['userDT']['run_time'] if (args.filter and runTime >= 1800.0 and runTime <= 3600.0): if (random.random() > 0.01): continue else: value['userDT']['probability'] = 0.01 stored = xalt.run_to_db(rmapT, u2acctT, value, timeRecord) XALT_Stack.pop() if (stored): runCnt += 1 elif (t['kind'] == "pkg"): XALT_Stack.push("pkg_to_db()") xalt.pkg_to_db(t['syshost'], value) XALT_Stack.pop() pkgCnt += 1 else: print("Error in xalt_syslog_to_db", file=sys.stderr) XALT_Stack.pop() except Exception as e: print(e, file=sys.stderr) print(traceback.format_exc()) badCnt += 1 f.close() pbar.fini() t2 = time.time() rt = t2 - t1 if (args.timer): print("Time: ", time.strftime("%T", time.gmtime(rt))) print("total processed : ", count, ", num links: ", lnkCnt, ", num runs: ", runCnt, ", pkgCnt: ", pkgCnt, ", badCnt: ", badCnt, ", badsyslog: ", badsyslog) timeRecord.print() # if there is anything left in recordT file write it out to the leftover file. parseSyslog.writeRecordT()