def main(): fileName = sys.argv[1] fileH = MY_FILEH(fileName) syshostPattern = re.compile(r'syshost:([^ ]*) ') fnSz = os.path.getsize(fileName) pbar = ProgressBar(maxVal=fnSz) sz = 0 t1 = time.time() with open(fileName) as f: for line in f: sz += len(line) pbar.update(sz) m = syshostPattern.search(line) if (not m): continue syshost = m.group(1) fh = fileH.get_file_handle(syshost) fh.write(line) fileH.close_file_handles() pbar.fini() t2 = time.time() print("Time: ", time.strftime("%T", time.gmtime(t2-t1)))
def main(): fileName = sys.argv[1] fileH = MY_FILEH(fileName) syshostPattern = re.compile(r'syshost:([^ ]*) ') fnSz = os.path.getsize(fileName) pbar = ProgressBar(maxVal=fnSz) sz = 0 t1 = time.time() with open(fileName) as f: for line in f: sz += len(line) pbar.update(sz) m = syshostPattern.search(line) if (not m): continue syshost = m.group(1) fh = fileH.get_file_handle(syshost) fh.write(line) fileH.close_file_handles() pbar.fini() t2 = time.time() print("Time: ", time.strftime("%T", time.gmtime(t2 - t1)))
def main(): """ Walks the list of users via the passwd_generator and load the link and run files. """ # Push command line on to XALT_Stack sA = [] sA.append("CommandLine:") for v in sys.argv: sA.append('"' + v + '"') XALT_Stack.push(" ".join(sA)) args = CmdLineOptions().execute() xalt = XALTdb(dbConfigFn(args.dbname)) num = int(capture("getent passwd | wc -l")) pbar = ProgressBar(maxVal=num) icnt = 0 t1 = time.time() rmapT = Rmap(args.rmapD).reverseMapT() iuser = 0 lnkCnt = 0 runCnt = 0 xaltDir = args.datadir if (os.path.isdir(xaltDir)): iuser += 1 linkFnA = files_in_tree(xaltDir, "*/link.*.json") XALT_Stack.push("link_json_to_db()") lnkCnt += link_json_to_db(xalt, args.listFn, rmapT, linkFnA) XALT_Stack.pop() if (args.delete): remove_files(linkFnA) #remove_files(files_in_tree(xaltDir, "*/.link.*.json")) runFnA = files_in_tree(xaltDir, "*/run.*.json") XALT_Stack.push("run_json_to_db()") runCnt += run_json_to_db(xalt, args.listFn, rmapT, runFnA) XALT_Stack.pop() if (args.delete): remove_files(runFnA) #remove_files(files_in_tree(xaltDir, "*/.run.*.json")) icnt += 1 pbar.update(icnt) xalt.connect().close() pbar.fini() t2 = time.time() rt = t2 - t1 if (args.timer): print("Time: ", time.strftime("%T", time.gmtime(rt))) print("num users: ", iuser, ", num links: ", lnkCnt, ", num runs: ", runCnt)
def main(): """ Walks the list of users via the passwd_generator and load the link and run files. """ # Push command line on to XALT_Stack sA = [] sA.append("CommandLine:") for v in sys.argv: sA.append('"'+v+'"') XALT_Stack.push(" ".join(sA)) args = CmdLineOptions().execute() xalt = XALTdb(dbConfigFn(args.dbname)) num = int(capture("getent passwd | wc -l")) pbar = ProgressBar(maxVal=num) icnt = 0 t1 = time.time() rmapT = Rmap(args.rmapD).reverseMapT() iuser = 0 lnkCnt = 0 runCnt = 0 xaltDir = args.datadir if (os.path.isdir(xaltDir)): iuser += 1 linkFnA = files_in_tree(xaltDir, "*/link.*.json") XALT_Stack.push("link_json_to_db()") lnkCnt += link_json_to_db(xalt, args.listFn, rmapT, linkFnA) XALT_Stack.pop() if (args.delete): remove_files(linkFnA) #remove_files(files_in_tree(xaltDir, "*/.link.*.json")) runFnA = files_in_tree(xaltDir, "*/run.*.json") XALT_Stack.push("run_json_to_db()") runCnt += run_json_to_db(xalt, args.listFn, rmapT, runFnA) XALT_Stack.pop() if (args.delete): remove_files(runFnA) #remove_files(files_in_tree(xaltDir, "*/.run.*.json")) icnt += 1 pbar.update(icnt) xalt.connect().close() pbar.fini() t2 = time.time() rt = t2 - t1 if (args.timer): print("Time: ", time.strftime("%T", time.gmtime(rt))) print("num users: ", iuser, ", num links: ", lnkCnt, ", num runs: ", runCnt)
def main(): args = CmdLineOptions().execute() xalt = XALTdb(ConfigFn) num = int(capture("getent passwd | wc -l")) pbar = ProgressBar(maxVal=num) icnt = 0 t1 = time.time() rmapT = Rmap(args.rmapD).reverseMapT() iuser = 0 lnkCnt = 0 runCnt = 0 for user, hdir in passwd_generator(): xaltDir = os.path.join(hdir, ".xalt.d") if (os.path.isdir(xaltDir)): iuser += 1 linkFnA = files_in_tree(xaltDir, "*/link.*.json") lnkCnt += link_json_to_db(xalt, user, rmapT, linkFnA) if (args.delete): remove_files(linkFnA) remove_files(files_in_tree(xaltDir, "*/.link.*.json")) runFnA = files_in_tree(xaltDir, "*/run.*.json") runCnt += run_json_to_db(xalt, user, rmapT, runFnA) if (args.delete): remove_files(runFnA) remove_files(files_in_tree(xaltDir, "*/.run.*.json")) icnt += 1 pbar.update(icnt) xalt.connect().close() pbar.fini() t2 = time.time() rt = t2 - t1 if (args.timer): print("Time: ", time.strftime("%T", time.gmtime(rt))) print("num users: ", iuser, ", num links: ", lnkCnt, ", num runs: ", runCnt)
def main(): args = CmdLineOptions().execute() xalt = XALTdb(ConfigFn) num = int(capture("getent passwd | wc -l")) pbar = ProgressBar(maxVal=num) icnt = 0 t1 = time.time() rmapT = Rmap(args.rmapD).reverseMapT() iuser = 0 lnkCnt = 0 runCnt = 0 for user, hdir in passwd_generator(): xaltDir = os.path.join(hdir,".xalt.d") if (os.path.isdir(xaltDir)): iuser += 1 linkFnA = files_in_tree(xaltDir, "*/link.*.json") lnkCnt += link_json_to_db(xalt, user, rmapT, linkFnA) if (args.delete): remove_files(linkFnA) remove_files(files_in_tree(xaltDir, "*/.link.*.json")) runFnA = files_in_tree(xaltDir, "*/run.*.json") runCnt += run_json_to_db(xalt, user, rmapT, runFnA) if (args.delete): remove_files(runFnA) remove_files(files_in_tree(xaltDir, "*/.run.*.json")) icnt += 1 pbar.update(icnt) xalt.connect().close() pbar.fini() t2 = time.time() rt = t2 - t1 if (args.timer): print("Time: ", time.strftime("%T", time.gmtime(rt))) print("num users: ", iuser, ", num links: ", lnkCnt, ", num runs: ", runCnt)
def main(): """ read from syslog file into XALT db. """ sA = [] sA.append("CommandLine:") for v in sys.argv: sA.append('"'+v+'"') XALT_Stack.push(" ".join(sA)) args = CmdLineOptions().execute() xalt = XALTdb(dbConfigFn(args.dbname)) syslogFile = args.syslog # should add a check if file exists num = int(capture("cat "+syslogFile+" | wc -l")) icnt = 0 t1 = time.time() rmapT = Rmap(args.rmapD).reverseMapT() lnkCnt = 0 runCnt = 0 badCnt = 0 count = 0 recordT = {} if (num == 0): return pbar = ProgressBar(maxVal=num) fnA = [ args.leftover, syslogFile ] for fn in fnA: if (not os.path.isfile(fn)): continue f=open(fn, 'r') for line in f: if (not ("XALT_LOGGING" in line)): continue t, done = parseSyslog(line, recordT) if (not done): continue try: XALT_Stack.push("XALT_LOGGING: " + t['kind'] + " " + t['syshost']) if ( t['kind'] == "link" ): XALT_Stack.push("link_to_db()") xalt.link_to_db(rmapT, json.loads(t['value'])) XALT_Stack.pop() lnkCnt += 1 elif ( t['kind'] == "run" ): XALT_Stack.push("run_to_db()") xalt.run_to_db(rmapT, json.loads(t['value'])) XALT_Stack.pop() runCnt += 1 else: print("Error in xalt_syslog_to_db", file=sys.stderr) XALT_Stack.pop() except Exception as e: print(e, file=sys.stderr) badCnt += 1 count += 1 pbar.update(count) f.close() pbar.fini() t2 = time.time() rt = t2 - t1 if (args.timer): print("Time: ", time.strftime("%T", time.gmtime(rt))) print("total processed : ", count, ", num links: ", lnkCnt, ", num runs: ", runCnt, ", badCnt: ", badCnt) leftover = args.leftover if (os.path.isfile(leftover)): os.rename(leftover, leftover + ".old") # if there is anything left in recordT file write it out to the leftover file. if (recordT): f = open(leftover, "w") for key in recordT: r = recordT[key] s = r.prt("XALT_LOGGING V=2", key) f.write(s) f.close()
def main(): """ Walks the list of users via the passwd_generator and load the link and run files. """ # Using home directories or a global location. xalt_file_prefix = os.environ.get("XALT_FILE_PREFIX", "@xalt_file_prefix@") xaltUserA = os.environ.get("XALT_USERS") if (xaltUserA): xaltUserStr = ", XALT_USERS: \"" + xaltUserA + "\"" else: xaltUserStr = "" print("\n################################################################") print("XALT Git Version: " + Version()) print(" Using XALT_FILE_PREFIX: \"" + xalt_file_prefix + "\"" + xaltUserStr) print("################################################################\n") # Find transmission style transmission = os.environ.get("XALT_TRANSMISSION_STYLE") if (not transmission): transmission = "@transmission@" if (not transmission): transmission = "file" transmission = transmission.lower() timeRecord = TimeRecord() # Push command line on to XALT_Stack sA = [] sA.append("CommandLine:") for v in sys.argv: sA.append('"' + v + '"') XALT_Stack.push(" ".join(sA)) args = CmdLineOptions().execute() xalt = XALTdb(args.confFn) if (xalt_file_prefix == "USE_HOME"): num = int(capture("LD_PRELOAD= getent passwd | wc -l")) pbar = ProgressBar(maxVal=num) else: xaltDir = build_resultDir("", transmission, "") allFnA = files_in_tree(xaltDir, "*/*." + args.syshost + ".*.json") pbar = ProgressBar(maxVal=len(allFnA)) icnt = 0 t1 = time.time() rmapT = Rmap(args.rmapD).reverseMapT() u2acctT = {} if (args.u2acct): fp = open(args.u2acct, "r") u2acctT = json.loads(fp.read()) fp.close() countT = {} countT['lnk'] = 0 countT['run'] = 0 countT['pkg'] = 0 countT['any'] = 0 if (xalt_file_prefix == "USE_HOME"): for user, homeDir in passwd_generator(): store_json_files(homeDir, transmission, xalt, rmapT, u2acctT, args, countT, pbar, timeRecord) else: store_json_files("", transmission, xalt, rmapT, u2acctT, args, countT, pbar, timeRecord) xalt.connect().close() pbar.fini() t2 = time.time() rt = t2 - t1 if (args.timer): print("Time: ", time.strftime("%T", time.gmtime(rt))) print("num links: ", countT['lnk'], ", num pkgs: ", countT['pkg'], ", num runs: ", countT['run']) timeRecord.print()
def main(): """ read from syslog file into XALT db. """ sA = [] sA.append("CommandLine:") for v in sys.argv: sA.append('"' + v + '"') XALT_Stack.push(" ".join(sA)) args = CmdLineOptions().execute() xalt = XALTdb(dbConfigFn(args.dbname)) syslogFile = args.syslog # should add a check if file exists num = int(capture("cat " + syslogFile + " | wc -l")) icnt = 0 t1 = time.time() rmapT = Rmap(args.rmapD).reverseMapT() lnkCnt = 0 runCnt = 0 badCnt = 0 count = 0 recordT = {} if (num == 0): return pbar = ProgressBar(maxVal=num) fnA = [args.leftover, syslogFile] for fn in fnA: if (not os.path.isfile(fn)): continue f = open(fn, 'r') for line in f: if (not ("XALT_LOGGING" in line)): continue t, done = parseSyslog(line, recordT) if (not done): continue try: XALT_Stack.push("XALT_LOGGING: " + t['kind'] + " " + t['syshost']) if (t['kind'] == "link"): XALT_Stack.push("link_to_db()") xalt.link_to_db(rmapT, json.loads(t['value'])) XALT_Stack.pop() lnkCnt += 1 elif (t['kind'] == "run"): XALT_Stack.push("run_to_db()") xalt.run_to_db(rmapT, json.loads(t['value'])) XALT_Stack.pop() runCnt += 1 else: print("Error in xalt_syslog_to_db", file=sys.stderr) XALT_Stack.pop() except Exception as e: print(e, file=sys.stderr) badCnt += 1 count += 1 pbar.update(count) f.close() pbar.fini() t2 = time.time() rt = t2 - t1 if (args.timer): print("Time: ", time.strftime("%T", time.gmtime(rt))) print("total processed : ", count, ", num links: ", lnkCnt, ", num runs: ", runCnt, ", badCnt: ", badCnt) leftover = args.leftover if (os.path.isfile(leftover)): os.rename(leftover, leftover + ".old") # if there is anything left in recordT file write it out to the leftover file. if (recordT): f = open(leftover, "w") for key in recordT: r = recordT[key] s = r.prt("XALT_LOGGING V=2", key) f.write(s) f.close()
def main(): """ read from syslog file into XALT db. """ sA = [] sA.append("CommandLine:") for v in sys.argv: sA.append('"' + v + '"') XALT_Stack.push(" ".join(sA)) args = CmdLineOptions().execute() xalt = XALTdb(ConfigFn) syslogFile = args.syslog # should add a check if file exists num = int(capture("cat " + syslogFile + " | wc -l")) icnt = 0 t1 = time.time() rmapT = Rmap(args.rmapD).reverseMapT() lnkCnt = 0 runCnt = 0 badCnt = 0 count = 0 if num != 0: pbar = ProgressBar(maxVal=num) if (os.path.isfile(syslogFile)): f = open(syslogFile, 'r') for line in f: if "XALT_LOGGING" in line: i = line.find("link:") if i == -1: i = line.find("run:") if i == -1: continue # did not find a link or run, continue to next line array = line[i:].split(":") type = array[0].strip() syshost = array[1].strip() try: resultT = json.loads(base64.b64decode(array[2])) XALT_Stack.push("XALT_LOGGING: " + type + " " + syshost) # XALT_Stack.push("XALT_LOGGING resultT: " + resultT) if (type == "link"): XALT_Stack.push("link_to_db()") xalt.link_to_db(rmapT, resultT) XALT_Stack.pop() lnkCnt += 1 elif (type == "run"): XALT_Stack.push("run_to_db()") xalt.run_to_db(rmapT, resultT) XALT_Stack.pop() runCnt += 1 else: print("Error in xalt_syslog_to_db") XALT_Stack.pop() except: badCnt += 1 # figure out length of array[2], as it might be # near the syslog message size limit strg = array[2] lens = len(strg) # lenx = lens - (lens % 4 if lens % 4 else 4) print( "xalt_syslog_to_db: undecodable entry! length: ", lens) # resultT = base64.decodestring(strg[:lenx]) # print("result: ",result) count += 1 pbar.update(count) pbar.fini() # what should be done if there are errors? # what went wrong? # how do we restart? # # xalt.connect().close() t2 = time.time() rt = t2 - t1 if (args.timer): print("Time: ", time.strftime("%T", time.gmtime(rt))) # print("total processed : ", count, ", num links: ", lnkCnt, ", num runs: ", runCnt, ", badCnt: ", badCnt)
def main(): """ Walks the list of users via the passwd_generator and load the link and run files. """ # Find transmission style transmission = os.environ.get("XALT_TRANSMISSION_STYLE") if (not transmission): transmission = "@XALT_TRANSMISSION_STYLE@" if (not transmission): transmission = "file" transmission = transmission.lower() # Push command line on to XALT_Stack sA = [] sA.append("CommandLine:") for v in sys.argv: sA.append('"'+v+'"') XALT_Stack.push(" ".join(sA)) args = CmdLineOptions().execute() xalt = XALTdb(args.confFn) num = int(capture("getent passwd | wc -l")) pbar = ProgressBar(maxVal=num) icnt = 0 t1 = time.time() rmapT = Rmap(args.rmapD).reverseMapT() iuser = 0 lnkCnt = 0 runCnt = 0 pkgCnt = 0 for user, hdir in passwd_generator(): XALT_Stack.push("User: "******"link") if (os.path.isdir(xaltDir)): iuser += 1 linkFnA = files_in_tree(xaltDir, "*/link." + args.syshost + ".*.json") XALT_Stack.push("link_json_to_db()") lnkCnt += link_json_to_db(xalt, args.listFn, rmapT, args.delete, linkFnA) XALT_Stack.pop() xaltDir = build_xaltDir(user, hdir, transmission, "run") if (os.path.isdir(xaltDir)): runFnA = files_in_tree(xaltDir, "*/run." + args.syshost + ".*.json") XALT_Stack.push("run_json_to_db()") runCnt += run_json_to_db(xalt, args.listFn, rmapT, args.delete, runFnA) XALT_Stack.pop() xaltDir = build_xaltDir(user, hdir, transmission, "pkg") if (os.path.isdir(xaltDir)): pkgFnA = files_in_tree(xaltDir, "*/pkg." + args.syshost + ".*.json") XALT_Stack.push("pkg_json_to_db()") pkgCnt += pkg_json_to_db(xalt, args.listFn, args.syshost, args.delete, pkgFnA) XALT_Stack.pop() icnt += 1 v = XALT_Stack.pop() carp("User",v) pbar.update(icnt) xalt.connect().close() pbar.fini() t2 = time.time() rt = t2 - t1 if (args.timer): print("Time: ", time.strftime("%T", time.gmtime(rt))) print("num users: ", iuser, ", num links: ", lnkCnt, ", num pkgs: ", pkgCnt, ", num runs: ", runCnt)
def main(): """ read from syslog file into XALT db. """ sA = [] sA.append("CommandLine:") for v in sys.argv: sA.append('"' + v + '"') XALT_Stack.push(" ".join(sA)) args = CmdLineOptions().execute() xalt = XALTdb(args.confFn) syslogFile = args.syslog icnt = 0 t1 = time.time() try: rmapT = Rmap(args.rmapD).reverseMapT() except Exception as e: print(e, file=sys.stderr) print("Failed to read reverseMap file -> exiting") sys.exit(1) lnkCnt = 0 pkgCnt = 0 runCnt = 0 badCnt = 0 count = 0 recordT = {} fnA = [args.leftover, syslogFile] parseSyslog = ParseSyslog(args.leftover) #----------------------------- # Figure out size in bytes. fnSz = 0 for fn in fnA: if (not os.path.isfile(fn)): continue fnSz += os.path.getsize(fn) #---------------------------------------------------------- # Count the number and sum the run_time for all scalar jobs filter = Filter(100) pbar = ProgressBar(maxVal=fnSz, fd=sys.stdout) for fn in fnA: if (not os.path.isfile(fn)): continue old = (fn == args.leftover) lineNo = 0 f = open(fn, 'r') for line in f: lineNo += 1 count += len(line) pbar.update(count) if (not ("XALT_LOGGING" in line)): continue try: t, done = parseSyslog.parse(line, args.syshost, old) except Exception as e: #print(e, file=sys.stderr) #print("lineNo:",lineNo,"file:",fn,"line:",line, file=sys.stderr) #print("Now continuing processing!", file=sys.stderr) continue if (not done or t['kind'] != "run"): continue ################################## # If the json conversion fails, # then ignore record and keep going value = False try: value = json.loads(t['value']) filter.register(value) except Exception as e: #print("fn:",fn,"line:",lineNo,"value:",t['value'],file=sys.stderr) continue f.close() pbar.fini() filter.report_stats() badsyslog = 0 count = 0 parseSyslog = ParseSyslog(args.leftover) pbar = ProgressBar(maxVal=max(fnSz, 1), fd=sys.stdout) for fn in fnA: if (not os.path.isfile(fn)): continue old = (fn == args.leftover) f = open(fn, 'r') for line in f: count += len(line) pbar.update(count) if (not ("XALT_LOGGING" in line)): continue try: t, done = parseSyslog.parse(line, args.syshost, old) except Exception as e: badsyslog += 1 continue if (not done): continue ################################## # If the json conversion fails, # then ignore record and keep going try: value = json.loads(t['value']) except Exception as e: continue try: XALT_Stack.push("XALT_LOGGING: " + t['kind'] + " " + t['syshost']) if (t['kind'] == "link"): XALT_Stack.push("link_to_db()") xalt.link_to_db(rmapT, value) XALT_Stack.pop() lnkCnt += 1 elif (t['kind'] == "run"): if (filter.apply(value)): XALT_Stack.push("run_to_db()") xalt.run_to_db(rmapT, value) XALT_Stack.pop() runCnt += 1 elif (t['kind'] == "pkg"): XALT_Stack.push("pkg_to_db()") xalt.pkg_to_db(t['syshost'], value) XALT_Stack.pop() pkgCnt += 1 else: print("Error in xalt_syslog_to_db", file=sys.stderr) XALT_Stack.pop() except Exception as e: print(e, file=sys.stderr) badCnt += 1 f.close() pbar.fini() t2 = time.time() rt = t2 - t1 if (args.timer): print("Time: ", time.strftime("%T", time.gmtime(rt))) print("total processed : ", count, ", num links: ", lnkCnt, ", num runs: ", runCnt, ", pkgCnt: ", pkgCnt, ", badCnt: ", badCnt, ", badsyslog: ", badsyslog) # if there is anything left in recordT file write it out to the leftover file. parseSyslog.writeRecordT()
def main(): """ Walks the list of users via the passwd_generator and load the link and run files. """ # Find transmission style transmission = os.environ.get("XALT_TRANSMISSION_STYLE") if (not transmission): transmission = "@XALT_TRANSMISSION_STYLE@" if (not transmission): transmission = "file" transmission = transmission.lower() # Push command line on to XALT_Stack sA = [] sA.append("CommandLine:") for v in sys.argv: sA.append('"' + v + '"') XALT_Stack.push(" ".join(sA)) args = CmdLineOptions().execute() xalt = XALTdb(args.confFn) num = int(capture("getent passwd | wc -l")) pbar = ProgressBar(maxVal=num) icnt = 0 t1 = time.time() rmapT = Rmap(args.rmapD).reverseMapT() iuser = 0 lnkCnt = 0 runCnt = 0 pkgCnt = 0 for user, hdir in passwd_generator(): XALT_Stack.push("User: "******"link") if (os.path.isdir(xaltDir)): iuser += 1 linkFnA = files_in_tree(xaltDir, "*/link." + args.syshost + ".*.json") XALT_Stack.push("link_json_to_db()") lnkCnt += link_json_to_db(xalt, args.listFn, rmapT, args.delete, linkFnA) XALT_Stack.pop() xaltDir = build_xaltDir(user, hdir, transmission, "run") if (os.path.isdir(xaltDir)): runFnA = files_in_tree(xaltDir, "*/run." + args.syshost + ".*.json") XALT_Stack.push("run_json_to_db()") runCnt += run_json_to_db(xalt, args.listFn, rmapT, args.delete, runFnA) XALT_Stack.pop() xaltDir = build_xaltDir(user, hdir, transmission, "pkg") if (os.path.isdir(xaltDir)): pkgFnA = files_in_tree(xaltDir, "*/pkg." + args.syshost + ".*.json") XALT_Stack.push("pkg_json_to_db()") pkgCnt += pkg_json_to_db(xalt, args.listFn, args.syshost, args.delete, pkgFnA) XALT_Stack.pop() icnt += 1 v = XALT_Stack.pop() carp("User", v) pbar.update(icnt) xalt.connect().close() pbar.fini() t2 = time.time() rt = t2 - t1 if (args.timer): print("Time: ", time.strftime("%T", time.gmtime(rt))) print("num users: ", iuser, ", num links: ", lnkCnt, ", num pkgs: ", pkgCnt, ", num runs: ", runCnt)
def main(): """ read from syslog file into XALT db. """ sA = [] sA.append("CommandLine:") for v in sys.argv: sA.append('"' + v + '"') XALT_Stack.push(" ".join(sA)) args = CmdLineOptions().execute() xalt = XALTdb(args.confFn) syslogFile = args.syslog icnt = 0 t1 = time.time() try: rmapT = Rmap(args.rmapD).reverseMapT() except Exception as e: print(e, file=sys.stderr) print("Failed to read reverseMap file -> exiting") print(traceback.format_exc()) sys.exit(1) u2acctT = {} if (args.u2acct): fp = open(args.u2acct, "r") u2acctT = json.loads(fp.read()) fp.close() lnkCnt = 0 pkgCnt = 0 runCnt = 0 badCnt = 0 count = 0 recordT = {} timeRecord = TimeRecord() fnA = [args.leftover, syslogFile] parseSyslog = ParseSyslog(args.leftover) #----------------------------- # Figure out size in bytes. fnSz = 0 for fn in fnA: if (not os.path.isfile(fn)): continue fnSz += os.path.getsize(fn) badsyslog = 0 count = 0 parseSyslog = ParseSyslog(args.leftover) pbar = ProgressBar(maxVal=max(fnSz, 1), fd=sys.stdout) random.seed() for fn in fnA: if (not os.path.isfile(fn)): continue old = (fn == args.leftover) f = open(fn, 'r') for line in f: count += len(line) pbar.update(count) if (not ("XALT_LOGGING" in line)): continue try: t, done = parseSyslog.parse(line, args.syshost, old) except Exception as e: badsyslog += 1 continue if (not done): continue ################################## # If the json conversion fails, # then ignore record and keep going try: value = json.loads(t['value']) except Exception as e: continue try: XALT_Stack.push("XALT_LOGGING: " + t['kind'] + " " + t['syshost']) if (t['kind'] == "link"): XALT_Stack.push("link_to_db()") xalt.link_to_db(rmapT, value) XALT_Stack.pop() lnkCnt += 1 elif (t['kind'] == "run"): XALT_Stack.push("run_to_db()") runTime = value['userDT']['run_time'] if (args.filter and runTime >= 1800.0 and runTime <= 3600.0): if (random.random() > 0.01): continue else: value['userDT']['probability'] = 0.01 stored = xalt.run_to_db(rmapT, u2acctT, value, timeRecord) XALT_Stack.pop() if (stored): runCnt += 1 elif (t['kind'] == "pkg"): XALT_Stack.push("pkg_to_db()") xalt.pkg_to_db(t['syshost'], value) XALT_Stack.pop() pkgCnt += 1 else: print("Error in xalt_syslog_to_db", file=sys.stderr) XALT_Stack.pop() except Exception as e: print(e, file=sys.stderr) print(traceback.format_exc()) badCnt += 1 f.close() pbar.fini() t2 = time.time() rt = t2 - t1 if (args.timer): print("Time: ", time.strftime("%T", time.gmtime(rt))) print("total processed : ", count, ", num links: ", lnkCnt, ", num runs: ", runCnt, ", pkgCnt: ", pkgCnt, ", badCnt: ", badCnt, ", badsyslog: ", badsyslog) timeRecord.print() # if there is anything left in recordT file write it out to the leftover file. parseSyslog.writeRecordT()
def main(): """ read from syslog file into XALT db. """ sA = [] sA.append("CommandLine:") for v in sys.argv: sA.append('"'+v+'"') XALT_Stack.push(" ".join(sA)) args = CmdLineOptions().execute() xalt = XALTdb(ConfigFn) syslogFile = args.syslog # should add a check if file exists num = int(capture("cat "+syslogFile+" | wc -l")) icnt = 0 t1 = time.time() rmapT = Rmap(args.rmapD).reverseMapT() lnkCnt = 0 runCnt = 0 badCnt = 0 count = 0 if num != 0: pbar = ProgressBar(maxVal=num) if (os.path.isfile(syslogFile)): f=open(syslogFile, 'r') for line in f: if "XALT_LOGGING" in line: i=line.find("link:") if i == -1 : i=line.find("run:") if i == -1: continue # did not find a link or run, continue to next line array=line[i:].split(":") type = array[0].strip() syshost = array[1].strip() try: resultT=json.loads(base64.b64decode(array[2])) XALT_Stack.push("XALT_LOGGING: " + type + " " + syshost) # XALT_Stack.push("XALT_LOGGING resultT: " + resultT) if ( type == "link" ): XALT_Stack.push("link_to_db()") xalt.link_to_db(rmapT, resultT) XALT_Stack.pop() lnkCnt += 1 elif ( type == "run" ): XALT_Stack.push("run_to_db()") xalt.run_to_db(rmapT, resultT) XALT_Stack.pop() runCnt += 1 else: print("Error in xalt_syslog_to_db") XALT_Stack.pop() except: badCnt += 1 # figure out length of array[2], as it might be # near the syslog message size limit strg=array[2] lens = len(strg) # lenx = lens - (lens % 4 if lens % 4 else 4) print("xalt_syslog_to_db: undecodable entry! length: ",lens) # resultT = base64.decodestring(strg[:lenx]) # print("result: ",result) count += 1 pbar.update(count) pbar.fini() # what should be done if there are errors? # what went wrong? # how do we restart? # # xalt.connect().close() t2 = time.time() rt = t2 - t1 if (args.timer): print("Time: ", time.strftime("%T", time.gmtime(rt))) # print("total processed : ", count, ", num links: ", lnkCnt, ", num runs: ", runCnt, ", badCnt: ", badCnt)