def main():
"""
read from syslog file into XALT db.
"""
sA = []
sA.append("CommandLine:")
for v in sys.argv:
sA.append('"' + v + '"')
XALT_Stack.push(" ".join(sA))
args = CmdLineOptions().execute()
xalt = XALTdb(args.confFn)
syslogFile = args.syslog
icnt = 0
t1 = time.time()
try:
rmapT = Rmap(args.rmapD).reverseMapT()
except Exception as e:
print(e, file=sys.stderr)
print("Failed to read reverseMap file -> exiting")
sys.exit(1)
lnkCnt = 0
pkgCnt = 0
runCnt = 0
badCnt = 0
count = 0
recordT = {}
fnA = [args.leftover, syslogFile]
parseSyslog = ParseSyslog(args.leftover)
#-----------------------------
# Figure out size in bytes.
fnSz = 0
for fn in fnA:
if (not os.path.isfile(fn)):
continue
fnSz += os.path.getsize(fn)
#----------------------------------------------------------
# Count the number and sum the run_time for all scalar jobs
filter = Filter(100)
pbar = ProgressBar(maxVal=fnSz, fd=sys.stdout)
for fn in fnA:
if (not os.path.isfile(fn)):
continue
old = (fn == args.leftover)
lineNo = 0
f = open(fn, 'r')
for line in f:
lineNo += 1
count += len(line)
pbar.update(count)
if (not ("XALT_LOGGING" in line)):
continue
try:
t, done = parseSyslog.parse(line, args.syshost, old)
except Exception as e:
#print(e, file=sys.stderr)
#print("lineNo:",lineNo,"file:",fn,"line:",line, file=sys.stderr)
#print("Now continuing processing!", file=sys.stderr)
continue
if (not done or t['kind'] != "run"):
continue
##################################
# If the json conversion fails,
# then ignore record and keep going
value = False
try:
value = json.loads(t['value'])
filter.register(value)
except Exception as e:
#print("fn:",fn,"line:",lineNo,"value:",t['value'],file=sys.stderr)
continue
f.close()
pbar.fini()
filter.report_stats()
badsyslog = 0
count = 0
parseSyslog = ParseSyslog(args.leftover)
pbar = ProgressBar(maxVal=max(fnSz, 1), fd=sys.stdout)
for fn in fnA:
if (not os.path.isfile(fn)):
continue
old = (fn == args.leftover)
f = open(fn, 'r')
for line in f:
count += len(line)
pbar.update(count)
if (not ("XALT_LOGGING" in line)):
continue
try:
t, done = parseSyslog.parse(line, args.syshost, old)
except Exception as e:
badsyslog += 1
continue
if (not done):
continue
##################################
# If the json conversion fails,
# then ignore record and keep going
try:
value = json.loads(t['value'])
except Exception as e:
continue
try:
XALT_Stack.push("XALT_LOGGING: " + t['kind'] + " " +
t['syshost'])
if (t['kind'] == "link"):
XALT_Stack.push("link_to_db()")
xalt.link_to_db(rmapT, value)
XALT_Stack.pop()
lnkCnt += 1
elif (t['kind'] == "run"):
if (filter.apply(value)):
XALT_Stack.push("run_to_db()")
xalt.run_to_db(rmapT, value)
XALT_Stack.pop()
runCnt += 1
elif (t['kind'] == "pkg"):
XALT_Stack.push("pkg_to_db()")
xalt.pkg_to_db(t['syshost'], value)
XALT_Stack.pop()
pkgCnt += 1
else:
print("Error in xalt_syslog_to_db", file=sys.stderr)
XALT_Stack.pop()
except Exception as e:
print(e, file=sys.stderr)
badCnt += 1
f.close()
pbar.fini()
t2 = time.time()
rt = t2 - t1
if (args.timer):
print("Time: ", time.strftime("%T", time.gmtime(rt)))
print("total processed : ", count, ", num links: ", lnkCnt, ", num runs: ",
runCnt, ", pkgCnt: ", pkgCnt, ", badCnt: ", badCnt, ", badsyslog: ",
badsyslog)
# if there is anything left in recordT file write it out to the leftover file.
parseSyslog.writeRecordT()
def main():
"""
read from syslog file into XALT db.
"""
sA = []
sA.append("CommandLine:")
for v in sys.argv:
sA.append('"' + v + '"')
XALT_Stack.push(" ".join(sA))
args = CmdLineOptions().execute()
xalt = XALTdb(args.confFn)
syslogFile = args.syslog
icnt = 0
t1 = time.time()
try:
rmapT = Rmap(args.rmapD).reverseMapT()
except Exception as e:
print(e, file=sys.stderr)
print("Failed to read reverseMap file -> exiting")
print(traceback.format_exc())
sys.exit(1)
u2acctT = {}
if (args.u2acct):
fp = open(args.u2acct, "r")
u2acctT = json.loads(fp.read())
fp.close()
lnkCnt = 0
pkgCnt = 0
runCnt = 0
badCnt = 0
count = 0
recordT = {}
timeRecord = TimeRecord()
fnA = [args.leftover, syslogFile]
parseSyslog = ParseSyslog(args.leftover)
#-----------------------------
# Figure out size in bytes.
fnSz = 0
for fn in fnA:
if (not os.path.isfile(fn)):
continue
fnSz += os.path.getsize(fn)
badsyslog = 0
count = 0
parseSyslog = ParseSyslog(args.leftover)
pbar = ProgressBar(maxVal=max(fnSz, 1), fd=sys.stdout)
random.seed()
for fn in fnA:
if (not os.path.isfile(fn)):
continue
old = (fn == args.leftover)
f = open(fn, 'r')
for line in f:
count += len(line)
pbar.update(count)
if (not ("XALT_LOGGING" in line)):
continue
try:
t, done = parseSyslog.parse(line, args.syshost, old)
except Exception as e:
badsyslog += 1
continue
if (not done):
continue
##################################
# If the json conversion fails,
# then ignore record and keep going
try:
value = json.loads(t['value'])
except Exception as e:
continue
try:
XALT_Stack.push("XALT_LOGGING: " + t['kind'] + " " +
t['syshost'])
if (t['kind'] == "link"):
XALT_Stack.push("link_to_db()")
xalt.link_to_db(rmapT, value)
XALT_Stack.pop()
lnkCnt += 1
elif (t['kind'] == "run"):
XALT_Stack.push("run_to_db()")
runTime = value['userDT']['run_time']
if (args.filter and runTime >= 1800.0
and runTime <= 3600.0):
if (random.random() > 0.01):
continue
else:
value['userDT']['probability'] = 0.01
stored = xalt.run_to_db(rmapT, u2acctT, value, timeRecord)
XALT_Stack.pop()
if (stored):
runCnt += 1
elif (t['kind'] == "pkg"):
XALT_Stack.push("pkg_to_db()")
xalt.pkg_to_db(t['syshost'], value)
XALT_Stack.pop()
pkgCnt += 1
else:
print("Error in xalt_syslog_to_db", file=sys.stderr)
XALT_Stack.pop()
except Exception as e:
print(e, file=sys.stderr)
print(traceback.format_exc())
badCnt += 1
f.close()
pbar.fini()
t2 = time.time()
rt = t2 - t1
if (args.timer):
print("Time: ", time.strftime("%T", time.gmtime(rt)))
print("total processed : ", count, ", num links: ", lnkCnt, ", num runs: ",
runCnt, ", pkgCnt: ", pkgCnt, ", badCnt: ", badCnt, ", badsyslog: ",
badsyslog)
timeRecord.print()
# if there is anything left in recordT file write it out to the leftover file.
parseSyslog.writeRecordT()
