def save(self, resultT):
ConfigFn = pathJoin(XALT_ETC_DIR,"xalt_db.conf")
RMapFn = pathJoin(XALT_ETC_DIR,"reverseMapD")
xalt = XALTdb(ConfigFn)
reverseMapT = Rmap(RMapFn).reverseMapT()
if (self._kind() == "link"):
xalt.link_to_db(reverseMapT, resultT)
else:
# kind == "run"
xalt.run_to_db(reverseMapT, resultT)
def save(self, resultT, uuid):
"""
The json table is written directly to the db.
@param resultT: The json record table
"""
if not XALTdb_available:
raise ImportError
ConfigFn = os.path.join(XALT_ETC_DIR, "xalt_db.conf")
RMapFn = os.path.join(XALT_ETC_DIR, "reverseMapD")
xalt = XALTdb(ConfigFn)
reverseMapT = Rmap(RMapFn).reverseMapT()
if self._kind() == "link":
xalt.link_to_db(reverseMapT, resultT)
else:
# kind == "run"
xalt.run_to_db(reverseMapT, resultT)
def save(self, resultT):
"""
The json table is written directly to the db.
@param resultT: The json record table
"""
if (not XALTdb_available):
raise ImportError
ConfigFn = os.path.join(XALT_ETC_DIR, "xalt_db.conf")
RMapFn = os.path.join(XALT_ETC_DIR, "reverseMapD")
xalt = XALTdb(ConfigFn)
reverseMapT = Rmap(RMapFn).reverseMapT()
if (self._kind() == "link"):
xalt.link_to_db(reverseMapT, resultT)
else:
# kind == "run"
xalt.run_to_db(reverseMapT, resultT)
def main():
"""
read from syslog file into XALT db.
"""
sA = []
sA.append("CommandLine:")
for v in sys.argv:
sA.append('"'+v+'"')
XALT_Stack.push(" ".join(sA))
args = CmdLineOptions().execute()
xalt = XALTdb(dbConfigFn(args.dbname))
syslogFile = args.syslog
# should add a check if file exists
num = int(capture("cat "+syslogFile+" | wc -l"))
icnt = 0
t1 = time.time()
rmapT = Rmap(args.rmapD).reverseMapT()
lnkCnt = 0
runCnt = 0
badCnt = 0
count = 0
recordT = {}
if (num == 0):
return
pbar = ProgressBar(maxVal=num)
fnA = [ args.leftover, syslogFile ]
for fn in fnA:
if (not os.path.isfile(fn)):
continue
f=open(fn, 'r')
for line in f:
if (not ("XALT_LOGGING" in line)):
continue
t, done = parseSyslog(line, recordT)
if (not done):
continue
try:
XALT_Stack.push("XALT_LOGGING: " + t['kind'] + " " + t['syshost'])
if ( t['kind'] == "link" ):
XALT_Stack.push("link_to_db()")
xalt.link_to_db(rmapT, json.loads(t['value']))
XALT_Stack.pop()
lnkCnt += 1
elif ( t['kind'] == "run" ):
XALT_Stack.push("run_to_db()")
xalt.run_to_db(rmapT, json.loads(t['value']))
XALT_Stack.pop()
runCnt += 1
else:
print("Error in xalt_syslog_to_db", file=sys.stderr)
XALT_Stack.pop()
except Exception as e:
print(e, file=sys.stderr)
badCnt += 1
count += 1
pbar.update(count)
f.close()
pbar.fini()
t2 = time.time()
rt = t2 - t1
if (args.timer):
print("Time: ", time.strftime("%T", time.gmtime(rt)))
print("total processed : ", count, ", num links: ", lnkCnt, ", num runs: ", runCnt, ", badCnt: ", badCnt)
leftover = args.leftover
if (os.path.isfile(leftover)):
os.rename(leftover, leftover + ".old")
# if there is anything left in recordT file write it out to the leftover file.
if (recordT):
f = open(leftover, "w")
for key in recordT:
r = recordT[key]
s = r.prt("XALT_LOGGING V=2", key)
f.write(s)
f.close()
def main():
"""
read from syslog file into XALT db.
"""
sA = []
sA.append("CommandLine:")
for v in sys.argv:
sA.append('"' + v + '"')
XALT_Stack.push(" ".join(sA))
args = CmdLineOptions().execute()
xalt = XALTdb(dbConfigFn(args.dbname))
syslogFile = args.syslog
# should add a check if file exists
num = int(capture("cat " + syslogFile + " | wc -l"))
icnt = 0
t1 = time.time()
rmapT = Rmap(args.rmapD).reverseMapT()
lnkCnt = 0
runCnt = 0
badCnt = 0
count = 0
recordT = {}
if (num == 0):
return
pbar = ProgressBar(maxVal=num)
fnA = [args.leftover, syslogFile]
for fn in fnA:
if (not os.path.isfile(fn)):
continue
f = open(fn, 'r')
for line in f:
if (not ("XALT_LOGGING" in line)):
continue
t, done = parseSyslog(line, recordT)
if (not done):
continue
try:
XALT_Stack.push("XALT_LOGGING: " + t['kind'] + " " +
t['syshost'])
if (t['kind'] == "link"):
XALT_Stack.push("link_to_db()")
xalt.link_to_db(rmapT, json.loads(t['value']))
XALT_Stack.pop()
lnkCnt += 1
elif (t['kind'] == "run"):
XALT_Stack.push("run_to_db()")
xalt.run_to_db(rmapT, json.loads(t['value']))
XALT_Stack.pop()
runCnt += 1
else:
print("Error in xalt_syslog_to_db", file=sys.stderr)
XALT_Stack.pop()
except Exception as e:
print(e, file=sys.stderr)
badCnt += 1
count += 1
pbar.update(count)
f.close()
pbar.fini()
t2 = time.time()
rt = t2 - t1
if (args.timer):
print("Time: ", time.strftime("%T", time.gmtime(rt)))
print("total processed : ", count, ", num links: ", lnkCnt, ", num runs: ",
runCnt, ", badCnt: ", badCnt)
leftover = args.leftover
if (os.path.isfile(leftover)):
os.rename(leftover, leftover + ".old")
# if there is anything left in recordT file write it out to the leftover file.
if (recordT):
f = open(leftover, "w")
for key in recordT:
r = recordT[key]
s = r.prt("XALT_LOGGING V=2", key)
f.write(s)
f.close()
def main():
"""
read from syslog file into XALT db.
"""
sA = []
sA.append("CommandLine:")
for v in sys.argv:
sA.append('"' + v + '"')
XALT_Stack.push(" ".join(sA))
args = CmdLineOptions().execute()
xalt = XALTdb(ConfigFn)
syslogFile = args.syslog
# should add a check if file exists
num = int(capture("cat " + syslogFile + " | wc -l"))
icnt = 0
t1 = time.time()
rmapT = Rmap(args.rmapD).reverseMapT()
lnkCnt = 0
runCnt = 0
badCnt = 0
count = 0
if num != 0:
pbar = ProgressBar(maxVal=num)
if (os.path.isfile(syslogFile)):
f = open(syslogFile, 'r')
for line in f:
if "XALT_LOGGING" in line:
i = line.find("link:")
if i == -1:
i = line.find("run:")
if i == -1:
continue # did not find a link or run, continue to next line
array = line[i:].split(":")
type = array[0].strip()
syshost = array[1].strip()
try:
resultT = json.loads(base64.b64decode(array[2]))
XALT_Stack.push("XALT_LOGGING: " + type + " " +
syshost)
# XALT_Stack.push("XALT_LOGGING resultT: " + resultT)
if (type == "link"):
XALT_Stack.push("link_to_db()")
xalt.link_to_db(rmapT, resultT)
XALT_Stack.pop()
lnkCnt += 1
elif (type == "run"):
XALT_Stack.push("run_to_db()")
xalt.run_to_db(rmapT, resultT)
XALT_Stack.pop()
runCnt += 1
else:
print("Error in xalt_syslog_to_db")
XALT_Stack.pop()
except:
badCnt += 1
# figure out length of array[2], as it might be
# near the syslog message size limit
strg = array[2]
lens = len(strg)
# lenx = lens - (lens % 4 if lens % 4 else 4)
print(
"xalt_syslog_to_db: undecodable entry! length: ",
lens)
# resultT = base64.decodestring(strg[:lenx])
# print("result: ",result)
count += 1
pbar.update(count)
pbar.fini()
# what should be done if there are errors?
# what went wrong?
# how do we restart?
#
# xalt.connect().close()
t2 = time.time()
rt = t2 - t1
if (args.timer):
print("Time: ", time.strftime("%T", time.gmtime(rt)))
#
print("total processed : ", count, ", num links: ", lnkCnt, ", num runs: ",
runCnt, ", badCnt: ", badCnt)
def main():
"""
read from syslog file into XALT db.
"""
sA = []
sA.append("CommandLine:")
for v in sys.argv:
sA.append('"' + v + '"')
XALT_Stack.push(" ".join(sA))
args = CmdLineOptions().execute()
xalt = XALTdb(args.confFn)
syslogFile = args.syslog
icnt = 0
t1 = time.time()
try:
rmapT = Rmap(args.rmapD).reverseMapT()
except Exception as e:
print(e, file=sys.stderr)
print("Failed to read reverseMap file -> exiting")
sys.exit(1)
lnkCnt = 0
pkgCnt = 0
runCnt = 0
badCnt = 0
count = 0
recordT = {}
fnA = [args.leftover, syslogFile]
parseSyslog = ParseSyslog(args.leftover)
#-----------------------------
# Figure out size in bytes.
fnSz = 0
for fn in fnA:
if (not os.path.isfile(fn)):
continue
fnSz += os.path.getsize(fn)
#----------------------------------------------------------
# Count the number and sum the run_time for all scalar jobs
filter = Filter(100)
pbar = ProgressBar(maxVal=fnSz, fd=sys.stdout)
for fn in fnA:
if (not os.path.isfile(fn)):
continue
old = (fn == args.leftover)
lineNo = 0
f = open(fn, 'r')
for line in f:
lineNo += 1
count += len(line)
pbar.update(count)
if (not ("XALT_LOGGING" in line)):
continue
try:
t, done = parseSyslog.parse(line, args.syshost, old)
except Exception as e:
#print(e, file=sys.stderr)
#print("lineNo:",lineNo,"file:",fn,"line:",line, file=sys.stderr)
#print("Now continuing processing!", file=sys.stderr)
continue
if (not done or t['kind'] != "run"):
continue
##################################
# If the json conversion fails,
# then ignore record and keep going
value = False
try:
value = json.loads(t['value'])
filter.register(value)
except Exception as e:
#print("fn:",fn,"line:",lineNo,"value:",t['value'],file=sys.stderr)
continue
f.close()
pbar.fini()
filter.report_stats()
badsyslog = 0
count = 0
parseSyslog = ParseSyslog(args.leftover)
pbar = ProgressBar(maxVal=max(fnSz, 1), fd=sys.stdout)
for fn in fnA:
if (not os.path.isfile(fn)):
continue
old = (fn == args.leftover)
f = open(fn, 'r')
for line in f:
count += len(line)
pbar.update(count)
if (not ("XALT_LOGGING" in line)):
continue
try:
t, done = parseSyslog.parse(line, args.syshost, old)
except Exception as e:
badsyslog += 1
continue
if (not done):
continue
##################################
# If the json conversion fails,
# then ignore record and keep going
try:
value = json.loads(t['value'])
except Exception as e:
continue
try:
XALT_Stack.push("XALT_LOGGING: " + t['kind'] + " " +
t['syshost'])
if (t['kind'] == "link"):
XALT_Stack.push("link_to_db()")
xalt.link_to_db(rmapT, value)
XALT_Stack.pop()
lnkCnt += 1
elif (t['kind'] == "run"):
if (filter.apply(value)):
XALT_Stack.push("run_to_db()")
xalt.run_to_db(rmapT, value)
XALT_Stack.pop()
runCnt += 1
elif (t['kind'] == "pkg"):
XALT_Stack.push("pkg_to_db()")
xalt.pkg_to_db(t['syshost'], value)
XALT_Stack.pop()
pkgCnt += 1
else:
print("Error in xalt_syslog_to_db", file=sys.stderr)
XALT_Stack.pop()
except Exception as e:
print(e, file=sys.stderr)
badCnt += 1
f.close()
pbar.fini()
t2 = time.time()
rt = t2 - t1
if (args.timer):
print("Time: ", time.strftime("%T", time.gmtime(rt)))
print("total processed : ", count, ", num links: ", lnkCnt, ", num runs: ",
runCnt, ", pkgCnt: ", pkgCnt, ", badCnt: ", badCnt, ", badsyslog: ",
badsyslog)
# if there is anything left in recordT file write it out to the leftover file.
parseSyslog.writeRecordT()
def main():
"""
read from syslog file into XALT db.
"""
sA = []
sA.append("CommandLine:")
for v in sys.argv:
sA.append('"' + v + '"')
XALT_Stack.push(" ".join(sA))
args = CmdLineOptions().execute()
xalt = XALTdb(args.confFn)
syslogFile = args.syslog
icnt = 0
t1 = time.time()
try:
rmapT = Rmap(args.rmapD).reverseMapT()
except Exception as e:
print(e, file=sys.stderr)
print("Failed to read reverseMap file -> exiting")
print(traceback.format_exc())
sys.exit(1)
u2acctT = {}
if (args.u2acct):
fp = open(args.u2acct, "r")
u2acctT = json.loads(fp.read())
fp.close()
lnkCnt = 0
pkgCnt = 0
runCnt = 0
badCnt = 0
count = 0
recordT = {}
timeRecord = TimeRecord()
fnA = [args.leftover, syslogFile]
parseSyslog = ParseSyslog(args.leftover)
#-----------------------------
# Figure out size in bytes.
fnSz = 0
for fn in fnA:
if (not os.path.isfile(fn)):
continue
fnSz += os.path.getsize(fn)
badsyslog = 0
count = 0
parseSyslog = ParseSyslog(args.leftover)
pbar = ProgressBar(maxVal=max(fnSz, 1), fd=sys.stdout)
random.seed()
for fn in fnA:
if (not os.path.isfile(fn)):
continue
old = (fn == args.leftover)
f = open(fn, 'r')
for line in f:
count += len(line)
pbar.update(count)
if (not ("XALT_LOGGING" in line)):
continue
try:
t, done = parseSyslog.parse(line, args.syshost, old)
except Exception as e:
badsyslog += 1
continue
if (not done):
continue
##################################
# If the json conversion fails,
# then ignore record and keep going
try:
value = json.loads(t['value'])
except Exception as e:
continue
try:
XALT_Stack.push("XALT_LOGGING: " + t['kind'] + " " +
t['syshost'])
if (t['kind'] == "link"):
XALT_Stack.push("link_to_db()")
xalt.link_to_db(rmapT, value)
XALT_Stack.pop()
lnkCnt += 1
elif (t['kind'] == "run"):
XALT_Stack.push("run_to_db()")
runTime = value['userDT']['run_time']
if (args.filter and runTime >= 1800.0
and runTime <= 3600.0):
if (random.random() > 0.01):
continue
else:
value['userDT']['probability'] = 0.01
stored = xalt.run_to_db(rmapT, u2acctT, value, timeRecord)
XALT_Stack.pop()
if (stored):
runCnt += 1
elif (t['kind'] == "pkg"):
XALT_Stack.push("pkg_to_db()")
xalt.pkg_to_db(t['syshost'], value)
XALT_Stack.pop()
pkgCnt += 1
else:
print("Error in xalt_syslog_to_db", file=sys.stderr)
XALT_Stack.pop()
except Exception as e:
print(e, file=sys.stderr)
print(traceback.format_exc())
badCnt += 1
f.close()
pbar.fini()
t2 = time.time()
rt = t2 - t1
if (args.timer):
print("Time: ", time.strftime("%T", time.gmtime(rt)))
print("total processed : ", count, ", num links: ", lnkCnt, ", num runs: ",
runCnt, ", pkgCnt: ", pkgCnt, ", badCnt: ", badCnt, ", badsyslog: ",
badsyslog)
timeRecord.print()
# if there is anything left in recordT file write it out to the leftover file.
parseSyslog.writeRecordT()
def main():
"""
read from syslog file into XALT db.
"""
sA = []
sA.append("CommandLine:")
for v in sys.argv:
sA.append('"'+v+'"')
XALT_Stack.push(" ".join(sA))
args = CmdLineOptions().execute()
xalt = XALTdb(ConfigFn)
syslogFile = args.syslog
# should add a check if file exists
num = int(capture("cat "+syslogFile+" | wc -l"))
icnt = 0
t1 = time.time()
rmapT = Rmap(args.rmapD).reverseMapT()
lnkCnt = 0
runCnt = 0
badCnt = 0
count = 0
if num != 0:
pbar = ProgressBar(maxVal=num)
if (os.path.isfile(syslogFile)):
f=open(syslogFile, 'r')
for line in f:
if "XALT_LOGGING" in line:
i=line.find("link:")
if i == -1 :
i=line.find("run:")
if i == -1:
continue # did not find a link or run, continue to next line
array=line[i:].split(":")
type = array[0].strip()
syshost = array[1].strip()
try:
resultT=json.loads(base64.b64decode(array[2]))
XALT_Stack.push("XALT_LOGGING: " + type + " " + syshost)
# XALT_Stack.push("XALT_LOGGING resultT: " + resultT)
if ( type == "link" ):
XALT_Stack.push("link_to_db()")
xalt.link_to_db(rmapT, resultT)
XALT_Stack.pop()
lnkCnt += 1
elif ( type == "run" ):
XALT_Stack.push("run_to_db()")
xalt.run_to_db(rmapT, resultT)
XALT_Stack.pop()
runCnt += 1
else:
print("Error in xalt_syslog_to_db")
XALT_Stack.pop()
except:
badCnt += 1
# figure out length of array[2], as it might be
# near the syslog message size limit
strg=array[2]
lens = len(strg)
# lenx = lens - (lens % 4 if lens % 4 else 4)
print("xalt_syslog_to_db: undecodable entry! length: ",lens)
# resultT = base64.decodestring(strg[:lenx])
# print("result: ",result)
count += 1
pbar.update(count)
pbar.fini()
# what should be done if there are errors?
# what went wrong?
# how do we restart?
#
# xalt.connect().close()
t2 = time.time()
rt = t2 - t1
if (args.timer):
print("Time: ", time.strftime("%T", time.gmtime(rt)))
#
print("total processed : ", count, ", num links: ", lnkCnt, ", num runs: ", runCnt, ", badCnt: ", badCnt)
