def main():
"""
This program removes the Database used by XALT.
"""
args = CmdLineOptions().execute()
configFn = dbConfigFn(args.dbname)
if (not os.path.isfile(configFn)):
dirNm, exe = os.path.split(sys.argv[0])
fn = os.path.join(dirNm, configFn)
if (os.path.isfile(fn)):
configFn = fn
else:
configFn = os.path.abspath(os.path.join(dirNm, "../site",
configFn))
xalt = XALTdb(configFn)
db = xalt.db()
try:
conn = xalt.connect()
cursor = conn.cursor()
# If MySQL version < 4.1, comment out the line below
cursor.execute("SET SQL_MODE=\"NO_AUTO_VALUE_ON_ZERO\"")
# If the database does not exist, create it, otherwise, switch to the database.
cursor.execute("DROP DATABASE IF EXISTS %s " % xalt.db())
cursor.close()
except MySQLdb.Error, e:
print("Error %d: %s" % (e.args[0], e.args[1]))
sys.exit(1)
def main():
"""
Check the entries for XALT db.
"""
args = CmdLineOptions().execute()
xalt = XALTdb(dbConfigFn(args.dbname))
conn = xalt.connect()
query = "USE "+xalt.db()
conn.query(query)
tableA = [ 'join_link_object', 'join_run_env', 'join_run_object', \
'xalt_env_name', 'xalt_link', 'xalt_object', 'xalt_run',
'xalt_function', 'join_link_function','xalt_total_env']
tableT = {}
for tableName in tableA:
count = count_rows(conn, tableName)
tableT[tableName] = count
print(tableName,":", count)
conn.close()
result = 'diff'
num = int(args.runs)
if (tableT['xalt_link'] > 1 and
tableT['xalt_run'] == num and
tableT['xalt_object'] > 10 and
tableT['xalt_env_name'] > 5 ):
result = "passed"
f = open(args.resultFn,"w")
f.write(result+"\n")
f.close()
def main():
ConfigBaseNm = "xalt_db"
ConfigFn = ConfigBaseNm + ".conf"
if (not os.path.isfile(ConfigFn)):
dirNm, prg = os.path.split(sys.argv[0])
fn = os.path.join(dirNm, ConfigFn)
if (os.path.isfile(fn)):
ConfigFn = fn
else:
ConfigFn = os.path.abspath(os.path.join(dirNm, "../sbin",
ConfigFn))
xalt = XALTdb(ConfigFn)
try:
conn = xalt.connect()
query = "USE " + xalt.db()
conn.query(query)
query = "SELECT job_id FROM xalt_job_id WHERE inc_id='1'"
conn.query(query)
result = conn.store_result()
if (result.num_rows() > 0):
row = result.fetch_row()
job_id = int(row[0][0]) + 1
query = "UPDATE xalt_job_id SET job_id='%d' WHERE inc_id='1'" % job_id
conn.query(query)
print(job_id)
except MySQLdb.Error, e:
print("Error %d: %s" % (e.args[0], e.args[1]))
sys.exit(1)
def main():
"""
This program removes the Database used by XALT.
"""
args = CmdLineOptions().execute()
configFn = dbConfigFn(args.dbname)
if (not os.path.isfile(configFn)):
dirNm, exe = os.path.split(sys.argv[0])
fn = os.path.join(dirNm, configFn)
if (os.path.isfile(fn)):
configFn = fn
else:
configFn = os.path.abspath(os.path.join(dirNm, "../site", configFn))
xalt = XALTdb(configFn)
db = xalt.db()
try:
conn = xalt.connect()
cursor = conn.cursor()
# If MySQL version < 4.1, comment out the line below
cursor.execute("SET SQL_MODE=\"NO_AUTO_VALUE_ON_ZERO\"")
# If the database does not exist, create it, otherwise, switch to the database.
cursor.execute("DROP DATABASE IF EXISTS %s " % xalt.db())
cursor.close()
except MySQLdb.Error as e:
print ("Error %d: %s" % (e.args[0], e.args[1]))
sys.exit (1)
def main():
"""
Walks the list of users via the passwd_generator and load the
link and run files.
"""
# Push command line on to XALT_Stack
sA = []
sA.append("CommandLine:")
for v in sys.argv:
sA.append('"' + v + '"')
XALT_Stack.push(" ".join(sA))
args = CmdLineOptions().execute()
xalt = XALTdb(dbConfigFn(args.dbname))
num = int(capture("getent passwd | wc -l"))
pbar = ProgressBar(maxVal=num)
icnt = 0
t1 = time.time()
rmapT = Rmap(args.rmapD).reverseMapT()
iuser = 0
lnkCnt = 0
runCnt = 0
xaltDir = args.datadir
if (os.path.isdir(xaltDir)):
iuser += 1
linkFnA = files_in_tree(xaltDir, "*/link.*.json")
XALT_Stack.push("link_json_to_db()")
lnkCnt += link_json_to_db(xalt, args.listFn, rmapT, linkFnA)
XALT_Stack.pop()
if (args.delete):
remove_files(linkFnA)
#remove_files(files_in_tree(xaltDir, "*/.link.*.json"))
runFnA = files_in_tree(xaltDir, "*/run.*.json")
XALT_Stack.push("run_json_to_db()")
runCnt += run_json_to_db(xalt, args.listFn, rmapT, runFnA)
XALT_Stack.pop()
if (args.delete):
remove_files(runFnA)
#remove_files(files_in_tree(xaltDir, "*/.run.*.json"))
icnt += 1
pbar.update(icnt)
xalt.connect().close()
pbar.fini()
t2 = time.time()
rt = t2 - t1
if (args.timer):
print("Time: ", time.strftime("%T", time.gmtime(rt)))
print("num users: ", iuser, ", num links: ", lnkCnt, ", num runs: ",
runCnt)
def main():
"""
Walks the list of users via the passwd_generator and load the
link and run files.
"""
# Push command line on to XALT_Stack
sA = []
sA.append("CommandLine:")
for v in sys.argv:
sA.append('"'+v+'"')
XALT_Stack.push(" ".join(sA))
args = CmdLineOptions().execute()
xalt = XALTdb(dbConfigFn(args.dbname))
num = int(capture("getent passwd | wc -l"))
pbar = ProgressBar(maxVal=num)
icnt = 0
t1 = time.time()
rmapT = Rmap(args.rmapD).reverseMapT()
iuser = 0
lnkCnt = 0
runCnt = 0
xaltDir = args.datadir
if (os.path.isdir(xaltDir)):
iuser += 1
linkFnA = files_in_tree(xaltDir, "*/link.*.json")
XALT_Stack.push("link_json_to_db()")
lnkCnt += link_json_to_db(xalt, args.listFn, rmapT, linkFnA)
XALT_Stack.pop()
if (args.delete):
remove_files(linkFnA)
#remove_files(files_in_tree(xaltDir, "*/.link.*.json"))
runFnA = files_in_tree(xaltDir, "*/run.*.json")
XALT_Stack.push("run_json_to_db()")
runCnt += run_json_to_db(xalt, args.listFn, rmapT, runFnA)
XALT_Stack.pop()
if (args.delete):
remove_files(runFnA)
#remove_files(files_in_tree(xaltDir, "*/.run.*.json"))
icnt += 1
pbar.update(icnt)
xalt.connect().close()
pbar.fini()
t2 = time.time()
rt = t2 - t1
if (args.timer):
print("Time: ", time.strftime("%T", time.gmtime(rt)))
print("num users: ", iuser, ", num links: ", lnkCnt, ", num runs: ", runCnt)
def save(self, resultT):
ConfigFn = pathJoin(XALT_ETC_DIR,"xalt_db.conf")
RMapFn = pathJoin(XALT_ETC_DIR,"reverseMapD")
xalt = XALTdb(ConfigFn)
reverseMapT = Rmap(RMapFn).reverseMapT()
if (self._kind() == "link"):
xalt.link_to_db(reverseMapT, resultT)
else:
# kind == "run"
xalt.run_to_db(reverseMapT, resultT)
def main():
"""
Check the entries for XALT db.
"""
args = CmdLineOptions().execute()
xalt = XALTdb(dbConfigFn(args.dbname))
conn = xalt.connect()
query = "USE "+xalt.db()
conn.query(query)
tableA = [ 'join_link_object', 'join_run_env', 'join_run_object', \
'xalt_env_name', 'xalt_link', 'xalt_object', 'xalt_run',
'xalt_function', 'join_link_function','xalt_pkg']
tableT = {}
for tableName in tableA:
count = count_rows(conn, tableName)
tableT[tableName] = count
print(tableName,":", count)
conn.close()
result = 'diff'
num = int(args.runs)
nfuncs = int(args.nfuncs)
nlinks = int(args.links)
pkgs = int(args.pkgs)
objs = int(args.objects)
if (tableT['xalt_link'] == nlinks and
tableT['xalt_run'] == num and
tableT['xalt_pkg'] == pkgs and
tableT['xalt_function'] >= nfuncs and
tableT['xalt_object'] >= objs and
tableT['xalt_env_name'] > 4 ):
result = "passed"
else:
print("Expected:")
print(" links: ",nlinks)
print(" runs: ",num)
print(" pkgs: ",pkgs)
print(" functions >= ",nfuncs)
print(" objects >= ",objs)
print(" env names > 4")
f = open(args.resultFn,"w")
f.write(result+"\n")
f.close()
def save(self, resultT, uuid):
"""
The json table is written directly to the db.
@param resultT: The json record table
"""
if not XALTdb_available:
raise ImportError
ConfigFn = os.path.join(XALT_ETC_DIR, "xalt_db.conf")
RMapFn = os.path.join(XALT_ETC_DIR, "reverseMapD")
xalt = XALTdb(ConfigFn)
reverseMapT = Rmap(RMapFn).reverseMapT()
if self._kind() == "link":
xalt.link_to_db(reverseMapT, resultT)
else:
# kind == "run"
xalt.run_to_db(reverseMapT, resultT)
def main():
args = CmdLineOptions().execute()
xalt = XALTdb(ConfigFn)
num = int(capture("getent passwd | wc -l"))
pbar = ProgressBar(maxVal=num)
icnt = 0
t1 = time.time()
rmapT = Rmap(args.rmapD).reverseMapT()
iuser = 0
lnkCnt = 0
runCnt = 0
for user, hdir in passwd_generator():
xaltDir = os.path.join(hdir, ".xalt.d")
if (os.path.isdir(xaltDir)):
iuser += 1
linkFnA = files_in_tree(xaltDir, "*/link.*.json")
lnkCnt += link_json_to_db(xalt, user, rmapT, linkFnA)
if (args.delete):
remove_files(linkFnA)
remove_files(files_in_tree(xaltDir, "*/.link.*.json"))
runFnA = files_in_tree(xaltDir, "*/run.*.json")
runCnt += run_json_to_db(xalt, user, rmapT, runFnA)
if (args.delete):
remove_files(runFnA)
remove_files(files_in_tree(xaltDir, "*/.run.*.json"))
icnt += 1
pbar.update(icnt)
xalt.connect().close()
pbar.fini()
t2 = time.time()
rt = t2 - t1
if (args.timer):
print("Time: ", time.strftime("%T", time.gmtime(rt)))
print("num users: ", iuser, ", num links: ", lnkCnt, ", num runs: ",
runCnt)
def main():
"""
This program upgrade the XALT database schema from 0.7.5 to 1.1.3
"""
args = CmdLineOptions().execute()
configFn = dbConfigFn(args.dbname)
if (not os.path.isfile(configFn)):
dirNm, exe = os.path.split(sys.argv[0])
fn = os.path.join(dirNm, configFn)
if (os.path.isfile(fn)):
configFn = fn
else:
configFn = os.path.abspath(os.path.join(dirNm, "../site",
configFn))
xalt = XALTdb(configFn)
db = xalt.db()
try:
conn = xalt.connect()
cursor = conn.cursor()
# If MySQL version < 4.1, comment out the line below
cursor.execute("SET SQL_MODE=\"NO_AUTO_VALUE_ON_ZERO\"")
cursor.execute("USE " + xalt.db())
idx = 1
print("start")
# 1
cursor.execute("""
ALTER TABLE `xalt_link`
ADD COLUMN `cwd` varchar(1024) NULL AFTER `link_line`
""")
print("(%d) upgraded xalt_link table" % idx)
idx += 1
cursor.close()
except MySQLdb.Error as e:
print("Error %d: %s" % (e.args[0], e.args[1]))
sys.exit(1)
def main():
ConfigBaseNm = "xalt_db"
ConfigFn = ConfigBaseNm + ".conf"
if (not os.path.isfile(ConfigFn)):
dirNm,prg = os.path.split(sys.argv[0])
fn = os.path.join(dirNm, ConfigFn)
if (os.path.isfile(fn)):
ConfigFn = fn
else:
ConfigFn = os.path.abspath(os.path.join(dirNm,"../sbin",ConfigFn))
xalt = XALTdb(ConfigFn)
try:
conn = xalt.connect()
except MySQLdb.Error, e:
print("-1")
sys.exit(1)
def main():
ConfigBaseNm = "xalt_db"
ConfigFn = ConfigBaseNm + ".conf"
if (not os.path.isfile(ConfigFn)):
dirNm,prg = os.path.split(sys.argv[0])
fn = os.path.join(dirNm, ConfigFn)
if (os.path.isfile(fn)):
ConfigFn = fn
else:
ConfigFn = os.path.abspath(os.path.join(dirNm,"../sbin",ConfigFn))
xalt = XALTdb(ConfigFn)
try:
conn = xalt.connect()
except MySQLdb.Error as e:
print("-1")
sys.exit(1)
try:
query = "USE "+xalt.db()
conn.query(query)
query = "SELECT job_id FROM xalt_job_id WHERE inc_id='1'"
conn.query(query)
result = conn.store_result()
if (result.num_rows() > 0):
row = result.fetch_row()
job_id = int(row[0][0]) + 1
query = "UPDATE xalt_job_id SET job_id='%d' WHERE inc_id='1'" % job_id
conn.query(query)
print(job_id)
except MySQLdb.Error as e:
print ("Error %d: %s" % (e.args[0], e.args[1]))
sys.exit (1)
def main():
ConfigBaseNm = "xalt_db"
ConfigFn = ConfigBaseNm + ".conf"
if (not os.path.isfile(ConfigFn)):
dirNm, prg = os.path.split(sys.argv[0])
fn = os.path.join(dirNm, ConfigFn)
if (os.path.isfile(fn)):
ConfigFn = fn
else:
ConfigFn = os.path.abspath(os.path.join(dirNm, "../sbin",
ConfigFn))
xalt = XALTdb(ConfigFn)
try:
conn = xalt.connect()
except MySQLdb.Error, e:
print("-1")
sys.exit(1)
def main():
args = CmdLineOptions().execute()
xalt = XALTdb(ConfigFn)
num = int(capture("getent passwd | wc -l"))
pbar = ProgressBar(maxVal=num)
icnt = 0
t1 = time.time()
rmapT = Rmap(args.rmapD).reverseMapT()
iuser = 0
lnkCnt = 0
runCnt = 0
for user, hdir in passwd_generator():
xaltDir = os.path.join(hdir,".xalt.d")
if (os.path.isdir(xaltDir)):
iuser += 1
linkFnA = files_in_tree(xaltDir, "*/link.*.json")
lnkCnt += link_json_to_db(xalt, user, rmapT, linkFnA)
if (args.delete):
remove_files(linkFnA)
remove_files(files_in_tree(xaltDir, "*/.link.*.json"))
runFnA = files_in_tree(xaltDir, "*/run.*.json")
runCnt += run_json_to_db(xalt, user, rmapT, runFnA)
if (args.delete):
remove_files(runFnA)
remove_files(files_in_tree(xaltDir, "*/.run.*.json"))
icnt += 1
pbar.update(icnt)
xalt.connect().close()
pbar.fini()
t2 = time.time()
rt = t2 - t1
if (args.timer):
print("Time: ", time.strftime("%T", time.gmtime(rt)))
print("num users: ", iuser, ", num links: ", lnkCnt, ", num runs: ", runCnt)
def main():
"""
This program upgrade the XALT database schema from 0.7.5 to 1.1.3
"""
args = CmdLineOptions().execute()
configFn = dbConfigFn(args.dbname)
if (not os.path.isfile(configFn)):
dirNm, exe = os.path.split(sys.argv[0])
fn = os.path.join(dirNm, configFn)
if (os.path.isfile(fn)):
configFn = fn
else:
configFn = os.path.abspath(os.path.join(dirNm, "../site", configFn))
xalt = XALTdb(configFn)
db = xalt.db()
try:
conn = xalt.connect()
cursor = conn.cursor()
# If MySQL version < 4.1, comment out the line below
cursor.execute("SET SQL_MODE=\"NO_AUTO_VALUE_ON_ZERO\"")
cursor.execute("USE "+xalt.db())
idx = 1
print("start")
# 1
cursor.execute("""
ALTER TABLE `xalt_link`
ADD COLUMN `cwd` varchar(1024) NULL AFTER `link_line`
""")
print("(%d) upgraded xalt_link table" % idx); idx += 1
cursor.close()
except MySQLdb.Error as e:
print ("Error %d: %s" % (e.args[0], e.args[1]))
sys.exit (1)
def save(self, resultT):
"""
The json table is written directly to the db.
@param resultT: The json record table
"""
if (not XALTdb_available):
raise ImportError
ConfigFn = os.path.join(XALT_ETC_DIR, "xalt_db.conf")
RMapFn = os.path.join(XALT_ETC_DIR, "reverseMapD")
xalt = XALTdb(ConfigFn)
reverseMapT = Rmap(RMapFn).reverseMapT()
if (self._kind() == "link"):
xalt.link_to_db(reverseMapT, resultT)
else:
# kind == "run"
xalt.run_to_db(reverseMapT, resultT)
def main():
"""
Check the entries for XALT db.
"""
args = CmdLineOptions().execute()
xalt = XALTdb(dbConfigFn(args.dbname))
conn = xalt.connect()
query = "USE " + xalt.db()
conn.query(query)
tableA = [ 'join_link_object', 'join_run_env', 'join_run_object', \
'xalt_env_name', 'xalt_link', 'xalt_object', \
'xalt_function', 'join_link_function','xalt_pkg']
tableT = {}
for tableName in tableA:
count = count_w_condition(conn, tableName, None)
tableT[tableName] = count
print(tableName, ":", count)
tableT['xalt_run'] = count_w_condition(conn, "xalt_run", "end_time > 0.0")
print("xalt_run :", tableT['xalt_run'])
tableT['build_uuid'] = count_w_condition(conn, "xalt_run",
"uuid is not null")
print("build_uuid :", tableT['build_uuid'])
conn.close()
result = 'passed'
testT = {
'xalt_link': [int(args.links), eq],
'xalt_run': [int(args.runs), eq],
'xalt_pkg': [int(args.pkgs), ge],
'xalt_function': [int(args.nfuncs), ge],
'xalt_object': [int(args.objects), ge],
'xalt_env_name': [int(args.numEnv), ge],
'build_uuid': [int(args.uuid), ge],
}
sA = []
for key in testT:
value = tableT[key]
goldV = testT[key][0]
tstFn = testT[key][1]
my_result, kind = tstFn(value, goldV)
if (not my_result):
sA.append("Not correct: " + key + ": " + str(value) +
" -> ( Should be: " + key + " " + kind + " " +
str(goldV) + " )")
if (sA):
result = "diff"
red = "31m"
banner = display_colored_text(
red,
"======================================================================"
)
print(banner)
for line in sA:
print(line)
print(banner)
f = open(args.resultFn, "w")
f.write(result + "\n")
f.close()
def main():
"""
Walks the list of users via the passwd_generator and load the
link and run files.
"""
# Using home directories or a global location.
xalt_file_prefix = os.environ.get("XALT_FILE_PREFIX", "@xalt_file_prefix@")
xaltUserA = os.environ.get("XALT_USERS")
if (xaltUserA):
xaltUserStr = ", XALT_USERS: \"" + xaltUserA + "\""
else:
xaltUserStr = ""
print("\n################################################################")
print("XALT Git Version: " + Version())
print(" Using XALT_FILE_PREFIX: \"" + xalt_file_prefix + "\"" +
xaltUserStr)
print("################################################################\n")
# Find transmission style
transmission = os.environ.get("XALT_TRANSMISSION_STYLE")
if (not transmission):
transmission = "@transmission@"
if (not transmission):
transmission = "file"
transmission = transmission.lower()
timeRecord = TimeRecord()
# Push command line on to XALT_Stack
sA = []
sA.append("CommandLine:")
for v in sys.argv:
sA.append('"' + v + '"')
XALT_Stack.push(" ".join(sA))
args = CmdLineOptions().execute()
xalt = XALTdb(args.confFn)
if (xalt_file_prefix == "USE_HOME"):
num = int(capture("LD_PRELOAD= getent passwd | wc -l"))
pbar = ProgressBar(maxVal=num)
else:
xaltDir = build_resultDir("", transmission, "")
allFnA = files_in_tree(xaltDir, "*/*." + args.syshost + ".*.json")
pbar = ProgressBar(maxVal=len(allFnA))
icnt = 0
t1 = time.time()
rmapT = Rmap(args.rmapD).reverseMapT()
u2acctT = {}
if (args.u2acct):
fp = open(args.u2acct, "r")
u2acctT = json.loads(fp.read())
fp.close()
countT = {}
countT['lnk'] = 0
countT['run'] = 0
countT['pkg'] = 0
countT['any'] = 0
if (xalt_file_prefix == "USE_HOME"):
for user, homeDir in passwd_generator():
store_json_files(homeDir, transmission, xalt, rmapT, u2acctT, args,
countT, pbar, timeRecord)
else:
store_json_files("", transmission, xalt, rmapT, u2acctT, args, countT,
pbar, timeRecord)
xalt.connect().close()
pbar.fini()
t2 = time.time()
rt = t2 - t1
if (args.timer):
print("Time: ", time.strftime("%T", time.gmtime(rt)))
print("num links: ", countT['lnk'], ", num pkgs: ", countT['pkg'],
", num runs: ", countT['run'])
timeRecord.print()
def main():
"""
read from syslog file into XALT db.
"""
sA = []
sA.append("CommandLine:")
for v in sys.argv:
sA.append('"'+v+'"')
XALT_Stack.push(" ".join(sA))
args = CmdLineOptions().execute()
xalt = XALTdb(dbConfigFn(args.dbname))
syslogFile = args.syslog
# should add a check if file exists
num = int(capture("cat "+syslogFile+" | wc -l"))
icnt = 0
t1 = time.time()
rmapT = Rmap(args.rmapD).reverseMapT()
lnkCnt = 0
runCnt = 0
badCnt = 0
count = 0
recordT = {}
if (num == 0):
return
pbar = ProgressBar(maxVal=num)
fnA = [ args.leftover, syslogFile ]
for fn in fnA:
if (not os.path.isfile(fn)):
continue
f=open(fn, 'r')
for line in f:
if (not ("XALT_LOGGING" in line)):
continue
t, done = parseSyslog(line, recordT)
if (not done):
continue
try:
XALT_Stack.push("XALT_LOGGING: " + t['kind'] + " " + t['syshost'])
if ( t['kind'] == "link" ):
XALT_Stack.push("link_to_db()")
xalt.link_to_db(rmapT, json.loads(t['value']))
XALT_Stack.pop()
lnkCnt += 1
elif ( t['kind'] == "run" ):
XALT_Stack.push("run_to_db()")
xalt.run_to_db(rmapT, json.loads(t['value']))
XALT_Stack.pop()
runCnt += 1
else:
print("Error in xalt_syslog_to_db", file=sys.stderr)
XALT_Stack.pop()
except Exception as e:
print(e, file=sys.stderr)
badCnt += 1
count += 1
pbar.update(count)
f.close()
pbar.fini()
t2 = time.time()
rt = t2 - t1
if (args.timer):
print("Time: ", time.strftime("%T", time.gmtime(rt)))
print("total processed : ", count, ", num links: ", lnkCnt, ", num runs: ", runCnt, ", badCnt: ", badCnt)
leftover = args.leftover
if (os.path.isfile(leftover)):
os.rename(leftover, leftover + ".old")
# if there is anything left in recordT file write it out to the leftover file.
if (recordT):
f = open(leftover, "w")
for key in recordT:
r = recordT[key]
s = r.prt("XALT_LOGGING V=2", key)
f.write(s)
f.close()
def main():
"""
read from syslog file into XALT db.
"""
sA = []
sA.append("CommandLine:")
for v in sys.argv:
sA.append('"' + v + '"')
XALT_Stack.push(" ".join(sA))
args = CmdLineOptions().execute()
xalt = XALTdb(args.confFn)
syslogFile = args.syslog
icnt = 0
t1 = time.time()
try:
rmapT = Rmap(args.rmapD).reverseMapT()
except Exception as e:
print(e, file=sys.stderr)
print("Failed to read reverseMap file -> exiting")
print(traceback.format_exc())
sys.exit(1)
u2acctT = {}
if (args.u2acct):
fp = open(args.u2acct, "r")
u2acctT = json.loads(fp.read())
fp.close()
lnkCnt = 0
pkgCnt = 0
runCnt = 0
badCnt = 0
count = 0
recordT = {}
timeRecord = TimeRecord()
fnA = [args.leftover, syslogFile]
parseSyslog = ParseSyslog(args.leftover)
#-----------------------------
# Figure out size in bytes.
fnSz = 0
for fn in fnA:
if (not os.path.isfile(fn)):
continue
fnSz += os.path.getsize(fn)
badsyslog = 0
count = 0
parseSyslog = ParseSyslog(args.leftover)
pbar = ProgressBar(maxVal=max(fnSz, 1), fd=sys.stdout)
random.seed()
for fn in fnA:
if (not os.path.isfile(fn)):
continue
old = (fn == args.leftover)
f = open(fn, 'r')
for line in f:
count += len(line)
pbar.update(count)
if (not ("XALT_LOGGING" in line)):
continue
try:
t, done = parseSyslog.parse(line, args.syshost, old)
except Exception as e:
badsyslog += 1
continue
if (not done):
continue
##################################
# If the json conversion fails,
# then ignore record and keep going
try:
value = json.loads(t['value'])
except Exception as e:
continue
try:
XALT_Stack.push("XALT_LOGGING: " + t['kind'] + " " +
t['syshost'])
if (t['kind'] == "link"):
XALT_Stack.push("link_to_db()")
xalt.link_to_db(rmapT, value)
XALT_Stack.pop()
lnkCnt += 1
elif (t['kind'] == "run"):
XALT_Stack.push("run_to_db()")
runTime = value['userDT']['run_time']
if (args.filter and runTime >= 1800.0
and runTime <= 3600.0):
if (random.random() > 0.01):
continue
else:
value['userDT']['probability'] = 0.01
stored = xalt.run_to_db(rmapT, u2acctT, value, timeRecord)
XALT_Stack.pop()
if (stored):
runCnt += 1
elif (t['kind'] == "pkg"):
XALT_Stack.push("pkg_to_db()")
xalt.pkg_to_db(t['syshost'], value)
XALT_Stack.pop()
pkgCnt += 1
else:
print("Error in xalt_syslog_to_db", file=sys.stderr)
XALT_Stack.pop()
except Exception as e:
print(e, file=sys.stderr)
print(traceback.format_exc())
badCnt += 1
f.close()
pbar.fini()
t2 = time.time()
rt = t2 - t1
if (args.timer):
print("Time: ", time.strftime("%T", time.gmtime(rt)))
print("total processed : ", count, ", num links: ", lnkCnt, ", num runs: ",
runCnt, ", pkgCnt: ", pkgCnt, ", badCnt: ", badCnt, ", badsyslog: ",
badsyslog)
timeRecord.print()
# if there is anything left in recordT file write it out to the leftover file.
parseSyslog.writeRecordT()
def main():
"""
read from syslog file into XALT db.
"""
sA = []
sA.append("CommandLine:")
for v in sys.argv:
sA.append('"' + v + '"')
XALT_Stack.push(" ".join(sA))
args = CmdLineOptions().execute()
xalt = XALTdb(dbConfigFn(args.dbname))
syslogFile = args.syslog
# should add a check if file exists
num = int(capture("cat " + syslogFile + " | wc -l"))
icnt = 0
t1 = time.time()
rmapT = Rmap(args.rmapD).reverseMapT()
lnkCnt = 0
runCnt = 0
badCnt = 0
count = 0
recordT = {}
if (num == 0):
return
pbar = ProgressBar(maxVal=num)
fnA = [args.leftover, syslogFile]
for fn in fnA:
if (not os.path.isfile(fn)):
continue
f = open(fn, 'r')
for line in f:
if (not ("XALT_LOGGING" in line)):
continue
t, done = parseSyslog(line, recordT)
if (not done):
continue
try:
XALT_Stack.push("XALT_LOGGING: " + t['kind'] + " " +
t['syshost'])
if (t['kind'] == "link"):
XALT_Stack.push("link_to_db()")
xalt.link_to_db(rmapT, json.loads(t['value']))
XALT_Stack.pop()
lnkCnt += 1
elif (t['kind'] == "run"):
XALT_Stack.push("run_to_db()")
xalt.run_to_db(rmapT, json.loads(t['value']))
XALT_Stack.pop()
runCnt += 1
else:
print("Error in xalt_syslog_to_db", file=sys.stderr)
XALT_Stack.pop()
except Exception as e:
print(e, file=sys.stderr)
badCnt += 1
count += 1
pbar.update(count)
f.close()
pbar.fini()
t2 = time.time()
rt = t2 - t1
if (args.timer):
print("Time: ", time.strftime("%T", time.gmtime(rt)))
print("total processed : ", count, ", num links: ", lnkCnt, ", num runs: ",
runCnt, ", badCnt: ", badCnt)
leftover = args.leftover
if (os.path.isfile(leftover)):
os.rename(leftover, leftover + ".old")
# if there is anything left in recordT file write it out to the leftover file.
if (recordT):
f = open(leftover, "w")
for key in recordT:
r = recordT[key]
s = r.prt("XALT_LOGGING V=2", key)
f.write(s)
f.close()
def main():
"""
read from syslog file into XALT db.
"""
sA = []
sA.append("CommandLine:")
for v in sys.argv:
sA.append('"' + v + '"')
XALT_Stack.push(" ".join(sA))
args = CmdLineOptions().execute()
xalt = XALTdb(ConfigFn)
syslogFile = args.syslog
# should add a check if file exists
num = int(capture("cat " + syslogFile + " | wc -l"))
icnt = 0
t1 = time.time()
rmapT = Rmap(args.rmapD).reverseMapT()
lnkCnt = 0
runCnt = 0
badCnt = 0
count = 0
if num != 0:
pbar = ProgressBar(maxVal=num)
if (os.path.isfile(syslogFile)):
f = open(syslogFile, 'r')
for line in f:
if "XALT_LOGGING" in line:
i = line.find("link:")
if i == -1:
i = line.find("run:")
if i == -1:
continue # did not find a link or run, continue to next line
array = line[i:].split(":")
type = array[0].strip()
syshost = array[1].strip()
try:
resultT = json.loads(base64.b64decode(array[2]))
XALT_Stack.push("XALT_LOGGING: " + type + " " +
syshost)
# XALT_Stack.push("XALT_LOGGING resultT: " + resultT)
if (type == "link"):
XALT_Stack.push("link_to_db()")
xalt.link_to_db(rmapT, resultT)
XALT_Stack.pop()
lnkCnt += 1
elif (type == "run"):
XALT_Stack.push("run_to_db()")
xalt.run_to_db(rmapT, resultT)
XALT_Stack.pop()
runCnt += 1
else:
print("Error in xalt_syslog_to_db")
XALT_Stack.pop()
except:
badCnt += 1
# figure out length of array[2], as it might be
# near the syslog message size limit
strg = array[2]
lens = len(strg)
# lenx = lens - (lens % 4 if lens % 4 else 4)
print(
"xalt_syslog_to_db: undecodable entry! length: ",
lens)
# resultT = base64.decodestring(strg[:lenx])
# print("result: ",result)
count += 1
pbar.update(count)
pbar.fini()
# what should be done if there are errors?
# what went wrong?
# how do we restart?
#
# xalt.connect().close()
t2 = time.time()
rt = t2 - t1
if (args.timer):
print("Time: ", time.strftime("%T", time.gmtime(rt)))
#
print("total processed : ", count, ", num links: ", lnkCnt, ", num runs: ",
runCnt, ", badCnt: ", badCnt)
def main():
"""
This program creates the Database used by XALT.
"""
args = CmdLineOptions().execute()
configFn = dbConfigFn(args.dbname)
if (args.confFn):
configFn = args.confFn
if (not os.path.isfile(configFn)):
dirNm, exe = os.path.split(sys.argv[0])
fn = os.path.join(dirNm, configFn)
if (os.path.isfile(fn)):
configFn = fn
else:
configFn = os.path.abspath(os.path.join(dirNm, "../site",
configFn))
xalt = XALTdb(configFn)
db = xalt.db()
try:
conn = xalt.connect()
cursor = conn.cursor()
cursor.execute("SHOW VARIABLES like 'version'")
row = cursor.fetchone()
mysqlV = row[1]
currentV = parseVersion(mysqlV)
if (currentV >= parseVersion("6.0")):
cursor.execute("SET SQL_MODE=\"NO_AUTO_VALUE_ON_ZERO\"")
elif (currentV > parseVersion("4.1")):
cursor.execute(
"SET SQL_MODE=\"NO_AUTO_VALUE_ON_ZERO,NO_AUTO_CREATE_USER\"")
# drop db if requested.
if (args.drop):
cursor.execute("DROP DATABASE IF EXISTS %s " % xalt.db())
# If the database does not exist, create it, otherwise, switch to the database.
cursor.execute(
"CREATE DATABASE IF NOT EXISTS %s DEFAULT CHARACTER SET utf8 COLLATE utf8_bin"
% xalt.db())
cursor.execute("USE " + xalt.db())
idx = 1
print("start")
# 1
cursor.execute("""
CREATE TABLE IF NOT EXISTS `xalt_link` (
`link_id` int(11) unsigned NOT NULL auto_increment,
`uuid` char(36) NOT NULL,
`hash_id` char(40) NOT NULL,
`date` DATETIME NOT NULL,
`link_program` varchar(64) NOT NULL,
`link_path` varchar(1024) NOT NULL,
`link_module_name` varchar(64) ,
`link_line` blob ,
`cwd` varchar(1024) ,
`build_user` varchar(64) NOT NULL,
`build_syshost` varchar(64) NOT NULL,
`build_epoch` double NOT NULL,
`exec_path` varchar(1024) NOT NULL,
PRIMARY KEY (`link_id`),
INDEX `index_date` (`date`),
UNIQUE KEY `uuid` (`uuid`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin AUTO_INCREMENT=1
""")
print("(%d) create xalt_link table" % idx)
idx += 1
# 2
cursor.execute("""
CREATE TABLE IF NOT EXISTS `xalt_object` (
`obj_id` int(11) unsigned NOT NULL auto_increment,
`object_path` varchar(1024) NOT NULL,
`syshost` varchar(64) NOT NULL,
`hash_id` char(40) NOT NULL,
`module_name` varchar(64) ,
`timestamp` TIMESTAMP ,
`lib_type` char(2) NOT NULL,
PRIMARY KEY (`obj_id`),
INDEX `index_hash_id` (`hash_id`),
INDEX `thekey` (`object_path`(128), `hash_id`, `syshost`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin AUTO_INCREMENT=1
""")
print("(%d) create xalt_object table" % idx)
idx += 1
# 3
cursor.execute("""
CREATE TABLE IF NOT EXISTS `join_link_object` (
`join_id` int(11) unsigned NOT NULL auto_increment,
`obj_id` int(11) unsigned NOT NULL,
`link_id` int(11) unsigned NOT NULL,
`date` DATE NOT NULL,
PRIMARY KEY (`join_id`),
FOREIGN KEY (`link_id`) REFERENCES `xalt_link`(`link_id`),
FOREIGN KEY (`obj_id`) REFERENCES `xalt_object`(`obj_id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin AUTO_INCREMENT=1
""")
print("(%d) create join_link_object table" % idx)
idx += 1
# 4
cursor.execute("""
CREATE TABLE IF NOT EXISTS `xalt_run` (
`run_id` int(11) unsigned NOT NULL auto_increment,
`job_id` char(64) NOT NULL,
`run_uuid` char(36) NOT NULL,
`date` datetime NOT NULL,
`syshost` varchar(64) NOT NULL,
`uuid` char(36) ,
`hash_id` char(40) NOT NULL,
`account` varchar(20) NOT NULL,
`exec_type` char(7) NOT NULL,
`start_time` double NOT NULL,
`end_time` double NOT NULL,
`run_time` double NOT NULL,
`probability` double NOT NULL,
`num_cores` int(11) unsigned NOT NULL,
`num_nodes` int(11) unsigned NOT NULL,
`num_threads` smallint(6) unsigned NOT NULL,
`num_gpus` int(11) unsigned NOT NULL,
`queue` varchar(32) NOT NULL,
`sum_runs` int(11) unsigned NOT NULL,
`sum_time` double NOT NULL,
`user` varchar(32) NOT NULL,
`exec_path` varchar(1024) NOT NULL,
`module_name` varchar(64) ,
`cwd` varchar(1024) NOT NULL,
`cmdline` blob NOT NULL,
`container` varchar(32) ,
PRIMARY KEY (`run_id` ),
INDEX `index_date` (`date` ),
INDEX `index_run_uuid` (`run_uuid` ),
INDEX `thekey` (`job_id`, `syshost` )
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin AUTO_INCREMENT=1
""")
print("(%d) create xalt_run table" % idx)
idx += 1
# 5
cursor.execute("""
CREATE TABLE IF NOT EXISTS `join_run_object` (
`join_id` int(11) unsigned NOT NULL auto_increment,
`obj_id` int(11) unsigned NOT NULL,
`run_id` int(11) unsigned NOT NULL,
`date` DATE NOT NULL,
PRIMARY KEY (`join_id`),
FOREIGN KEY (`run_id`) REFERENCES `xalt_run`(`run_id`),
FOREIGN KEY (`obj_id`) REFERENCES `xalt_object`(`obj_id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin AUTO_INCREMENT=1
""")
print("(%d) create join_run_object table" % idx)
idx += 1
# 6
cursor.execute("""
CREATE TABLE IF NOT EXISTS `xalt_env_name` (
`env_id` int(20) unsigned NOT NULL auto_increment,
`env_name` varchar(64) NOT NULL,
PRIMARY KEY (`env_id`),
UNIQUE KEY `env_name` (`env_name`),
INDEX `a_env_name` (`env_name`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin AUTO_INCREMENT=1
""")
print("(%d) create xalt_env_name table" % idx)
idx += 1
# 7
cursor.execute("""
CREATE TABLE IF NOT EXISTS `join_run_env` (
`join_id` bigint(20) unsigned NOT NULL auto_increment,
`env_id` int(11) unsigned NOT NULL,
`run_id` int(11) unsigned NOT NULL,
`date` DATE NOT NULL,
`env_value` blob NOT NULL,
PRIMARY KEY (`join_id`),
FOREIGN KEY (`env_id`) REFERENCES `xalt_env_name`(`env_id`),
FOREIGN KEY (`run_id`) REFERENCES `xalt_run`(`run_id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin AUTO_INCREMENT=1
""")
print("(%d) create join_run_env table" % idx)
idx += 1
# 8
cursor.execute("""
CREATE TABLE IF NOT EXISTS `xalt_user` (
`usr_id` int(11) unsigned NOT NULL auto_increment,
`user` varchar(32) NOT NULL,
`anon_user` varchar(12) NOT NULL,
PRIMARY KEY (`usr_id`),
INDEX `the_user` (`user`),
INDEX `a_user` (`anon_user`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin AUTO_INCREMENT=1
""")
print("(%d) create xalt_user table" % idx)
idx += 1
# 9
cursor.execute("""
CREATE TABLE IF NOT EXISTS `xalt_account` (
`acct_id` int(11) unsigned NOT NULL auto_increment,
`account` varchar(32) NOT NULL,
`anon_user` varchar(10) NOT NULL,
`field_of_science` varchar(64) NOT NULL,
PRIMARY KEY (`acct_id`),
INDEX `the_account` (`account`),
INDEX `a_acct` (`anon_user`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin AUTO_INCREMENT=1
""")
print("(%d) create xalt_account table" % idx)
idx += 1
# 10
cursor.execute("""
CREATE TABLE IF NOT EXISTS `xalt_function` (
`func_id` int(11) unsigned NOT NULL auto_increment,
`function_name` varchar(255) NOT NULL,
PRIMARY KEY (`func_id`),
UNIQUE KEY `function_name` (`function_name`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin AUTO_INCREMENT=1
""")
print("(%d) create xalt_function table" % idx)
idx += 1
# 11
cursor.execute("""
CREATE TABLE IF NOT EXISTS `join_link_function` (
`join_id` int(11) unsigned NOT NULL auto_increment,
`func_id` int(11) unsigned NOT NULL,
`link_id` int(11) unsigned NOT NULL,
`date` DATE NOT NULL,
PRIMARY KEY (`join_id`),
FOREIGN KEY (`func_id`) REFERENCES `xalt_function`(`func_id`),
FOREIGN KEY (`link_id`) REFERENCES `xalt_link`(`link_id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin AUTO_INCREMENT=1
""")
print("(%d) create join_link_function table" % idx)
idx += 1
# 12
cursor.execute("""
CREATE TABLE IF NOT EXISTS `xalt_total_env` (
`envT_id` bigint(20) unsigned NOT NULL auto_increment,
`run_id` int(11) unsigned NOT NULL,
`date` DATE NOT NULL,
`env_blob` blob NOT NULL,
PRIMARY KEY (`envT_id`),
FOREIGN KEY (`run_id`) REFERENCES `xalt_run`(`run_id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin AUTO_INCREMENT=1
""")
print("(%d) create xalt_env table" % idx)
idx += 1
# 13
cursor.execute("""
ALTER TABLE `join_link_function`
ADD UNIQUE `unique_func_link_id` ( `func_id`, `link_id` )
""")
print("(%d) create unique key on join_link_function table" % idx)
idx += 1
# 14
cursor.execute("""
CREATE TABLE IF NOT EXISTS `xalt_pkg` (
`pkg_id` bigint(20) unsigned NOT NULL auto_increment,
`run_id` int(11) unsigned NOT NULL,
`program` varchar(12) NOT NULL,
`pkg_name` varchar(64) NOT NULL,
`pkg_version` varchar(32) ,
`pkg_path` varchar(1024) ,
PRIMARY KEY (`pkg_id`),
FOREIGN KEY (`run_id`) REFERENCES `xalt_run`(`run_id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin AUTO_INCREMENT=1
""")
print("(%d) create xalt_pkg table" % idx)
idx += 1
# 15
cursor.execute("""
CREATE TABLE IF NOT EXISTS `xalt_field_of_science` (
`fos_id` int(11) unsigned NOT NULL auto_increment,
`account` varchar(20) NOT NULL,
`field_of_science` varchar(80) NOT NULL,
`directorate` varchar(80) NOT NULL,
PRIMARY KEY (`fos_id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin AUTO_INCREMENT=1
""")
print("(%d) create xalt_field_of_science table" % idx)
idx += 1
cursor.close()
except MySQLdb.Error as e:
print("Error %d: %s" % (e.args[0], e.args[1]))
sys.exit(1)
def main():
ConfigBaseNm = "xalt_db"
ConfigFn = ConfigBaseNm + ".conf"
if (not os.path.isfile(ConfigFn)):
dirNm, exe = os.path.split(sys.argv[0])
fn = os.path.join(dirNm, ConfigFn)
if (os.path.isfile(fn)):
ConfigFn = fn
else:
ConfigFn = os.path.abspath(os.path.join(dirNm, "../site",
ConfigFn))
xalt = XALTdb(ConfigFn)
db = xalt.db()
try:
conn = xalt.connect()
cursor = conn.cursor()
# If MySQL version < 4.1, comment out the line below
cursor.execute("SET SQL_MODE=\"NO_AUTO_VALUE_ON_ZERO\"")
# If the database does not exist, create it, otherwise, switch to the database.
cursor.execute(
"CREATE DATABASE IF NOT EXISTS %s DEFAULT CHARACTER SET utf8 COLLATE utf8_general_ci"
% xalt.db())
cursor.execute("USE " + xalt.db())
idx = 1
print("start")
# 1
cursor.execute("""
CREATE TABLE `xalt_link` (
`link_id` int(11) NOT NULL auto_increment,
`uuid` char(36) NOT NULL,
`hash_id` char(40) NOT NULL,
`date` DATETIME NOT NULL,
`link_program` varchar(10) NOT NULL,
`build_user` varchar(64) NOT NULL,
`build_syshost` varchar(64) NOT NULL,
`build_epoch` double NOT NULL,
`exit_code` tinyint(4) NOT NULL,
`exec_path` varchar(1024) NOT NULL,
PRIMARY KEY (`link_id`),
UNIQUE KEY `uuid` (`uuid`)
) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_bin AUTO_INCREMENT=1
""")
print("(%d) create xalt_link table" % idx)
idx += 1
# 2
cursor.execute("""
CREATE TABLE `xalt_object` (
`obj_id` int(11) NOT NULL auto_increment,
`object_path` varchar(1024) NOT NULL,
`syshost` varchar(64) NOT NULL,
`hash_id` char(40) NOT NULL,
`module_name` varchar(64) ,
`timestamp` TIMESTAMP ,
`lib_type` char(2) NOT NULL,
PRIMARY KEY (`obj_id`),
INDEX `index_hash_id` (`hash_id`),
UNIQUE KEY `thekey` (`object_path`(512), `hash_id`, `syshost`)
) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_bin AUTO_INCREMENT=1
""")
print("(%d) create xalt_object table" % idx)
idx += 1
# 3
cursor.execute("""
CREATE TABLE `join_link_object` (
`join_id` int(11) NOT NULL auto_increment,
`obj_id` int(11) NOT NULL,
`link_id` int(11) NOT NULL,
PRIMARY KEY (`join_id`),
FOREIGN KEY (`link_id`) REFERENCES `xalt_link`(`link_id`),
FOREIGN KEY (`obj_id`) REFERENCES `xalt_object`(`obj_id`)
) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_bin AUTO_INCREMENT=1
""")
print("(%d) create join_link_object table" % idx)
idx += 1
# 4
cursor.execute("""
CREATE TABLE `xalt_run` (
`run_id` int(11) NOT NULL auto_increment,
`job_id` char(11) NOT NULL,
`run_uuid` char(36) NOT NULL,
`date` datetime NOT NULL,
`syshost` varchar(64) NOT NULL,
`uuid` char(36) ,
`hash_id` char(40) NOT NULL,
`account` char(11) NOT NULL,
`exec_type` char(7) NOT NULL,
`start_time` double NOT NULL,
`end_time` double NOT NULL,
`run_time` double NOT NULL,
`num_cores` int(11) NOT NULL,
`num_nodes` int(11) NOT NULL,
`num_threads` tinyint(4) NOT NULL,
`queue` varchar(32) NOT NULL,
`user` varchar(32) NOT NULL,
`exec_path` varchar(1024) NOT NULL,
`module_name` varchar(64) ,
`cwd` varchar(1024) NOT NULL,
PRIMARY KEY (`run_id`),
INDEX `index_run_uuid` (`run_uuid`),
INDEX `thekey` (`job_id`, `syshost`)
) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_bin AUTO_INCREMENT=1
""")
print("(%d) create xalt_run table" % idx)
idx += 1
# 5
cursor.execute("""
CREATE TABLE `join_run_object` (
`join_id` int(11) NOT NULL auto_increment,
`obj_id` int(11) NOT NULL,
`run_id` int(11) NOT NULL,
PRIMARY KEY (`join_id`),
FOREIGN KEY (`run_id`) REFERENCES `xalt_run`(`run_id`),
FOREIGN KEY (`obj_id`) REFERENCES `xalt_object`(`obj_id`)
) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_bin AUTO_INCREMENT=1
""")
print("(%d) create join_run_object table" % idx)
idx += 1
# 6
cursor.execute("""
CREATE TABLE `xalt_env_name` (
`env_id` int(11) NOT NULL auto_increment,
`env_name` varchar(64) NOT NULL,
PRIMARY KEY (`env_id`),
UNIQUE KEY `env_name` (`env_name`)
) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_bin AUTO_INCREMENT=1
""")
print("(%d) create xalt_env_name table" % idx)
idx += 1
# 7
cursor.execute("""
CREATE TABLE `join_run_env` (
`join_id` int(11) NOT NULL auto_increment,
`env_id` int(11) NOT NULL,
`run_id` int(11) NOT NULL,
`env_value` blob NOT NULL,
PRIMARY KEY (`join_id`),
FOREIGN KEY (`env_id`) REFERENCES `xalt_env_name`(`env_id`),
FOREIGN KEY (`run_id`) REFERENCES `xalt_run`(`run_id`)
) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_bin AUTO_INCREMENT=1
""")
print("(%d) create join_run_env table" % idx)
idx += 1
# 8
cursor.execute("""
CREATE TABLE `xalt_job_id` (
`inc_id` int(11) NOT NULL auto_increment,
`job_id` int(11) NOT NULL,
PRIMARY KEY (`inc_id`)
) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_bin AUTO_INCREMENT=1
""")
query = "INSERT INTO xalt_job_id VALUES(NULL, '0')"
conn.query(query)
cursor.close()
except MySQLdb.Error, e:
print("Error %d: %s" % (e.args[0], e.args[1]))
sys.exit(1)
def main():
"""
This program creates the Database used by XALT.
"""
args = CmdLineOptions().execute()
configFn = dbConfigFn(args.dbname)
if (not os.path.isfile(configFn)):
dirNm, exe = os.path.split(sys.argv[0])
fn = os.path.join(dirNm, configFn)
if (os.path.isfile(fn)):
configFn = fn
else:
configFn = os.path.abspath(os.path.join(dirNm, "../site", configFn))
xalt = XALTdb(configFn)
db = xalt.db()
print (db)
try:
conn = xalt.connect()
cursor = conn.cursor()
# ftrack2015 start >>
# Modified below script to -
# 1) Make a new db if no 'xalt' db is present
# 2) If xalt db already exists then check for what all tables present
# 3) Create new tables (xalt_function & join_function_link)
#
tables = []
# If MySQL version < 4.1, comment out the line below
cursor.execute("SET SQL_MODE=\"NO_AUTO_VALUE_ON_ZERO\"")
# Check if database exists, if yes get all table names.
query = "SELECT SCHEMA_NAME FROM INFORMATION_SCHEMA.SCHEMATA WHERE SCHEMA_NAME = 'test'"
conn.query(query)
result = conn.store_result()
database = result.fetch_row()
if len(database) != 0 :
if (database[0][0] == 'test'):
cursor.execute("USE "+xalt.db()) # Use Database
# Get list of tables that exists already
cursor.execute("SHOW TABLES")
for (table_names,) in cursor:
tables.append(table_names)
print (tables)
else:
# If the database does not exist, create it, otherwise, switch to the database.
cursor.execute("CREATE DATABASE IF NOT EXISTS %s DEFAULT CHARACTER SET utf8 COLLATE utf8_general_ci" % xalt.db())
cursor.execute("USE "+xalt.db())
idx = 1
print("start")
if ('xalt_link' not in tables):
# 1
cursor.execute("""
CREATE TABLE `xalt_link` (
`link_id` int(11) NOT NULL auto_increment,
`uuid` char(36) NOT NULL,
`hash_id` char(40) NOT NULL,
`date` DATETIME NOT NULL,
`link_program` varchar(10) NOT NULL,
`build_user` varchar(64) NOT NULL,
`build_syshost` varchar(64) NOT NULL,
`build_epoch` double NOT NULL,
`exit_code` tinyint(4) NOT NULL,
`exec_path` varchar(1024) NOT NULL,
PRIMARY KEY (`link_id`),
UNIQUE KEY `uuid` (`uuid`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_general_ci AUTO_INCREMENT=1
""")
print("(%d) create xalt_link table" % idx); idx += 1
if ('xalt_object' not in tables):
# 2
cursor.execute("""
CREATE TABLE `xalt_object` (
`obj_id` int(11) NOT NULL auto_increment,
`object_path` varchar(1024) NOT NULL,
`syshost` varchar(64) NOT NULL,
`hash_id` char(40) NOT NULL,
`module_name` varchar(64) ,
`timestamp` TIMESTAMP ,
`lib_type` char(2) NOT NULL,
PRIMARY KEY (`obj_id`),
INDEX `index_hash_id` (`hash_id`),
INDEX `thekey` (`object_path`(128), `hash_id`, `syshost`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_general_ci AUTO_INCREMENT=1
""")
print("(%d) create xalt_object table" % idx ); idx += 1;
if ('join_link_object' not in tables):
# 3
cursor.execute("""
CREATE TABLE `join_link_object` (
`join_id` int(11) NOT NULL auto_increment,
`obj_id` int(11) NOT NULL,
`link_id` int(11) NOT NULL,
PRIMARY KEY (`join_id`),
FOREIGN KEY (`link_id`) REFERENCES `xalt_link`(`link_id`),
FOREIGN KEY (`obj_id`) REFERENCES `xalt_object`(`obj_id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_general_ci AUTO_INCREMENT=1
""")
print("(%d) create join_link_object table" % idx); idx += 1
if ('xalt_run' not in tables):
# 4
cursor.execute("""
CREATE TABLE `xalt_run` (
`run_id` int(11) NOT NULL auto_increment,
`job_id` char(11) NOT NULL,
`run_uuid` char(36) NOT NULL,
`date` datetime NOT NULL,
`syshost` varchar(64) NOT NULL,
`uuid` char(36) ,
`hash_id` char(40) NOT NULL,
`account` char(11) NOT NULL,
`exec_type` char(7) NOT NULL,
`start_time` double NOT NULL,
`end_time` double NOT NULL,
`run_time` double NOT NULL,
`num_cores` int(11) NOT NULL,
`job_num_cores` int(11) NOT NULL,
`num_nodes` int(11) NOT NULL,
`num_threads` tinyint(4) NOT NULL,
`queue` varchar(32) NOT NULL,
`exit_code` tinyint(4) NOT NULL,
`user` varchar(32) NOT NULL,
`exec_path` varchar(1024) NOT NULL,
`module_name` varchar(64) ,
`cwd` varchar(1024) NOT NULL,
PRIMARY KEY (`run_id`),
INDEX `index_run_uuid` (`run_uuid`),
INDEX `thekey` (`job_id`, `syshost`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_general_ci AUTO_INCREMENT=1
""")
print("(%d) create xalt_run table" % idx)
idx += 1
if ('join_run_object' not in tables):
# 5
cursor.execute("""
CREATE TABLE `join_run_object` (
`join_id` int(11) NOT NULL auto_increment,
`obj_id` int(11) NOT NULL,
`run_id` int(11) NOT NULL,
PRIMARY KEY (`join_id`),
FOREIGN KEY (`run_id`) REFERENCES `xalt_run`(`run_id`),
FOREIGN KEY (`obj_id`) REFERENCES `xalt_object`(`obj_id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_general_ci AUTO_INCREMENT=1
""")
print("(%d) create join_run_object table" % idx); idx += 1
if ('xalt_env_name' not in tables):
# 6
cursor.execute("""
CREATE TABLE `xalt_env_name` (
`env_id` int(11) NOT NULL auto_increment,
`env_name` varchar(64) NOT NULL,
PRIMARY KEY (`env_id`),
UNIQUE KEY `env_name` (`env_name`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_general_ci AUTO_INCREMENT=1
""")
print("(%d) create xalt_env_name table" % idx); idx += 1
if ('join_run_env' not in tables):
# 7
cursor.execute("""
CREATE TABLE `join_run_env` (
`join_id` int(11) NOT NULL auto_increment,
`env_id` int(11) NOT NULL,
`run_id` int(11) NOT NULL,
`env_value` blob NOT NULL,
PRIMARY KEY (`join_id`),
FOREIGN KEY (`env_id`) REFERENCES `xalt_env_name`(`env_id`),
FOREIGN KEY (`run_id`) REFERENCES `xalt_run`(`run_id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_general_ci AUTO_INCREMENT=1
""")
print("(%d) create join_run_env table" % idx); idx += 1
if ('xalt_user' not in tables):
# 8
cursor.execute("""
CREATE TABLE `xalt_user` (
`usr_id` int(11) NOT NULL auto_increment,
`user` varchar(32) NOT NULL,
`anon_user` varchar(12) NOT NULL,
PRIMARY KEY (`usr_id`),
INDEX `the_user` (`user`),
INDEX `a_user` (`anon_user`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_general_ci AUTO_INCREMENT=1
""")
print("(%d) create xalt_user table" % idx); idx += 1
if ('xalt_account' not in tables):
# 9
cursor.execute("""
CREATE TABLE `xalt_account` (
`acct_id` int(11) NOT NULL auto_increment,
`account` varchar(32) NOT NULL,
`anon_acct` varchar(10) NOT NULL,
`field_of_science` varchar(64) NOT NULL,
PRIMARY KEY (`acct_id`),
INDEX `the_account` (`account`),
INDEX `a_acct` (`anon_acct`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_general_ci AUTO_INCREMENT=1
""")
print("(%d) create xalt_account table" % idx); idx += 1
if ('xalt_function' not in tables):
# 10
cursor.execute("""
CREATE TABLE `xalt_function` (
`func_id` int(11) NOT NULL auto_increment,
`function_name` varchar(1024) NOT NULL,
PRIMARY KEY (`func_id`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_bin AUTO_INCREMENT=1
""")
print("(%d) create xalt_function table" % idx); idx += 1
if ('join_function_link' not in tables):
# 11
cursor.execute("""
CREATE TABLE `join_function_link` (
`join_id` int(11) NOT NULL auto_increment,
`func_id` int(11) NOT NULL,
`link_id` int(11) NOT NULL,
PRIMARY KEY (`join_id`),
FOREIGN KEY (`link_id`) REFERENCES `xalt_link`(`link_id`),
FOREIGN KEY (`func_id`) REFERENCES `xalt_function`(`func_id`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_bin AUTO_INCREMENT=1
""")
print("(%d) create join_function_link table" % idx); idx += 1
# ftrack2015 ends <<
cursor.close()
except MySQLdb.Error, e:
print ("Error %d: %s" % (e.args[0], e.args[1]))
sys.exit (1)
def main():
"""
read from syslog file into XALT db.
"""
sA = []
sA.append("CommandLine:")
for v in sys.argv:
sA.append('"' + v + '"')
XALT_Stack.push(" ".join(sA))
args = CmdLineOptions().execute()
xalt = XALTdb(args.confFn)
syslogFile = args.syslog
icnt = 0
t1 = time.time()
try:
rmapT = Rmap(args.rmapD).reverseMapT()
except Exception as e:
print(e, file=sys.stderr)
print("Failed to read reverseMap file -> exiting")
sys.exit(1)
lnkCnt = 0
pkgCnt = 0
runCnt = 0
badCnt = 0
count = 0
recordT = {}
fnA = [args.leftover, syslogFile]
parseSyslog = ParseSyslog(args.leftover)
#-----------------------------
# Figure out size in bytes.
fnSz = 0
for fn in fnA:
if (not os.path.isfile(fn)):
continue
fnSz += os.path.getsize(fn)
#----------------------------------------------------------
# Count the number and sum the run_time for all scalar jobs
filter = Filter(100)
pbar = ProgressBar(maxVal=fnSz, fd=sys.stdout)
for fn in fnA:
if (not os.path.isfile(fn)):
continue
old = (fn == args.leftover)
lineNo = 0
f = open(fn, 'r')
for line in f:
lineNo += 1
count += len(line)
pbar.update(count)
if (not ("XALT_LOGGING" in line)):
continue
try:
t, done = parseSyslog.parse(line, args.syshost, old)
except Exception as e:
#print(e, file=sys.stderr)
#print("lineNo:",lineNo,"file:",fn,"line:",line, file=sys.stderr)
#print("Now continuing processing!", file=sys.stderr)
continue
if (not done or t['kind'] != "run"):
continue
##################################
# If the json conversion fails,
# then ignore record and keep going
value = False
try:
value = json.loads(t['value'])
filter.register(value)
except Exception as e:
#print("fn:",fn,"line:",lineNo,"value:",t['value'],file=sys.stderr)
continue
f.close()
pbar.fini()
filter.report_stats()
badsyslog = 0
count = 0
parseSyslog = ParseSyslog(args.leftover)
pbar = ProgressBar(maxVal=max(fnSz, 1), fd=sys.stdout)
for fn in fnA:
if (not os.path.isfile(fn)):
continue
old = (fn == args.leftover)
f = open(fn, 'r')
for line in f:
count += len(line)
pbar.update(count)
if (not ("XALT_LOGGING" in line)):
continue
try:
t, done = parseSyslog.parse(line, args.syshost, old)
except Exception as e:
badsyslog += 1
continue
if (not done):
continue
##################################
# If the json conversion fails,
# then ignore record and keep going
try:
value = json.loads(t['value'])
except Exception as e:
continue
try:
XALT_Stack.push("XALT_LOGGING: " + t['kind'] + " " +
t['syshost'])
if (t['kind'] == "link"):
XALT_Stack.push("link_to_db()")
xalt.link_to_db(rmapT, value)
XALT_Stack.pop()
lnkCnt += 1
elif (t['kind'] == "run"):
if (filter.apply(value)):
XALT_Stack.push("run_to_db()")
xalt.run_to_db(rmapT, value)
XALT_Stack.pop()
runCnt += 1
elif (t['kind'] == "pkg"):
XALT_Stack.push("pkg_to_db()")
xalt.pkg_to_db(t['syshost'], value)
XALT_Stack.pop()
pkgCnt += 1
else:
print("Error in xalt_syslog_to_db", file=sys.stderr)
XALT_Stack.pop()
except Exception as e:
print(e, file=sys.stderr)
badCnt += 1
f.close()
pbar.fini()
t2 = time.time()
rt = t2 - t1
if (args.timer):
print("Time: ", time.strftime("%T", time.gmtime(rt)))
print("total processed : ", count, ", num links: ", lnkCnt, ", num runs: ",
runCnt, ", pkgCnt: ", pkgCnt, ", badCnt: ", badCnt, ", badsyslog: ",
badsyslog)
# if there is anything left in recordT file write it out to the leftover file.
parseSyslog.writeRecordT()
def main():
"""
Walks the list of users via the passwd_generator and load the
link and run files.
"""
# Find transmission style
transmission = os.environ.get("XALT_TRANSMISSION_STYLE")
if (not transmission):
transmission = "@transmission@"
if (not transmission):
transmission = "file"
transmission = transmission.lower()
# Push command line on to XALT_Stack
sA = []
sA.append("CommandLine:")
for v in sys.argv:
sA.append('"' + v + '"')
XALT_Stack.push(" ".join(sA))
args = CmdLineOptions().execute()
xalt = XALTdb(args.confFn)
num = int(capture("getent passwd | wc -l"))
pbar = ProgressBar(maxVal=num)
icnt = 0
t1 = time.time()
rmapT = Rmap(args.rmapD).reverseMapT()
u2acctT = {}
if (args.u2acct):
fp = open(args.u2acct, "r")
u2acctT = json.loads(fp.read())
fp.close()
countT = {}
countT['lnk'] = 0
countT['run'] = 0
countT['pkg'] = 0
xalt_file_prefix = os.environ.get("XALT_FILE_PREFIX", "@xalt_file_prefix@")
if (xalt_file_prefix == "USE_HOME"):
for user, homeDir in passwd_generator():
store_json_files(homeDir, transmission, xalt, rmapT, u2acctT, args,
countT)
else:
store_json_files("", transmission, xalt, rmapT, u2acctT, args, countT)
xalt.connect().close()
#pbar.fini()
t2 = time.time()
rt = t2 - t1
if (args.timer):
print("Time: ", time.strftime("%T", time.gmtime(rt)))
print("num links: ", countT['lnk'], ", num pkgs: ", countT['pkg'],
", num runs: ", countT['run'])
def main():
"""
This program upgrade the Database used by XALT to the current version
from the version specified in script name.
"""
args = CmdLineOptions().execute()
configFn = dbConfigFn(args.dbname)
if (not os.path.isfile(configFn)):
dirNm, exe = os.path.split(sys.argv[0])
fn = os.path.join(dirNm, configFn)
if (os.path.isfile(fn)):
configFn = fn
else:
configFn = os.path.abspath(os.path.join(dirNm, "../site", configFn))
xalt = XALTdb(configFn)
db = xalt.db()
try:
conn = xalt.connect()
cursor = conn.cursor()
# If MySQL version < 4.1, comment out the line below
cursor.execute("SET SQL_MODE=\"NO_AUTO_VALUE_ON_ZERO\"")
cursor.execute("USE "+xalt.db())
idx = 1
print("start")
###########################################################################
# NOTE: New DB schema createDB.py uses unsigned int for some columns, but #
# modifying a signed to unsigned is problematic for columns already #
# referenced as FOREIGN KEYS. Therefor this script does not update #
# those columns. #
###########################################################################
# 1
cursor.execute("""
ALTER TABLE `xalt_link`
MODIFY COLUMN `link_program` varchar(64) NOT NULL,
ADD COLUMN `link_path` varchar(1024) NOT NULL AFTER `link_program`,
ADD COLUMN `link_module_name` varchar(64) AFTER `link_path`,
ADD COLUMN `link_line` blob AFTER `link_module_name`,
ADD INDEX `index_date` (`date`)
""")
print("(%d) upgraded xalt_link table" % idx); idx += 1
# 4
cursor.execute("""
ALTER TABLE `xalt_run`
MODIFY COLUMN `job_id` char(64) NOT NULL,
MODIFY COLUMN `num_cores` int(11) unsigned NOT NULL,
MODIFY COLUMN `job_num_cores` int(11) unsigned NOT NULL,
MODIFY COLUMN `num_nodes` int(11) unsigned NOT NULL,
MODIFY COLUMN `num_threads` smallint(6) unsigned NOT NULL,
MODIFY COLUMN `exit_code` smallint(6) NOT NULL,
ADD COLUMN `cmdline` blob NOT NULL AFTER `cwd`,
ADD INDEX `index_date` (`date` )
""")
print("(%d) upgraded xalt_run table" % idx); idx += 1
# 6
cursor.execute("""
ALTER TABLE `xalt_env_name`
ADD INDEX `a_env_name` (`env_name`)
""")
print("(%d) upgraded xalt_env_name table" % idx)
idx += 1
# 7
cursor.execute("""
ALTER TABLE `join_run_env`
MODIFY COLUMN `join_id` bigint(20) unsigned NOT NULL auto_increment
""")
print("(%d) upgraded join_run_env table" % idx)
idx += 1
#11
cursor.execute("""
ALTER TABLE `join_link_function`
MODIFY COLUMN `join_id` int(11) unsigned NOT NULL auto_increment
""")
print("(%d) upgraded xalt_function table" % idx)
idx += 1
# 12
cursor.execute("""
CREATE TABLE IF NOT EXISTS `xalt_total_env` (
`envT_id` bigint(20) unsigned NOT NULL auto_increment,
`run_id` int(11) NOT NULL,
`env_blob` blob NOT NULL,
PRIMARY KEY (`envT_id`),
FOREIGN KEY (`run_id`) REFERENCES `xalt_run`(`run_id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_general_ci AUTO_INCREMENT=1
""")
print("(%d) created xalt_env table" % idx); idx += 1
cursor.close()
except MySQLdb.Error as e:
print ("Error %d: %s" % (e.args[0], e.args[1]))
sys.exit (1)
def main():
"""
Walks the list of users via the passwd_generator and load the
link and run files.
"""
# Find transmission style
transmission = os.environ.get("XALT_TRANSMISSION_STYLE")
if (not transmission):
transmission = "@XALT_TRANSMISSION_STYLE@"
if (not transmission):
transmission = "file"
transmission = transmission.lower()
# Push command line on to XALT_Stack
sA = []
sA.append("CommandLine:")
for v in sys.argv:
sA.append('"'+v+'"')
XALT_Stack.push(" ".join(sA))
args = CmdLineOptions().execute()
xalt = XALTdb(args.confFn)
num = int(capture("getent passwd | wc -l"))
pbar = ProgressBar(maxVal=num)
icnt = 0
t1 = time.time()
rmapT = Rmap(args.rmapD).reverseMapT()
iuser = 0
lnkCnt = 0
runCnt = 0
pkgCnt = 0
for user, hdir in passwd_generator():
XALT_Stack.push("User: "******"link")
if (os.path.isdir(xaltDir)):
iuser += 1
linkFnA = files_in_tree(xaltDir, "*/link." + args.syshost + ".*.json")
XALT_Stack.push("link_json_to_db()")
lnkCnt += link_json_to_db(xalt, args.listFn, rmapT, args.delete, linkFnA)
XALT_Stack.pop()
xaltDir = build_xaltDir(user, hdir, transmission, "run")
if (os.path.isdir(xaltDir)):
runFnA = files_in_tree(xaltDir, "*/run." + args.syshost + ".*.json")
XALT_Stack.push("run_json_to_db()")
runCnt += run_json_to_db(xalt, args.listFn, rmapT, args.delete, runFnA)
XALT_Stack.pop()
xaltDir = build_xaltDir(user, hdir, transmission, "pkg")
if (os.path.isdir(xaltDir)):
pkgFnA = files_in_tree(xaltDir, "*/pkg." + args.syshost + ".*.json")
XALT_Stack.push("pkg_json_to_db()")
pkgCnt += pkg_json_to_db(xalt, args.listFn, args.syshost, args.delete, pkgFnA)
XALT_Stack.pop()
icnt += 1
v = XALT_Stack.pop()
carp("User",v)
pbar.update(icnt)
xalt.connect().close()
pbar.fini()
t2 = time.time()
rt = t2 - t1
if (args.timer):
print("Time: ", time.strftime("%T", time.gmtime(rt)))
print("num users: ", iuser, ", num links: ", lnkCnt, ", num pkgs: ", pkgCnt, ", num runs: ", runCnt)
def main():
"""
Walks the list of users via the passwd_generator and load the
link and run files.
"""
# Find transmission style
transmission = os.environ.get("XALT_TRANSMISSION_STYLE")
if (not transmission):
transmission = "@XALT_TRANSMISSION_STYLE@"
if (not transmission):
transmission = "file"
transmission = transmission.lower()
# Push command line on to XALT_Stack
sA = []
sA.append("CommandLine:")
for v in sys.argv:
sA.append('"' + v + '"')
XALT_Stack.push(" ".join(sA))
args = CmdLineOptions().execute()
xalt = XALTdb(args.confFn)
num = int(capture("getent passwd | wc -l"))
pbar = ProgressBar(maxVal=num)
icnt = 0
t1 = time.time()
rmapT = Rmap(args.rmapD).reverseMapT()
iuser = 0
lnkCnt = 0
runCnt = 0
pkgCnt = 0
for user, hdir in passwd_generator():
XALT_Stack.push("User: "******"link")
if (os.path.isdir(xaltDir)):
iuser += 1
linkFnA = files_in_tree(xaltDir,
"*/link." + args.syshost + ".*.json")
XALT_Stack.push("link_json_to_db()")
lnkCnt += link_json_to_db(xalt, args.listFn, rmapT, args.delete,
linkFnA)
XALT_Stack.pop()
xaltDir = build_xaltDir(user, hdir, transmission, "run")
if (os.path.isdir(xaltDir)):
runFnA = files_in_tree(xaltDir,
"*/run." + args.syshost + ".*.json")
XALT_Stack.push("run_json_to_db()")
runCnt += run_json_to_db(xalt, args.listFn, rmapT, args.delete,
runFnA)
XALT_Stack.pop()
xaltDir = build_xaltDir(user, hdir, transmission, "pkg")
if (os.path.isdir(xaltDir)):
pkgFnA = files_in_tree(xaltDir,
"*/pkg." + args.syshost + ".*.json")
XALT_Stack.push("pkg_json_to_db()")
pkgCnt += pkg_json_to_db(xalt, args.listFn, args.syshost,
args.delete, pkgFnA)
XALT_Stack.pop()
icnt += 1
v = XALT_Stack.pop()
carp("User", v)
pbar.update(icnt)
xalt.connect().close()
pbar.fini()
t2 = time.time()
rt = t2 - t1
if (args.timer):
print("Time: ", time.strftime("%T", time.gmtime(rt)))
print("num users: ", iuser, ", num links: ", lnkCnt, ", num pkgs: ",
pkgCnt, ", num runs: ", runCnt)
def main():
ConfigBaseNm = "xalt_db"
ConfigFn = ConfigBaseNm + ".conf"
if (not os.path.isfile(ConfigFn)):
dirNm, exe = os.path.split(sys.argv[0])
fn = os.path.join(dirNm, ConfigFn)
if (os.path.isfile(fn)):
ConfigFn = fn
else:
ConfigFn = os.path.abspath(os.path.join(dirNm, "../site", ConfigFn))
xalt = XALTdb(ConfigFn)
db = xalt.db()
try:
conn = xalt.connect()
cursor = conn.cursor()
# If MySQL version < 4.1, comment out the line below
cursor.execute("SET SQL_MODE=\"NO_AUTO_VALUE_ON_ZERO\"")
# If the database does not exist, create it, otherwise, switch to the database.
cursor.execute("CREATE DATABASE IF NOT EXISTS %s DEFAULT CHARACTER SET utf8 COLLATE utf8_general_ci" % xalt.db())
cursor.execute("USE "+xalt.db())
idx = 1
print("start")
# 1
cursor.execute("""
CREATE TABLE `xalt_link` (
`link_id` int(11) NOT NULL auto_increment,
`uuid` char(36) NOT NULL,
`hash_id` char(40) NOT NULL,
`date` DATETIME NOT NULL,
`link_program` varchar(10) NOT NULL,
`build_user` varchar(64) NOT NULL,
`build_syshost` varchar(64) NOT NULL,
`build_epoch` double NOT NULL,
`exit_code` tinyint(4) NOT NULL,
`exec_path` varchar(1024) NOT NULL,
PRIMARY KEY (`link_id`),
UNIQUE KEY `uuid` (`uuid`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_bin AUTO_INCREMENT=1
""")
print("(%d) create xalt_link table" % idx); idx += 1
# 2
cursor.execute("""
CREATE TABLE `xalt_object` (
`obj_id` int(11) NOT NULL auto_increment,
`object_path` varchar(1024) NOT NULL,
`syshost` varchar(64) NOT NULL,
`hash_id` char(40) NOT NULL,
`module_name` varchar(64) ,
`timestamp` TIMESTAMP ,
`lib_type` char(2) NOT NULL,
PRIMARY KEY (`obj_id`),
INDEX `index_hash_id` (`hash_id`),
UNIQUE KEY `thekey` (`object_path`(512), `hash_id`, `syshost`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_bin AUTO_INCREMENT=1
""")
print("(%d) create xalt_object table" % idx ); idx += 1;
# 3
cursor.execute("""
CREATE TABLE `join_link_object` (
`join_id` int(11) NOT NULL auto_increment,
`obj_id` int(11) NOT NULL,
`link_id` int(11) NOT NULL,
PRIMARY KEY (`join_id`),
FOREIGN KEY (`link_id`) REFERENCES `xalt_link`(`link_id`),
FOREIGN KEY (`obj_id`) REFERENCES `xalt_object`(`obj_id`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_bin AUTO_INCREMENT=1
""")
print("(%d) create join_link_object table" % idx); idx += 1
# 4
cursor.execute("""
CREATE TABLE `xalt_run` (
`run_id` int(11) NOT NULL auto_increment,
`job_id` char(11) NOT NULL,
`run_uuid` char(36) NOT NULL,
`date` datetime NOT NULL,
`syshost` varchar(64) NOT NULL,
`uuid` char(36) ,
`hash_id` char(40) NOT NULL,
`account` char(11) NOT NULL,
`exec_type` char(7) NOT NULL,
`start_time` double NOT NULL,
`end_time` double NOT NULL,
`run_time` double NOT NULL,
`num_cores` int(11) NOT NULL,
`num_nodes` int(11) NOT NULL,
`num_threads` tinyint(4) NOT NULL,
`queue` varchar(32) NOT NULL,
`user` varchar(32) NOT NULL,
`exec_path` varchar(1024) NOT NULL,
`module_name` varchar(64) ,
`cwd` varchar(1024) NOT NULL,
PRIMARY KEY (`run_id`),
INDEX `index_run_uuid` (`run_uuid`),
INDEX `thekey` (`job_id`, `syshost`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_bin AUTO_INCREMENT=1
""")
print("(%d) create xalt_run table" % idx)
idx += 1
# 5
cursor.execute("""
CREATE TABLE `join_run_object` (
`join_id` int(11) NOT NULL auto_increment,
`obj_id` int(11) NOT NULL,
`run_id` int(11) NOT NULL,
PRIMARY KEY (`join_id`),
FOREIGN KEY (`run_id`) REFERENCES `xalt_run`(`run_id`),
FOREIGN KEY (`obj_id`) REFERENCES `xalt_object`(`obj_id`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_bin AUTO_INCREMENT=1
""")
print("(%d) create join_run_object table" % idx); idx += 1
# 6
cursor.execute("""
CREATE TABLE `xalt_env_name` (
`env_id` int(11) NOT NULL auto_increment,
`env_name` varchar(64) NOT NULL,
PRIMARY KEY (`env_id`),
UNIQUE KEY `env_name` (`env_name`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_bin AUTO_INCREMENT=1
""")
print("(%d) create xalt_env_name table" % idx); idx += 1
# 7
cursor.execute("""
CREATE TABLE `join_run_env` (
`join_id` int(11) NOT NULL auto_increment,
`env_id` int(11) NOT NULL,
`run_id` int(11) NOT NULL,
`env_value` blob NOT NULL,
PRIMARY KEY (`join_id`),
FOREIGN KEY (`env_id`) REFERENCES `xalt_env_name`(`env_id`),
FOREIGN KEY (`run_id`) REFERENCES `xalt_run`(`run_id`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_bin AUTO_INCREMENT=1
""")
print("(%d) create join_run_env table" % idx); idx += 1
# 8
cursor.execute("""
CREATE TABLE `xalt_job_id` (
`inc_id` int(11) NOT NULL auto_increment,
`job_id` int(11) NOT NULL,
PRIMARY KEY (`inc_id`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_bin AUTO_INCREMENT=1
""")
query = "INSERT INTO xalt_job_id VALUES(NULL, '0')"
conn.query(query)
cursor.close()
except MySQLdb.Error, e:
print ("Error %d: %s" % (e.args[0], e.args[1]))
sys.exit (1)
def main():
"""
This program upgrade the Database used by XALT to the current version
from the version specified in script name.
"""
args = CmdLineOptions().execute()
configFn = dbConfigFn(args.dbname)
if (not os.path.isfile(configFn)):
dirNm, exe = os.path.split(sys.argv[0])
fn = os.path.join(dirNm, configFn)
if (os.path.isfile(fn)):
configFn = fn
else:
configFn = os.path.abspath(os.path.join(dirNm, "../site",
configFn))
xalt = XALTdb(configFn)
db = xalt.db()
try:
conn = xalt.connect()
cursor = conn.cursor()
# If MySQL version < 4.1, comment out the line below
cursor.execute("SET SQL_MODE=\"NO_AUTO_VALUE_ON_ZERO\"")
cursor.execute("USE " + xalt.db())
idx = 1
print("start")
###########################################################################
# NOTE: New DB schema createDB.py uses unsigned int for some columns, but #
# modifying a signed to unsigned is problematic for columns already #
# referenced as FOREIGN KEYS. Therefor this script does not update #
# those columns. #
###########################################################################
# 1
cursor.execute("""
ALTER TABLE `xalt_link`
MODIFY COLUMN `link_program` varchar(64) NOT NULL,
ADD COLUMN `link_path` varchar(1024) NOT NULL AFTER `link_program`,
ADD COLUMN `link_module_name` varchar(64) AFTER `link_path`,
ADD COLUMN `link_line` blob AFTER `link_module_name`,
ADD INDEX `index_date` (`date`)
""")
print("(%d) upgraded xalt_link table" % idx)
idx += 1
# 4
cursor.execute("""
ALTER TABLE `xalt_run`
MODIFY COLUMN `job_id` char(64) NOT NULL,
MODIFY COLUMN `num_cores` int(11) unsigned NOT NULL,
MODIFY COLUMN `job_num_cores` int(11) unsigned NOT NULL,
MODIFY COLUMN `num_nodes` int(11) unsigned NOT NULL,
MODIFY COLUMN `num_threads` smallint(6) unsigned NOT NULL,
MODIFY COLUMN `exit_code` smallint(6) NOT NULL,
ADD COLUMN `cmdline` blob NOT NULL AFTER `cwd`,
ADD INDEX `index_date` (`date` )
""")
print("(%d) upgraded xalt_run table" % idx)
idx += 1
# 6
cursor.execute("""
ALTER TABLE `xalt_env_name`
ADD INDEX `a_env_name` (`env_name`)
""")
print("(%d) upgraded xalt_env_name table" % idx)
idx += 1
# 7
cursor.execute("""
ALTER TABLE `join_run_env`
MODIFY COLUMN `join_id` bigint(20) unsigned NOT NULL auto_increment
""")
print("(%d) upgraded join_run_env table" % idx)
idx += 1
#11
cursor.execute("""
ALTER TABLE `join_link_function`
MODIFY COLUMN `join_id` int(11) unsigned NOT NULL auto_increment
""")
print("(%d) upgraded xalt_function table" % idx)
idx += 1
# 12
cursor.execute("""
CREATE TABLE IF NOT EXISTS `xalt_total_env` (
`envT_id` bigint(20) unsigned NOT NULL auto_increment,
`run_id` int(11) NOT NULL,
`env_blob` blob NOT NULL,
PRIMARY KEY (`envT_id`),
FOREIGN KEY (`run_id`) REFERENCES `xalt_run`(`run_id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_general_ci AUTO_INCREMENT=1
""")
print("(%d) created xalt_env table" % idx)
idx += 1
cursor.close()
except MySQLdb.Error as e:
print("Error %d: %s" % (e.args[0], e.args[1]))
sys.exit(1)
def main():
"""
This program creates the Database used by XALT.
"""
args = CmdLineOptions().execute()
configFn = dbConfigFn(args.dbname)
if (not os.path.isfile(configFn)):
dirNm, exe = os.path.split(sys.argv[0])
fn = os.path.join(dirNm, configFn)
if (os.path.isfile(fn)):
configFn = fn
else:
configFn = os.path.abspath(os.path.join(dirNm, "../site", configFn))
xalt = XALTdb(configFn)
db = xalt.db()
try:
conn = xalt.connect()
cursor = conn.cursor()
# If MySQL version < 4.1, comment out the line below
cursor.execute("SET SQL_MODE=\"NO_AUTO_VALUE_ON_ZERO\"")
# If the database does not exist, create it, otherwise, switch to the database.
cursor.execute("CREATE DATABASE IF NOT EXISTS %s DEFAULT CHARACTER SET utf8 COLLATE utf8_general_ci" % xalt.db())
cursor.execute("USE "+xalt.db())
idx = 1
print("start")
# 1
cursor.execute("""
CREATE TABLE IF NOT EXISTS `xalt_link` (
`link_id` int(11) unsigned NOT NULL auto_increment,
`uuid` char(36) NOT NULL,
`hash_id` char(40) NOT NULL,
`date` DATETIME NOT NULL,
`link_program` varchar(64) NOT NULL,
`link_path` varchar(1024) NOT NULL,
`link_module_name` varchar(64) ,
`link_line` blob ,
`cwd` varchar(1024) ,
`build_user` varchar(64) NOT NULL,
`build_syshost` varchar(64) NOT NULL,
`build_epoch` double NOT NULL,
`exit_code` tinyint(4) NOT NULL,
`exec_path` varchar(1024) NOT NULL,
PRIMARY KEY (`link_id`),
INDEX `index_date` (`date`),
UNIQUE KEY `uuid` (`uuid`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_general_ci AUTO_INCREMENT=1
""")
print("(%d) create xalt_link table" % idx); idx += 1
# 2
cursor.execute("""
CREATE TABLE IF NOT EXISTS `xalt_object` (
`obj_id` int(11) unsigned NOT NULL auto_increment,
`object_path` varchar(1024) NOT NULL,
`syshost` varchar(64) NOT NULL,
`hash_id` char(40) NOT NULL,
`module_name` varchar(64) ,
`timestamp` TIMESTAMP ,
`lib_type` char(2) NOT NULL,
PRIMARY KEY (`obj_id`),
INDEX `index_hash_id` (`hash_id`),
INDEX `thekey` (`object_path`(128), `hash_id`, `syshost`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_general_ci AUTO_INCREMENT=1
""")
print("(%d) create xalt_object table" % idx ); idx += 1;
# 3
cursor.execute("""
CREATE TABLE IF NOT EXISTS `join_link_object` (
`join_id` int(11) unsigned NOT NULL auto_increment,
`obj_id` int(11) unsigned NOT NULL,
`link_id` int(11) unsigned NOT NULL,
PRIMARY KEY (`join_id`),
FOREIGN KEY (`link_id`) REFERENCES `xalt_link`(`link_id`),
FOREIGN KEY (`obj_id`) REFERENCES `xalt_object`(`obj_id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_general_ci AUTO_INCREMENT=1
""")
print("(%d) create join_link_object table" % idx); idx += 1
# 4
cursor.execute("""
CREATE TABLE IF NOT EXISTS `xalt_run` (
`run_id` int(11) unsigned NOT NULL auto_increment,
`job_id` char(64) NOT NULL,
`run_uuid` char(36) NOT NULL,
`date` datetime NOT NULL,
`syshost` varchar(64) NOT NULL,
`uuid` char(36) ,
`hash_id` char(40) NOT NULL,
`account` varchar(20) NOT NULL,
`exec_type` char(7) NOT NULL,
`start_time` double NOT NULL,
`end_time` double NOT NULL,
`run_time` double NOT NULL,
`num_cores` int(11) unsigned NOT NULL,
`job_num_cores` int(11) unsigned NOT NULL,
`num_nodes` int(11) unsigned NOT NULL,
`num_threads` smallint(6) unsigned NOT NULL,
`queue` varchar(32) NOT NULL,
`exit_code` smallint(6) NOT NULL,
`user` varchar(32) NOT NULL,
`exec_path` varchar(1024) NOT NULL,
`module_name` varchar(64) ,
`cwd` varchar(1024) NOT NULL,
`cmdline` blob NOT NULL,
PRIMARY KEY (`run_id` ),
INDEX `index_date` (`date` ),
INDEX `index_run_uuid` (`run_uuid` ),
INDEX `thekey` (`job_id`, `syshost` )
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_general_ci AUTO_INCREMENT=1
""")
print("(%d) create xalt_run table" % idx)
idx += 1
# 5
cursor.execute("""
CREATE TABLE IF NOT EXISTS `join_run_object` (
`join_id` int(11) unsigned NOT NULL auto_increment,
`obj_id` int(11) unsigned NOT NULL,
`run_id` int(11) unsigned NOT NULL,
PRIMARY KEY (`join_id`),
FOREIGN KEY (`run_id`) REFERENCES `xalt_run`(`run_id`),
FOREIGN KEY (`obj_id`) REFERENCES `xalt_object`(`obj_id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_general_ci AUTO_INCREMENT=1
""")
print("(%d) create join_run_object table" % idx); idx += 1
# 6
cursor.execute("""
CREATE TABLE IF NOT EXISTS `xalt_env_name` (
`env_id` int(20) unsigned NOT NULL auto_increment,
`env_name` varchar(64) NOT NULL,
PRIMARY KEY (`env_id`),
UNIQUE KEY `env_name` (`env_name`),
INDEX `a_env_name` (`env_name`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_general_ci AUTO_INCREMENT=1
""")
print("(%d) create xalt_env_name table" % idx); idx += 1
# 7
cursor.execute("""
CREATE TABLE IF NOT EXISTS `join_run_env` (
`join_id` bigint(20) unsigned NOT NULL auto_increment,
`env_id` int(11) unsigned NOT NULL,
`run_id` int(11) unsigned NOT NULL,
`env_value` blob NOT NULL,
PRIMARY KEY (`join_id`),
FOREIGN KEY (`env_id`) REFERENCES `xalt_env_name`(`env_id`),
FOREIGN KEY (`run_id`) REFERENCES `xalt_run`(`run_id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_general_ci AUTO_INCREMENT=1
""")
print("(%d) create join_run_env table" % idx); idx += 1
# 8
cursor.execute("""
CREATE TABLE IF NOT EXISTS `xalt_user` (
`usr_id` int(11) unsigned NOT NULL auto_increment,
`user` varchar(32) NOT NULL,
`anon_user` varchar(12) NOT NULL,
PRIMARY KEY (`usr_id`),
INDEX `the_user` (`user`),
INDEX `a_user` (`anon_user`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_general_ci AUTO_INCREMENT=1
""")
print("(%d) create xalt_user table" % idx); idx += 1
# 9
cursor.execute("""
CREATE TABLE IF NOT EXISTS `xalt_account` (
`acct_id` int(11) unsigned NOT NULL auto_increment,
`account` varchar(32) NOT NULL,
`anon_user` varchar(10) NOT NULL,
`field_of_science` varchar(64) NOT NULL,
PRIMARY KEY (`acct_id`),
INDEX `the_account` (`account`),
INDEX `a_acct` (`anon_user`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_general_ci AUTO_INCREMENT=1
""")
print("(%d) create xalt_account table" % idx ); idx += 1;
# 10
cursor.execute("""
CREATE TABLE IF NOT EXISTS `xalt_function` (
`func_id` int(11) unsigned NOT NULL auto_increment,
`function_name` varchar(255) NOT NULL,
PRIMARY KEY (`func_id`),
UNIQUE KEY `function_name` (`function_name`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_general_ci AUTO_INCREMENT=1
""")
print("(%d) create xalt_function table" % idx ); idx += 1;
# 11
cursor.execute("""
CREATE TABLE IF NOT EXISTS `join_link_function` (
`join_id` int(11) unsigned NOT NULL auto_increment,
`func_id` int(11) unsigned NOT NULL,
`link_id` int(11) unsigned NOT NULL,
PRIMARY KEY (`join_id`),
FOREIGN KEY (`func_id`) REFERENCES `xalt_function`(`func_id`),
FOREIGN KEY (`link_id`) REFERENCES `xalt_link`(`link_id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_general_ci AUTO_INCREMENT=1
""")
print("(%d) create join_link_function table" % idx); idx += 1
# 12
cursor.execute("""
CREATE TABLE IF NOT EXISTS `xalt_total_env` (
`envT_id` bigint(20) unsigned NOT NULL auto_increment,
`run_id` int(11) unsigned NOT NULL,
`env_blob` blob NOT NULL,
PRIMARY KEY (`envT_id`),
FOREIGN KEY (`run_id`) REFERENCES `xalt_run`(`run_id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_general_ci AUTO_INCREMENT=1
""")
print("(%d) create xalt_env table" % idx); idx += 1
# 13
cursor.execute("""
ALTER TABLE `join_link_function`
ADD UNIQUE `unique_func_link_id` ( `func_id`, `link_id` )
""")
print("(%d) create unique key on join_link_function table" % idx); idx += 1
cursor.close()
except MySQLdb.Error as e:
print ("Error %d: %s" % (e.args[0], e.args[1]))
sys.exit (1)
def main():
"""
read from syslog file into XALT db.
"""
sA = []
sA.append("CommandLine:")
for v in sys.argv:
sA.append('"'+v+'"')
XALT_Stack.push(" ".join(sA))
args = CmdLineOptions().execute()
xalt = XALTdb(ConfigFn)
syslogFile = args.syslog
# should add a check if file exists
num = int(capture("cat "+syslogFile+" | wc -l"))
icnt = 0
t1 = time.time()
rmapT = Rmap(args.rmapD).reverseMapT()
lnkCnt = 0
runCnt = 0
badCnt = 0
count = 0
if num != 0:
pbar = ProgressBar(maxVal=num)
if (os.path.isfile(syslogFile)):
f=open(syslogFile, 'r')
for line in f:
if "XALT_LOGGING" in line:
i=line.find("link:")
if i == -1 :
i=line.find("run:")
if i == -1:
continue # did not find a link or run, continue to next line
array=line[i:].split(":")
type = array[0].strip()
syshost = array[1].strip()
try:
resultT=json.loads(base64.b64decode(array[2]))
XALT_Stack.push("XALT_LOGGING: " + type + " " + syshost)
# XALT_Stack.push("XALT_LOGGING resultT: " + resultT)
if ( type == "link" ):
XALT_Stack.push("link_to_db()")
xalt.link_to_db(rmapT, resultT)
XALT_Stack.pop()
lnkCnt += 1
elif ( type == "run" ):
XALT_Stack.push("run_to_db()")
xalt.run_to_db(rmapT, resultT)
XALT_Stack.pop()
runCnt += 1
else:
print("Error in xalt_syslog_to_db")
XALT_Stack.pop()
except:
badCnt += 1
# figure out length of array[2], as it might be
# near the syslog message size limit
strg=array[2]
lens = len(strg)
# lenx = lens - (lens % 4 if lens % 4 else 4)
print("xalt_syslog_to_db: undecodable entry! length: ",lens)
# resultT = base64.decodestring(strg[:lenx])
# print("result: ",result)
count += 1
pbar.update(count)
pbar.fini()
# what should be done if there are errors?
# what went wrong?
# how do we restart?
#
# xalt.connect().close()
t2 = time.time()
rt = t2 - t1
if (args.timer):
print("Time: ", time.strftime("%T", time.gmtime(rt)))
#
print("total processed : ", count, ", num links: ", lnkCnt, ", num runs: ", runCnt, ", badCnt: ", badCnt)
