def run_to_db(self, reverseMapT, runT):
"""
Store the "run" data into the database.
@param: reverseMapT: The map between directories and modules
@param: runT: The run data stored in a table
"""
nameA = [
'num_cores', 'num_nodes', 'account', 'job_id', 'queue',
'submit_host'
]
query = ""
try:
conn = self.connect()
query = "USE " + self.db()
conn.query(query)
query = "START TRANSACTION"
conn.query(query)
translate(nameA, runT['envT'], runT['userT'])
XALT_Stack.push("SUBMIT_HOST: " + runT['userT']['submit_host'])
dateTimeStr = time.strftime(
"%Y-%m-%d %H:%M:%S",
time.localtime(float(runT['userT']['start_time'])))
uuid = runT['xaltLinkT'].get('Build.UUID')
if (uuid):
uuid = "'" + uuid + "'"
else:
uuid = "NULL"
#print( "Looking for run_uuid: ",runT['userT']['run_uuid'])
query = "SELECT run_id FROM xalt_run WHERE run_uuid='%s'" % runT[
'userT']['run_uuid']
conn.query(query)
result = conn.store_result()
if (result.num_rows() > 0):
#print("found")
row = result.fetch_row()
run_id = int(row[0][0])
if (runT['userT']['end_time'] > 0):
query = "UPDATE xalt_run SET run_time='%.2f', end_time='%.2f' WHERE run_id='%d'" % (
runT['userT']['run_time'], runT['userT']['end_time'],
run_id)
conn.query(query)
query = "COMMIT"
conn.query(query)
v = XALT_Stack.pop()
carp("SUBMIT_HOST", v)
return
else:
#print("not found")
moduleName = obj2module(runT['userT']['exec_path'],
reverseMapT)
exit_status = runT['userT'].get('exit_status', 0)
query = "INSERT INTO xalt_run VALUES (NULL,'%s','%s','%s', '%s',%s,'%s', '%s','%s','%.2f', '%.2f','%.2f','%d', '%d','%d','%s', '%d','%s','%s', '%s','%s') " % (
runT['userT']['job_id'], runT['userT']['run_uuid'],
dateTimeStr, runT['userT']['syshost'], uuid,
runT['hash_id'], runT['userT']['account'],
runT['userT']['exec_type'], runT['userT']['start_time'],
runT['userT']['end_time'], runT['userT']['run_time'],
runT['userT']['num_cores'], runT['userT']['num_nodes'],
runT['userT']['num_threads'], runT['userT']['queue'],
exit_status, runT['userT']['user'],
runT['userT']['exec_path'], moduleName,
runT['userT']['cwd'])
conn.query(query)
run_id = conn.insert_id()
self.load_objects(conn, runT['libA'], reverseMapT,
runT['userT']['syshost'], "join_run_object",
run_id)
# loop over env. vars.
for key in runT['envT']:
# use the single quote pattern to protect all the single quotes in env vars.
value = patSQ.sub(r"\\'", runT['envT'][key])
query = "SELECT env_id FROM xalt_env_name WHERE env_name='%s'" % key
conn.query(query)
result = conn.store_result()
if (result.num_rows() > 0):
row = result.fetch_row()
env_id = int(row[0][0])
found = True
else:
query = "INSERT INTO xalt_env_name VALUES(NULL, '%s')" % key
conn.query(query)
env_id = conn.insert_id()
found = False
query = "INSERT INTO join_run_env VALUES (NULL, '%d', '%d', '%s')" % (
env_id, run_id, value.encode("ascii", "ignore"))
try:
conn.query(query)
except Exception as e:
query = "INSERT INTO join_run_env VALUES (NULL, '%d', '%d', '%s')" % (
env_id, run_id, "XALT_ILLEGAL_VALUE")
conn.query(query)
v = XALT_Stack.pop()
carp("SUBMIT_HOST", v)
query = "COMMIT"
conn.query(query)
conn.close()
except Exception as e:
print(XALT_Stack.contents())
print(query.encode("ascii", "ignore"))
print("run_to_db(): ", e)
sys.exit(1)
def run_to_db(self, reverseMapT, runT):
"""
Store the "run" data into the database.
@param: reverseMapT: The map between directories and modules
@param: runT: The run data stored in a table
"""
nameA = [ 'num_cores', 'num_nodes', 'account', 'job_id', 'queue' , 'submit_host']
query = ""
try:
conn = self.connect()
cursor = conn.cursor()
query = "USE "+self.db()
conn.query(query)
query = "START TRANSACTION"
conn.query(query)
translate(nameA, runT['envT'], runT['userT']);
XALT_Stack.push("SUBMIT_HOST: "+ runT['userT']['submit_host'])
runTime = "%.2f" % float(runT['userT']['run_time'])
endTime = "%.2f" % float(runT['userT']['end_time'])
dateTimeStr = time.strftime("%Y-%m-%d %H:%M:%S",
time.localtime(float(runT['userT']['start_time'])))
uuid = runT['xaltLinkT'].get('Build.UUID',None)
#print( "Looking for run_uuid: ",runT['userT']['run_uuid'])
query = "SELECT run_id FROM xalt_run WHERE run_uuid=%s"
cursor.execute(query,[runT['userT']['run_uuid']])
if (cursor.rowcount > 0):
#print("found")
row = cursor.fetchone()
run_id = int(row[0])
if (runT['userT']['end_time'] > 0):
query = "UPDATE xalt_run SET run_time=%s, end_time=%s WHERE run_id=%s"
cursor.execute(query,(runTime, endTime, run_id))
query = "COMMIT"
conn.query(query)
v = XALT_Stack.pop()
carp("SUBMIT_HOST",v)
return
else:
#print("not found")
moduleName = obj2module(runT['userT']['exec_path'], reverseMapT)
exit_status = convertToTinyInt(runT['userT'].get('exit_status',0))
num_threads = convertToTinyInt(runT['userT'].get('num_threads',0))
usr_cmdline = runT.get('cmdlineA', "UNKNOWN")
job_num_cores = int(runT['userT'].get('job_num_cores',0))
startTime = "%.f" % float(runT['userT']['start_time'])
query = "INSERT INTO xalt_run VALUES (NULL, %s,%s,%s, %s,%s,%s, %s,%s,%s, %s,%s,%s, %s,%s,%s, %s,%s,%s, %s,%s,%s, COMPRESS(%s))"
cursor.execute(query, (runT['userT']['job_id'], runT['userT']['run_uuid'], dateTimeStr,
runT['userT']['syshost'], uuid, runT['hash_id'],
runT['userT']['account'][:20],runT['userT']['exec_type'], startTime,
endTime, runTime, runT['userT']['num_cores'],
job_num_cores, runT['userT']['num_nodes'], num_threads,
runT['userT']['queue'], exit_status, runT['userT']['user'],
runT['userT']['exec_path'], moduleName, runT['userT']['cwd'],
usr_cmdline))
run_id = cursor.lastrowid
self.load_objects(conn, runT['libA'], reverseMapT, runT['userT']['syshost'],
"join_run_object", run_id)
envT = runT['envT']
jsonStr = json.dumps(envT)
query = "INSERT INTO xalt_total_env VALUES(NULL, %s, COMPRESS(%s))"
cursor.execute(query, [run_id, jsonStr])
# loop over env. vars.
for key in envT:
if (not keep_env_var(key)):
continue
value = envT[key]
query = "SELECT env_id FROM xalt_env_name WHERE env_name=%s"
cursor.execute(query,[key])
if (cursor.rowcount > 0):
row = cursor.fetchone()
env_id = int(row[0])
found = True
else:
query = "INSERT INTO xalt_env_name VALUES(NULL, %s)"
cursor.execute(query,[key[:64]])
env_id = cursor.lastrowid
found = False
query = "INSERT INTO join_run_env VALUES (NULL, %s, %s, %s)"
cursor.execute(query,(env_id, run_id, value))
v = XALT_Stack.pop()
carp("SUBMIT_HOST",v)
query = "COMMIT"
conn.query(query)
conn.close()
except Exception as e:
print(XALT_Stack.contents())
print(query.encode("ascii","ignore"))
print ("run_to_db(): %s" % e)
def run_to_db(self, reverseMapT, runT):
"""
Store the "run" data into the database.
@param: reverseMapT: The map between directories and modules
@param: runT: The run data stored in a table
"""
nameA = [
'num_cores', 'num_nodes', 'account', 'job_id', 'queue',
'submit_host'
]
query = ""
try:
conn = self.connect()
cursor = conn.cursor()
query = "USE " + self.db()
conn.query(query)
query = "START TRANSACTION"
conn.query(query)
translate(nameA, runT['envT'], runT['userT'])
XALT_Stack.push("SUBMIT_HOST: " + runT['userT']['submit_host'])
runTime = "%.2f" % float(runT['userT']['run_time'])
endTime = "%.2f" % float(runT['userT']['end_time'])
dateTimeStr = time.strftime(
"%Y-%m-%d %H:%M:%S",
time.localtime(float(runT['userT']['start_time'])))
uuid = runT['xaltLinkT'].get('Build.UUID', None)
#print( "Looking for run_uuid: ",runT['userT']['run_uuid'])
query = "SELECT run_id FROM xalt_run WHERE run_uuid=%s"
cursor.execute(query, [runT['userT']['run_uuid']])
if (cursor.rowcount > 0):
#print("found")
row = cursor.fetchone()
run_id = int(row[0])
if (runT['userT']['end_time'] > 0):
query = "UPDATE xalt_run SET run_time=%s, end_time=%s WHERE run_id=%s"
cursor.execute(query, (runTime, endTime, run_id))
query = "COMMIT"
conn.query(query)
v = XALT_Stack.pop()
carp("SUBMIT_HOST", v)
return
else:
#print("not found")
moduleName = obj2module(runT['userT']['exec_path'],
reverseMapT)
exit_status = convertToTinyInt(runT['userT'].get(
'exit_status', 0))
num_threads = convertToTinyInt(runT['userT'].get(
'num_threads', 0))
usr_cmdline = json.dumps(runT['cmdlineA'])
job_num_cores = int(runT['userT'].get('job_num_cores', 0))
startTime = "%.f" % float(runT['userT']['start_time'])
query = "INSERT INTO xalt_run VALUES (NULL, %s,%s,%s, %s,%s,%s, %s,%s,%s, %s,%s,%s, %s,%s,%s, %s,%s,%s, %s,%s,%s, COMPRESS(%s))"
cursor.execute(
query, (runT['userT']['job_id'], runT['userT']['run_uuid'],
dateTimeStr, runT['userT']['syshost'], uuid,
runT['hash_id'], runT['userT']['account'][:20],
runT['userT']['exec_type'], startTime, endTime,
runTime, runT['userT']['num_cores'], job_num_cores,
runT['userT']['num_nodes'], num_threads,
runT['userT']['queue'], exit_status,
runT['userT']['user'], runT['userT']['exec_path'],
moduleName, runT['userT']['cwd'], usr_cmdline))
run_id = cursor.lastrowid
self.load_objects(conn, runT['libA'], reverseMapT,
runT['userT']['syshost'], "join_run_object",
run_id)
envT = runT['envT']
jsonStr = json.dumps(envT)
query = "INSERT INTO xalt_total_env VALUES(NULL, %s, COMPRESS(%s))"
cursor.execute(query, [run_id, jsonStr])
# loop over env. vars.
for key in envT:
if (not keep_env_var(key)):
continue
value = envT[key]
query = "SELECT env_id FROM xalt_env_name WHERE env_name=%s"
cursor.execute(query, [key])
if (cursor.rowcount > 0):
row = cursor.fetchone()
env_id = int(row[0])
found = True
else:
query = "INSERT INTO xalt_env_name VALUES(NULL, %s)"
cursor.execute(query, [key[:64]])
env_id = cursor.lastrowid
found = False
query = "INSERT INTO join_run_env VALUES (NULL, %s, %s, %s)"
cursor.execute(query, (env_id, run_id, value))
v = XALT_Stack.pop()
carp("SUBMIT_HOST", v)
query = "COMMIT"
conn.query(query)
conn.close()
except Exception as e:
print(XALT_Stack.contents())
print(query.encode("ascii", "ignore"))
print("run_to_db(): %s" % e)
sys.exit(1)
def run_to_db(self, reverseMapT, runT):
"""
Store the "run" data into the database.
@param: reverseMapT: The map between directories and modules
@param: runT: The run data stored in a table
"""
nameA = [ 'num_cores', 'num_nodes', 'account', 'job_id', 'queue' , 'submit_host']
try:
conn = self.connect()
query = "USE "+self.db()
conn.query(query)
translate(nameA, runT['envT'], runT['userT']);
XALT_Stack.push("SUBMIT_HOST: "+ runT['userT']['submit_host'])
dateTimeStr = time.strftime("%Y-%m-%d %H:%M:%S",
time.localtime(float(runT['userT']['start_time'])))
uuid = runT['xaltLinkT'].get('Build.UUID')
if (uuid):
uuid = "'" + uuid + "'"
else:
uuid = "NULL"
#print( "Looking for run_uuid: ",runT['userT']['run_uuid'])
query = "SELECT run_id FROM xalt_run WHERE run_uuid='%s'" % runT['userT']['run_uuid']
conn.query(query)
result = conn.store_result()
if (result.num_rows() > 0):
#print("found")
row = result.fetch_row()
run_id = int(row[0][0])
query = "UPDATE xalt_run SET run_time='%.2f', end_time='%.2f' WHERE run_id='%d'" % (
runT['userT']['run_time'], runT['userT']['end_time'], run_id)
conn.query(query)
v = XALT_Stack.pop()
carp("SUBMIT_HOST",v)
return
else:
#print("not found")
moduleName = obj2module(runT['userT']['exec_path'], reverseMapT)
query = "INSERT INTO xalt_run VALUES (NULL,'%s','%s','%s', '%s',%s,'%s', '%s','%s','%.2f', '%.2f','%.2f','%d', '%d','%d','%s', '%s','%s',%s,'%s') " % (
runT['userT']['job_id'], runT['userT']['run_uuid'], dateTimeStr,
runT['userT']['syshost'], uuid, runT['hash_id'],
runT['userT']['account'], runT['userT']['exec_type'], runT['userT']['start_time'],
runT['userT']['end_time'], runT['userT']['run_time'], runT['userT']['num_cores'],
runT['userT']['num_nodes'], runT['userT']['num_threads'], runT['userT']['queue'],
runT['userT']['user'], runT['userT']['exec_path'], moduleName,
runT['userT']['cwd'])
conn.query(query)
run_id = conn.insert_id()
self.load_objects(conn, runT['libA'], reverseMapT, runT['userT']['syshost'],
"join_run_object", run_id)
# loop over env. vars.
for key in runT['envT']:
# use the single quote pattern to protect all the single quotes in env vars.
value = patSQ.sub(r"\\'", runT['envT'][key])
query = "SELECT env_id FROM xalt_env_name WHERE env_name='%s'" % key
conn.query(query)
result = conn.store_result()
if (result.num_rows() > 0):
row = result.fetch_row()
env_id = int(row[0][0])
found = True
else:
query = "INSERT INTO xalt_env_name VALUES(NULL, '%s')" % key
conn.query(query)
env_id = conn.insert_id()
found = False
#print("env_id: ", env_id, ", found: ",found)
query = "INSERT INTO join_run_env VALUES (NULL, '%d', '%d', '%s')" % (
env_id, run_id, value.encode("ascii","ignore"))
conn.query(query)
v = XALT_Stack.pop()
carp("SUBMIT_HOST",v)
except Exception as e:
print(XALT_Stack.contents())
print(query.encode("ascii","ignore"))
print ("run_to_db(): ",e)
sys.exit (1)