def run_to_db(self, reverseMapT, runT): """ Store the "run" data into the database. @param: reverseMapT: The map between directories and modules @param: runT: The run data stored in a table """ nameA = [ 'num_cores', 'num_nodes', 'account', 'job_id', 'queue', 'submit_host' ] query = "" try: conn = self.connect() query = "USE " + self.db() conn.query(query) query = "START TRANSACTION" conn.query(query) translate(nameA, runT['envT'], runT['userT']) XALT_Stack.push("SUBMIT_HOST: " + runT['userT']['submit_host']) dateTimeStr = time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(float(runT['userT']['start_time']))) uuid = runT['xaltLinkT'].get('Build.UUID') if (uuid): uuid = "'" + uuid + "'" else: uuid = "NULL" #print( "Looking for run_uuid: ",runT['userT']['run_uuid']) query = "SELECT run_id FROM xalt_run WHERE run_uuid='%s'" % runT[ 'userT']['run_uuid'] conn.query(query) result = conn.store_result() if (result.num_rows() > 0): #print("found") row = result.fetch_row() run_id = int(row[0][0]) if (runT['userT']['end_time'] > 0): query = "UPDATE xalt_run SET run_time='%.2f', end_time='%.2f' WHERE run_id='%d'" % ( runT['userT']['run_time'], runT['userT']['end_time'], run_id) conn.query(query) query = "COMMIT" conn.query(query) v = XALT_Stack.pop() carp("SUBMIT_HOST", v) return else: #print("not found") moduleName = obj2module(runT['userT']['exec_path'], reverseMapT) exit_status = runT['userT'].get('exit_status', 0) query = "INSERT INTO xalt_run VALUES (NULL,'%s','%s','%s', '%s',%s,'%s', '%s','%s','%.2f', '%.2f','%.2f','%d', '%d','%d','%s', '%d','%s','%s', '%s','%s') " % ( runT['userT']['job_id'], runT['userT']['run_uuid'], dateTimeStr, runT['userT']['syshost'], uuid, runT['hash_id'], runT['userT']['account'], runT['userT']['exec_type'], runT['userT']['start_time'], runT['userT']['end_time'], runT['userT']['run_time'], runT['userT']['num_cores'], runT['userT']['num_nodes'], runT['userT']['num_threads'], runT['userT']['queue'], exit_status, runT['userT']['user'], runT['userT']['exec_path'], moduleName, runT['userT']['cwd']) conn.query(query) run_id = conn.insert_id() self.load_objects(conn, runT['libA'], reverseMapT, runT['userT']['syshost'], "join_run_object", run_id) # loop over env. vars. for key in runT['envT']: # use the single quote pattern to protect all the single quotes in env vars. value = patSQ.sub(r"\\'", runT['envT'][key]) query = "SELECT env_id FROM xalt_env_name WHERE env_name='%s'" % key conn.query(query) result = conn.store_result() if (result.num_rows() > 0): row = result.fetch_row() env_id = int(row[0][0]) found = True else: query = "INSERT INTO xalt_env_name VALUES(NULL, '%s')" % key conn.query(query) env_id = conn.insert_id() found = False query = "INSERT INTO join_run_env VALUES (NULL, '%d', '%d', '%s')" % ( env_id, run_id, value.encode("ascii", "ignore")) try: conn.query(query) except Exception as e: query = "INSERT INTO join_run_env VALUES (NULL, '%d', '%d', '%s')" % ( env_id, run_id, "XALT_ILLEGAL_VALUE") conn.query(query) v = XALT_Stack.pop() carp("SUBMIT_HOST", v) query = "COMMIT" conn.query(query) conn.close() except Exception as e: print(XALT_Stack.contents()) print(query.encode("ascii", "ignore")) print("run_to_db(): ", e) sys.exit(1)
def run_to_db(self, reverseMapT, runT): """ Store the "run" data into the database. @param: reverseMapT: The map between directories and modules @param: runT: The run data stored in a table """ nameA = [ 'num_cores', 'num_nodes', 'account', 'job_id', 'queue' , 'submit_host'] query = "" try: conn = self.connect() cursor = conn.cursor() query = "USE "+self.db() conn.query(query) query = "START TRANSACTION" conn.query(query) translate(nameA, runT['envT'], runT['userT']); XALT_Stack.push("SUBMIT_HOST: "+ runT['userT']['submit_host']) runTime = "%.2f" % float(runT['userT']['run_time']) endTime = "%.2f" % float(runT['userT']['end_time']) dateTimeStr = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(float(runT['userT']['start_time']))) uuid = runT['xaltLinkT'].get('Build.UUID',None) #print( "Looking for run_uuid: ",runT['userT']['run_uuid']) query = "SELECT run_id FROM xalt_run WHERE run_uuid=%s" cursor.execute(query,[runT['userT']['run_uuid']]) if (cursor.rowcount > 0): #print("found") row = cursor.fetchone() run_id = int(row[0]) if (runT['userT']['end_time'] > 0): query = "UPDATE xalt_run SET run_time=%s, end_time=%s WHERE run_id=%s" cursor.execute(query,(runTime, endTime, run_id)) query = "COMMIT" conn.query(query) v = XALT_Stack.pop() carp("SUBMIT_HOST",v) return else: #print("not found") moduleName = obj2module(runT['userT']['exec_path'], reverseMapT) exit_status = convertToTinyInt(runT['userT'].get('exit_status',0)) num_threads = convertToTinyInt(runT['userT'].get('num_threads',0)) usr_cmdline = runT.get('cmdlineA', "UNKNOWN") job_num_cores = int(runT['userT'].get('job_num_cores',0)) startTime = "%.f" % float(runT['userT']['start_time']) query = "INSERT INTO xalt_run VALUES (NULL, %s,%s,%s, %s,%s,%s, %s,%s,%s, %s,%s,%s, %s,%s,%s, %s,%s,%s, %s,%s,%s, COMPRESS(%s))" cursor.execute(query, (runT['userT']['job_id'], runT['userT']['run_uuid'], dateTimeStr, runT['userT']['syshost'], uuid, runT['hash_id'], runT['userT']['account'][:20],runT['userT']['exec_type'], startTime, endTime, runTime, runT['userT']['num_cores'], job_num_cores, runT['userT']['num_nodes'], num_threads, runT['userT']['queue'], exit_status, runT['userT']['user'], runT['userT']['exec_path'], moduleName, runT['userT']['cwd'], usr_cmdline)) run_id = cursor.lastrowid self.load_objects(conn, runT['libA'], reverseMapT, runT['userT']['syshost'], "join_run_object", run_id) envT = runT['envT'] jsonStr = json.dumps(envT) query = "INSERT INTO xalt_total_env VALUES(NULL, %s, COMPRESS(%s))" cursor.execute(query, [run_id, jsonStr]) # loop over env. vars. for key in envT: if (not keep_env_var(key)): continue value = envT[key] query = "SELECT env_id FROM xalt_env_name WHERE env_name=%s" cursor.execute(query,[key]) if (cursor.rowcount > 0): row = cursor.fetchone() env_id = int(row[0]) found = True else: query = "INSERT INTO xalt_env_name VALUES(NULL, %s)" cursor.execute(query,[key[:64]]) env_id = cursor.lastrowid found = False query = "INSERT INTO join_run_env VALUES (NULL, %s, %s, %s)" cursor.execute(query,(env_id, run_id, value)) v = XALT_Stack.pop() carp("SUBMIT_HOST",v) query = "COMMIT" conn.query(query) conn.close() except Exception as e: print(XALT_Stack.contents()) print(query.encode("ascii","ignore")) print ("run_to_db(): %s" % e)
def run_to_db(self, reverseMapT, runT): """ Store the "run" data into the database. @param: reverseMapT: The map between directories and modules @param: runT: The run data stored in a table """ nameA = [ 'num_cores', 'num_nodes', 'account', 'job_id', 'queue', 'submit_host' ] query = "" try: conn = self.connect() cursor = conn.cursor() query = "USE " + self.db() conn.query(query) query = "START TRANSACTION" conn.query(query) translate(nameA, runT['envT'], runT['userT']) XALT_Stack.push("SUBMIT_HOST: " + runT['userT']['submit_host']) runTime = "%.2f" % float(runT['userT']['run_time']) endTime = "%.2f" % float(runT['userT']['end_time']) dateTimeStr = time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(float(runT['userT']['start_time']))) uuid = runT['xaltLinkT'].get('Build.UUID', None) #print( "Looking for run_uuid: ",runT['userT']['run_uuid']) query = "SELECT run_id FROM xalt_run WHERE run_uuid=%s" cursor.execute(query, [runT['userT']['run_uuid']]) if (cursor.rowcount > 0): #print("found") row = cursor.fetchone() run_id = int(row[0]) if (runT['userT']['end_time'] > 0): query = "UPDATE xalt_run SET run_time=%s, end_time=%s WHERE run_id=%s" cursor.execute(query, (runTime, endTime, run_id)) query = "COMMIT" conn.query(query) v = XALT_Stack.pop() carp("SUBMIT_HOST", v) return else: #print("not found") moduleName = obj2module(runT['userT']['exec_path'], reverseMapT) exit_status = convertToTinyInt(runT['userT'].get( 'exit_status', 0)) num_threads = convertToTinyInt(runT['userT'].get( 'num_threads', 0)) usr_cmdline = json.dumps(runT['cmdlineA']) job_num_cores = int(runT['userT'].get('job_num_cores', 0)) startTime = "%.f" % float(runT['userT']['start_time']) query = "INSERT INTO xalt_run VALUES (NULL, %s,%s,%s, %s,%s,%s, %s,%s,%s, %s,%s,%s, %s,%s,%s, %s,%s,%s, %s,%s,%s, COMPRESS(%s))" cursor.execute( query, (runT['userT']['job_id'], runT['userT']['run_uuid'], dateTimeStr, runT['userT']['syshost'], uuid, runT['hash_id'], runT['userT']['account'][:20], runT['userT']['exec_type'], startTime, endTime, runTime, runT['userT']['num_cores'], job_num_cores, runT['userT']['num_nodes'], num_threads, runT['userT']['queue'], exit_status, runT['userT']['user'], runT['userT']['exec_path'], moduleName, runT['userT']['cwd'], usr_cmdline)) run_id = cursor.lastrowid self.load_objects(conn, runT['libA'], reverseMapT, runT['userT']['syshost'], "join_run_object", run_id) envT = runT['envT'] jsonStr = json.dumps(envT) query = "INSERT INTO xalt_total_env VALUES(NULL, %s, COMPRESS(%s))" cursor.execute(query, [run_id, jsonStr]) # loop over env. vars. for key in envT: if (not keep_env_var(key)): continue value = envT[key] query = "SELECT env_id FROM xalt_env_name WHERE env_name=%s" cursor.execute(query, [key]) if (cursor.rowcount > 0): row = cursor.fetchone() env_id = int(row[0]) found = True else: query = "INSERT INTO xalt_env_name VALUES(NULL, %s)" cursor.execute(query, [key[:64]]) env_id = cursor.lastrowid found = False query = "INSERT INTO join_run_env VALUES (NULL, %s, %s, %s)" cursor.execute(query, (env_id, run_id, value)) v = XALT_Stack.pop() carp("SUBMIT_HOST", v) query = "COMMIT" conn.query(query) conn.close() except Exception as e: print(XALT_Stack.contents()) print(query.encode("ascii", "ignore")) print("run_to_db(): %s" % e) sys.exit(1)
def run_to_db(self, reverseMapT, runT): """ Store the "run" data into the database. @param: reverseMapT: The map between directories and modules @param: runT: The run data stored in a table """ nameA = [ 'num_cores', 'num_nodes', 'account', 'job_id', 'queue' , 'submit_host'] try: conn = self.connect() query = "USE "+self.db() conn.query(query) translate(nameA, runT['envT'], runT['userT']); XALT_Stack.push("SUBMIT_HOST: "+ runT['userT']['submit_host']) dateTimeStr = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(float(runT['userT']['start_time']))) uuid = runT['xaltLinkT'].get('Build.UUID') if (uuid): uuid = "'" + uuid + "'" else: uuid = "NULL" #print( "Looking for run_uuid: ",runT['userT']['run_uuid']) query = "SELECT run_id FROM xalt_run WHERE run_uuid='%s'" % runT['userT']['run_uuid'] conn.query(query) result = conn.store_result() if (result.num_rows() > 0): #print("found") row = result.fetch_row() run_id = int(row[0][0]) query = "UPDATE xalt_run SET run_time='%.2f', end_time='%.2f' WHERE run_id='%d'" % ( runT['userT']['run_time'], runT['userT']['end_time'], run_id) conn.query(query) v = XALT_Stack.pop() carp("SUBMIT_HOST",v) return else: #print("not found") moduleName = obj2module(runT['userT']['exec_path'], reverseMapT) query = "INSERT INTO xalt_run VALUES (NULL,'%s','%s','%s', '%s',%s,'%s', '%s','%s','%.2f', '%.2f','%.2f','%d', '%d','%d','%s', '%s','%s',%s,'%s') " % ( runT['userT']['job_id'], runT['userT']['run_uuid'], dateTimeStr, runT['userT']['syshost'], uuid, runT['hash_id'], runT['userT']['account'], runT['userT']['exec_type'], runT['userT']['start_time'], runT['userT']['end_time'], runT['userT']['run_time'], runT['userT']['num_cores'], runT['userT']['num_nodes'], runT['userT']['num_threads'], runT['userT']['queue'], runT['userT']['user'], runT['userT']['exec_path'], moduleName, runT['userT']['cwd']) conn.query(query) run_id = conn.insert_id() self.load_objects(conn, runT['libA'], reverseMapT, runT['userT']['syshost'], "join_run_object", run_id) # loop over env. vars. for key in runT['envT']: # use the single quote pattern to protect all the single quotes in env vars. value = patSQ.sub(r"\\'", runT['envT'][key]) query = "SELECT env_id FROM xalt_env_name WHERE env_name='%s'" % key conn.query(query) result = conn.store_result() if (result.num_rows() > 0): row = result.fetch_row() env_id = int(row[0][0]) found = True else: query = "INSERT INTO xalt_env_name VALUES(NULL, '%s')" % key conn.query(query) env_id = conn.insert_id() found = False #print("env_id: ", env_id, ", found: ",found) query = "INSERT INTO join_run_env VALUES (NULL, '%d', '%d', '%s')" % ( env_id, run_id, value.encode("ascii","ignore")) conn.query(query) v = XALT_Stack.pop() carp("SUBMIT_HOST",v) except Exception as e: print(XALT_Stack.contents()) print(query.encode("ascii","ignore")) print ("run_to_db(): ",e) sys.exit (1)