def process_jobs(args,UserSubmissionID):
fs.DEBUG = getattr(args,fs.debug_long)
# Grabs UserSubmission and gcards as described in respective files
gcards = utils.db_grab("SELECT GcardID, gcard_text FROM Gcards WHERE UserSubmissionID = {0};".format(UserSubmissionID))
username = utils.db_grab("SELECT User FROM UserSubmissions WHERE UserSubmissionID = {0};".format(UserSubmissionID))[0][0]
scard = scard_helper.scard_class(utils.db_grab( "SELECT scard FROM UserSubmissions WHERE UserSubmissionID = {0};".format(UserSubmissionID))[0][0])
#This block picks up the scard type from arguements and throws an error if it was not an int
try:
scard_type = int(args.scard_type)
except Exception as err:
print("There was an error in recognizing scard type: ")
print(err)
exit()
#Setting sub_type to the right directory based on the scard_type
if scard_type in fs.valid_scard_types:
sub_type = "type_{0}".format(scard_type)
print("Using scard type {0} template".format(scard_type))
elif scard_type == 0:
sub_type = "type_{0}".format(type_manager.manage_type(args,scard))
else:
print("Poorly defined scard_type: {0}. Below is a list of valid scard types. Exiting".format(scard_type))
for type in fs.valid_scard_types:
print("Valid scard type: {0}".format(type))
exit()
print("sub_type is {0}".format(sub_type))
#This is creating an array of script generating functions.
script_set = [fs.runscript_file_obj,fs.condor_file_obj,fs.run_job_obj]
funcs_rs, funcs_condor,funcs_runjob = [], [], [] #initialize empty function arrays
script_set_funcs = [funcs_rs,funcs_condor,funcs_runjob]
#Please note, the ordering of this array must match the ordering of the above
scripts = ["/runscript_generators/","/clas12condor_generators/","/run_job_generators/"]
#Now we will loop through directories to import the script generation functions
for index, script_dir in enumerate(scripts):
top_dir = os.path.dirname(os.path.abspath(__file__))
script_path = os.path.abspath(top_dir + '/../submission_files/script_generators/' + sub_type + script_dir)
for function in os.listdir(script_path):
if "init" not in function:
if ".pyc" not in function:
module_name = function[:-3]
module = import_module(sub_type+'.'+script_dir[1:-1]+'.'+module_name,module_name)
func = getattr(module,module_name)
script_set_funcs[index].append(func)
if 'http' in scard.data.get('generator'):
lund_dir = lund_helper.Lund_Entry(scard.data.get('generator'))
scard.data['genExecutable'] = "Null"
scard.data['genOutput'] = "Null"
else:
lund_dir = 0
scard.data['genExecutable'] = fs.genExecutable.get(scard.data.get('generator'))
scard.data['genOutput'] = fs.genOutput.get(scard.data.get('generator'))
# Now we create job submissions for all jobs that were recognized
for gcard in gcards:
GcardID = gcard[0]
if scard.data['gcards'] == fs.gcard_default:
gcard_loc = scard.data['gcards']
elif 'http' in scard.data['gcards']:
utils.printer('Writing gcard to local file')
newfile = "gcard_{0}_UserSubmission_{1}.gcard".format(GcardID,UserSubmissionID)
gfile= fs.sub_files_path+fs.gcards_dir+newfile
if not os.path.exists(gfile):
newdir = fs.sub_files_path+fs.gcards_dir
print("newdir is {0}".format(newdir))
Popen(['mkdir','-p',newdir], stdout=PIPE)
Popen(['touch',gfile], stdout=PIPE)
with open(gfile,"w") as file: file.write(gcard[1])
gcard_loc = 'submission_files/gcards/'+newfile
else:
print('gcard not recognized as default option or online repository, please inspect scard')
exit()
file_extension = "_gcard_{0}_UserSubmission_{1}".format(GcardID,UserSubmissionID)
if fs.use_mysql:
DB_path = fs.MySQL_DB_path
else:
DB_path = fs.SQLite_DB_path
params = {'table':'Scards','UserSubmissionID':UserSubmissionID,'GcardID':GcardID,
'database_filename':DB_path+fs.DB_name,
'username':username,'gcard_loc':gcard_loc,'lund_dir':lund_dir,
'file_extension':file_extension,'scard':scard}
""" This is where we actually pass all arguements to write the scripts"""
for index, script in enumerate(script_set):
script_factory.script_factory(args, script, script_set_funcs[index], params)
print("\tSuccessfully generated submission files for UserSubmission {0} with GcardID {1}".format(UserSubmissionID,GcardID))
submission_string = 'Submission scripts generated'.format(scard.data['farm_name'])
strn = "UPDATE FarmSubmissions SET {0} = '{1}' WHERE UserSubmissionID = {2};".format('run_status',submission_string,UserSubmissionID)
utils.db_write(strn)
if args.submit:
print("\tSubmitting jobs to {0} \n".format(scard.data['farm_name']))
farm_submission_manager.farm_submission_manager(args,GcardID,file_extension,scard,params)
submission_string = 'Submitted to {0}'.format(scard.data['farm_name'])
strn = "UPDATE FarmSubmissions SET {0} = '{1}' WHERE UserSubmissionID = {2};".format('run_status',submission_string,UserSubmissionID)
utils.db_write(strn)
def process_jobs(args, UserSubmissionID, db_conn, sql):
""" Submit the job for UserSubmissionID.
Detail described in the header of this file.
I'll write a more useful comment here once the function
has been refactored.
Inputs:
-------
args - command line arguments
UserSubmissionID - (int) from UserSubmissions.UserSubmissionID
that drives the submission.
db_conn - Active database connection.
sql - Cursor object for database.
"""
logger = logging.getLogger('SubMit')
fs.DEBUG = getattr(args, fs.debug_long)
# Grabs UserSubmission and gcards as described in respective files
username = database.get_username_for_submission(UserSubmissionID, sql)
user_id = database.get_user_id(username, sql)
scard = scard_helper.scard_class(database.get_scard_text_for_submission(
UserSubmissionID, sql))
logging.debug('For UserSubmissionID = {}, user is {}'.format(
UserSubmissionID, username))
scard_type = type_manager.manage_type(args, scard)
sub_type = 'type_{}'.format(scard_type)
print("sub_type is {0}".format(sub_type))
logger.debug('Type manager has determined type is: {}'.format(
sub_type))
# Determine the number of jobs this submission
# will produce in total.
njobs = 1
if scard_type == 1:
njobs = int(scard.jobs)
elif scard_type == 2:
njobs = lund_helper.count_files(scard.generator)
# Dynamically load the script generation functions
# from the type{sub_type} folder.
script_set, script_set_funcs = script_factory.load_script_generators(sub_type)
# Setup for different scard types the proper generation options.
# If external lund files are provided, we go get them.
set_scard_generator_options(scard, scard_type)
gcard_loc = scard.configuration
file_extension = "_UserSubmission_{0}".format(UserSubmissionID)
if fs.use_mysql:
DB_path = fs.MySQL_DB_path
else:
DB_path = fs.SQLite_DB_path
params = {'table': 'Scards','UserSubmissionID': UserSubmissionID,
'username': username,'gcard_loc': gcard_loc,
'file_extension': file_extension,'scard': scard}
# This is where we actually pass all arguements to write the scripts
for index, script in enumerate(script_set):
script_factory.script_factory(args, script, script_set_funcs[index],
params, db_conn, sql)
# Update entries in database
submission_string = 'Submission scripts generated'
update_tables.update_run_status(submission_string, UserSubmissionID,
db_conn, sql)
print("Submitting jobs to {0} \n".format("OSG")) #Hardcoded for this moment as we removed farm_name from scard
farm_submission_manager.farm_submission_manager(args, UserSubmissionID,
file_extension, scard, params,
db_conn, sql)
submission_string = 'Submitted to {0}'.format("OSG")
update_tables.update_run_status(submission_string, UserSubmissionID,
db_conn, sql)
def submission_script_maker(args):
file_struct.DEBUG = getattr(args, file_struct.debug_long)
#Grabs batch and gcards as described in respective files
BatchID = grab_batchID(args)
gcards = grab_gcards(BatchID)
username = grab_username(BatchID)
funcs_rs = (startup, initialization, run_gemc, run_evio2hipo, run_cooking,
file_mover)
fname_rs = ('startup', 'initialization', 'run_gemc', 'run_evio2hipo',
'run_cooking', 'file_mover')
funcs_condor = (condor_startup, condor_1, condor_2)
fname_condor = ('condor_startup', 'condor_1', 'condor_2')
funcs_runjob = (run_job1, )
fname_runjob = ('run_job1', )
strn = "SELECT scard FROM Batches WHERE BatchID = {0};".format(BatchID)
scard_text = utils.sql3_grab(strn)[0][
0] #sql3_grab returns a list of tuples, we need the 0th element of the 0th element
scard = scard_helper.scard_class(scard_text)
scard.data['genExecutable'] = file_struct.genExecutable.get(
scard.data.get('generator'))
scard.data['genOutput'] = file_struct.genOutput.get(
scard.data.get('generator'))
for gcard in gcards:
GcardID = gcard[0]
strn = "INSERT INTO Submissions(BatchID,GcardID) VALUES ({0},{1});".format(
BatchID, GcardID)
utils.sql3_exec(strn)
strn = "UPDATE Submissions SET submission_pool = '{0}' WHERE GcardID = '{1}';".format(
scard.data['farm_name'], GcardID)
utils.sql3_exec(strn)
strn = "UPDATE Submissions SET run_status = 'not yet in pool' WHERE GcardID = '{0}';".format(
GcardID)
utils.sql3_exec(strn)
if scard.data['gcards'] == file_struct.gcard_default:
gcard_loc = scard.data['gcards']
elif 'https://' in scard.data['gcards']:
utils.printer('Writing gcard to local file')
newfile = "gcard_{0}_batch_{1}.gcard".format(GcardID, BatchID)
gfile = file_struct.sub_files_path + file_struct.gcards_dir + newfile
with open(gfile, "w") as file:
file.write(gcard[1])
gcard_loc = 'submission_files/gcards/' + newfile
else:
print(
'gcard not recognized as default option or online repository, please inspect scard'
)
exit()
file_extension = "_gcard_{0}_batch_{1}".format(GcardID, BatchID)
params = {
'table': 'Scards',
'BatchID': BatchID,
'GcardID': GcardID,
'gfile': 'gfile',
'username': username[0][0],
'gcard_loc': gcard_loc
}
script_factory(file_struct.runscript_file_obj, funcs_rs, fname_rs,
scard, params, file_extension)
script_factory(file_struct.condor_file_obj, funcs_condor, fname_condor,
scard, params, file_extension)
script_factory(file_struct.run_job_obj, funcs_runjob, fname_runjob,
scard, params, file_extension)
print(
"\tSuccessfully generated submission files for Batch {0} with GcardID {1}\n"
.format(BatchID, GcardID))
if args.submit:
print("\tSubmitting jobs to HTCondor \n")
htcondor_submit.htcondor_submit(args, GcardID, file_extension)