def lhcbdirac_outputfile_jdl(output_files): DiracScript = dirac_outputfile_jdl(output_files) DiracScript = DiracScript.replace('###OUTPUT_SE###', '###OUTPUT_SE###,replicate=\'###REPLICATE###\'') DiracScript = DiracScript.replace('outputPath', 'OutputPath').replace('outputSE', 'OutputSE') return DiracScript
def lhcbdirac_outputfile_jdl(output_files): """ Construct the setOutputFile section of the JDL for this job""" DiracScript = dirac_outputfile_jdl(output_files, False) DiracScript = DiracScript.replace('###OUTPUT_SE###', '###OUTPUT_SE###,replicate=\'###REPLICATE###\'') DiracScript = DiracScript.replace('outputPath', 'OutputPath').replace('outputSE', 'OutputSE') return DiracScript
def lhcbdirac_outputfile_jdl(output_files): DiracScript = dirac_outputfile_jdl(output_files) DiracScript = DiracScript.replace( '###OUTPUT_SE###', '###OUTPUT_SE###,replicate=\'###REPLICATE###\'') DiracScript = DiracScript.replace('outputPath', 'OutputPath').replace( 'outputSE', 'OutputSE') return DiracScript
def lhcbdirac_outputfile_jdl(output_files): """ Construct the setOutputFile section of the JDL for this job""" DiracScript = dirac_outputfile_jdl(output_files, False) DiracScript = DiracScript.replace( '###OUTPUT_SE###', '###OUTPUT_SE###,replicate=\'###REPLICATE###\'') DiracScript = DiracScript.replace('outputPath', 'OutputPath').replace( 'outputSE', 'OutputSE') return DiracScript
def lhcbdirac_outputfile_jdl(output_files): """ Construct the setOutputFile section of the JDL for this job Args: output_files (list): A list of DiracFile objects to have their output injected into the dirac submit script """ DiracScript = dirac_outputfile_jdl(output_files, False) DiracScript = DiracScript.replace( '###OUTPUT_SE###', '###OUTPUT_SE###,replicate=\'###REPLICATE###\'') DiracScript = DiracScript.replace('outputPath', 'OutputPath').replace( 'outputSE', 'OutputSE') return DiracScript
def prepare(self, app, appsubconfig, appmasterconfig, jobmasterconfig): """ This function prepares the application of the actual job being submitted, master or not Args: app (IApplication): This is the application actually being submitted belonging to the master or sub job being configured appsubconfig (tuple): This is used to prepare the inputsandbox according to the configuration for each subjob if it varies appmasterconfig (tuple): This is also used to prepare the inputsandbox but contains the config of the app for the master job jobmasterconfig (StandardJobConfig): This is the configuration of the master job which may or may not be the same job as owning the app """ # Construct some common objects used in job submission here inputsandbox, outputsandbox = sandbox_prepare(app, appsubconfig, appmasterconfig, jobmasterconfig) input_data, parametricinput_data = dirac_inputdata(app, hasOtherInputData=True) job = app.getJobObject() # Construct the im3shape-script which is used by this job. i.e. the script and full command line to be used in this job exe_script_name = "im3shape-script.py" output_filename = os.path.basename(job.inputdata[0].lfn) + "." + str(app.rank) + "." + str(app.size) im3shape_args = " ".join( [ os.path.basename(job.inputdata[0].lfn), os.path.basename(app.ini_location.namePattern), # input.fz, config.ini app.catalog, output_filename, # catalog, output str(app.rank), str(app.size), ] ) full_cmd = app.exe_name + " " + im3shape_args outputfiles = [this_file for this_file in job.outputfiles if isinstance(this_file, DiracFile)] inputsandbox.append( FileBuffer( name=exe_script_name, contents=script_generator( Im3Shape_script_template(), ## ARGS for app from job.app RUN_DIR=app.run_dir, BLACKLIST=os.path.basename(app.blacklist.namePattern), COMMAND=full_cmd, ## Stuff for Ganga OUTPUTFILES=repr([this_file.namePattern for this_file in job.outputfiles]), OUTPUTFILESINJECTEDCODE=getWNCodeForOutputPostprocessing(job, " "), ), executable=True, ) ) # TODO once there is a common, IApplication.getMeFilesForThisApp function replace this list with a getter ad it shouldn't really be hard-coded app_file_list = [app.im3_location, app.ini_location, app.blacklist] app_file_list = [this_file for this_file in app_file_list if isinstance(this_file, DiracFile)] job.inputfiles.extend(app_file_list) # Slightly mis-using this here but it would be nice to have these files # job.inputfiles.extend(job.inputdata) # NOTE special case for replicas: replicate string must be empty for no # replication dirac_script = script_generator( diracAPI_script_template(), DIRAC_IMPORT="from DIRAC.Interfaces.API.Dirac import Dirac", DIRAC_JOB_IMPORT="from DIRAC.Interfaces.API.Job import Job", DIRAC_OBJECT="Dirac()", JOB_OBJECT="Job()", NAME=mangle_job_name(app), EXE=exe_script_name, EXE_ARG_STR="", EXE_LOG_FILE="Ganga_Executable.log", ENVIRONMENT=None, INPUTDATA=input_data, PARAMETRIC_INPUTDATA=parametricinput_data, OUTPUT_SANDBOX=API_nullifier(outputsandbox), OUTPUTFILESSCRIPT=dirac_outputfile_jdl(outputfiles, False), OUTPUT_PATH="", # job.fqid, SETTINGS=diracAPI_script_settings(app), DIRAC_OPTS=job.backend.diracOpts, REPLICATE="True" if getConfig("DIRAC")["ReplicateOutputData"] else "", # leave the sandbox for altering later as needs # to be done in backend.submit to combine master. # Note only using 2 #s as auto-remove 3 INPUT_SANDBOX="##INPUT_SANDBOX##", ) return StandardJobConfig(dirac_script, inputbox=unique(inputsandbox), outputbox=unique(outputsandbox))
def prepare(self, app, appsubconfig, appmasterconfig, jobmasterconfig): inputsandbox, outputsandbox = sandbox_prepare(app, appsubconfig, appmasterconfig, jobmasterconfig) input_data, parametricinput_data = dirac_inputdata(app) # outputdata, outputdata_path = dirac_ouputdata(app) job = stripProxy(app).getJobObject() outputfiles = [ this_file for this_file in job.outputfiles if isType(this_file, DiracFile) ] commandline = [] commandline.append(app.exe) if isType(app.exe, File): #logger.info("app: %s" % str(app.exe.name)) #fileName = os.path.join(get_share_path(app), os.path.basename(app.exe.name)) #logger.info("EXE: %s" % str(fileName)) #inputsandbox.append(File(name=fileName)) inputsandbox.append(app.exe) commandline[0] = os.path.join('.', os.path.basename(app.exe.name)) commandline.extend([str(arg) for arg in app.args]) logger.debug('Command line: %s: ', commandline) #exe_script_path = os.path.join(job.getInputWorkspace().getPath(), "exe-script.py") exe_script_name = 'exe-script.py' logger.info("Setting Command to be: '%s'" % repr(commandline)) inputsandbox.append( FileBuffer( name=exe_script_name, contents=script_generator( exe_script_template(), #remove_unreplaced = False, # , COMMAND=repr(commandline), OUTPUTFILESINJECTEDCODE=getWNCodeForOutputPostprocessing( job, ' ')), executable=True)) contents = script_generator( exe_script_template(), COMMAND=repr(commandline), OUTPUTFILESINJECTEDCODE=getWNCodeForOutputPostprocessing( job, ' ')) #logger.info("Script is: %s" % str(contents)) from os.path import abspath, expanduser for this_file in job.inputfiles: if isinstance(this_file, LocalFile): for name in this_file.getFilenameList(): inputsandbox.append(File(abspath(expanduser(name)))) elif isinstance(this_file, DiracFile): name = this_file.lfn if isinstance(input_data, list): input_data.append(name) else: input_data = [name] dirac_outputfiles = dirac_outputfile_jdl(outputfiles, config['RequireDefaultSE']) # NOTE special case for replicas: replicate string must be empty for no # replication dirac_script = script_generator( diracAPI_script_template(), DIRAC_IMPORT='from DIRAC.Interfaces.API.Dirac import Dirac', DIRAC_JOB_IMPORT='from DIRAC.Interfaces.API.Job import Job', DIRAC_OBJECT='Dirac()', JOB_OBJECT='Job()', NAME=mangle_job_name(app), # os.path.basename(exe_script_path), EXE=exe_script_name, # ' '.join([str(arg) for arg in app.args]), EXE_ARG_STR='', EXE_LOG_FILE='Ganga_Executable.log', ENVIRONMENT=None, # app.env, INPUTDATA=input_data, PARAMETRIC_INPUTDATA=parametricinput_data, OUTPUT_SANDBOX=API_nullifier(outputsandbox), OUTPUTFILESSCRIPT=dirac_outputfiles, OUTPUT_PATH="", # job.fqid, SETTINGS=diracAPI_script_settings(app), DIRAC_OPTS=job.backend.diracOpts, REPLICATE='True' if config['ReplicateOutputData'] else '', # leave the sandbox for altering later as needs # to be done in backend.submit to combine master. # Note only using 2 #s as auto-remove 3 INPUT_SANDBOX='##INPUT_SANDBOX##') #logger.info("dirac_script: %s" % dirac_script) #logger.info("inbox: %s" % str(unique(inputsandbox))) #logger.info("outbox: %s" % str(unique(outputsandbox))) return StandardJobConfig(dirac_script, inputbox=unique(inputsandbox), outputbox=unique(outputsandbox))
def prepare(self, app, appsubconfig, appmasterconfig, jobmasterconfig): inputsandbox, outputsandbox = sandbox_prepare(app, appsubconfig, appmasterconfig, jobmasterconfig) input_data, parametricinput_data = dirac_inputdata(app) # outputdata, outputdata_path = dirac_ouputdata(app) job = stripProxy(app).getJobObject() outputfiles = [this_file for this_file in job.outputfiles if isType(this_file, DiracFile)] commandline = [] commandline.append(app.exe) if isType(app.exe, File): #logger.info("app: %s" % str(app.exe.name)) #fileName = os.path.join(get_share_path(app), os.path.basename(app.exe.name)) #logger.info("EXE: %s" % str(fileName)) #inputsandbox.append(File(name=fileName)) inputsandbox.append(app.exe) commandline[0]=os.path.join('.', os.path.basename(app.exe.name)) commandline.extend([str(arg) for arg in app.args]) logger.debug('Command line: %s: ', commandline) #exe_script_path = os.path.join(job.getInputWorkspace().getPath(), "exe-script.py") exe_script_name = 'exe-script.py' logger.info("Setting Command to be: '%s'" % repr(commandline)) inputsandbox.append(FileBuffer(name=exe_script_name, contents=script_generator(exe_script_template(), #remove_unreplaced = False, # , COMMAND=repr(commandline), OUTPUTFILESINJECTEDCODE = getWNCodeForOutputPostprocessing(job, ' ') ), executable=True)) contents=script_generator(exe_script_template(), COMMAND=repr(commandline), OUTPUTFILESINJECTEDCODE = getWNCodeForOutputPostprocessing(job, ' ') ) #logger.info("Script is: %s" % str(contents)) from os.path import abspath, expanduser for this_file in job.inputfiles: if isinstance(this_file, LocalFile): for name in this_file.getFilenameList(): inputsandbox.append(File(abspath(expanduser(name)))) elif isinstance(this_file, DiracFile): name = this_file.lfn if isinstance(input_data, list): input_data.append(name) else: input_data = [name] dirac_outputfiles = dirac_outputfile_jdl(outputfiles, config['RequireDefaultSE']) # NOTE special case for replicas: replicate string must be empty for no # replication dirac_script = script_generator(diracAPI_script_template(), DIRAC_IMPORT='from DIRAC.Interfaces.API.Dirac import Dirac', DIRAC_JOB_IMPORT='from DIRAC.Interfaces.API.Job import Job', DIRAC_OBJECT='Dirac()', JOB_OBJECT='Job()', NAME=mangle_job_name(app), # os.path.basename(exe_script_path), EXE=exe_script_name, # ' '.join([str(arg) for arg in app.args]), EXE_ARG_STR='', EXE_LOG_FILE='Ganga_Executable.log', ENVIRONMENT=None, # app.env, INPUTDATA=input_data, PARAMETRIC_INPUTDATA=parametricinput_data, OUTPUT_SANDBOX=API_nullifier(outputsandbox), OUTPUTFILESSCRIPT=dirac_outputfiles, OUTPUT_PATH="", # job.fqid, SETTINGS=diracAPI_script_settings(app), DIRAC_OPTS=job.backend.diracOpts, REPLICATE='True' if config['ReplicateOutputData'] else '', # leave the sandbox for altering later as needs # to be done in backend.submit to combine master. # Note only using 2 #s as auto-remove 3 INPUT_SANDBOX='##INPUT_SANDBOX##' ) #logger.info("dirac_script: %s" % dirac_script) #logger.info("inbox: %s" % str(unique(inputsandbox))) #logger.info("outbox: %s" % str(unique(outputsandbox))) return StandardJobConfig(dirac_script, inputbox=unique(inputsandbox), outputbox=unique(outputsandbox))
def prepare(self, app, appsubconfig, appmasterconfig, jobmasterconfig): inputsandbox, outputsandbox = sandbox_prepare( app, appsubconfig, appmasterconfig, jobmasterconfig) input_data, parametricinput_data = dirac_inputdata(app) # outputdata, outputdata_path = dirac_ouputdata(app) job = app.getJobObject() outputfiles = [this_file for this_file in job.outputfiles if isType(this_file, DiracFile)] commandline = app.exe if type(app.exe) == File: inputsandbox.append(File(name=os.path.join(get_share_path(app), os.path.basename(app.exe.name)))) commandline = os.path.basename(app.exe.name) commandline += ' ' commandline += ' '.join([str(arg) for arg in app.args]) logger.debug('Command line: %s: ', commandline) #exe_script_path = os.path.join(job.getInputWorkspace().getPath(), "exe-script.py") exe_script_name = 'exe-script.py' inputsandbox.append(FileBuffer(name=exe_script_name, contents=script_generator(exe_script_template(), #remove_unreplaced = False, # , COMMAND=commandline #OUTPUTFILESINJECTEDCODE = getWNCodeForOutputPostprocessing(job, ' ') ), executable=True)) dirac_outputfiles = dirac_outputfile_jdl(outputfiles) # NOTE special case for replicas: replicate string must be empty for no # replication dirac_script = script_generator(diracAPI_script_template(), DIRAC_IMPORT='from DIRAC.Interfaces.API.Dirac import Dirac', DIRAC_JOB_IMPORT='from DIRAC.Interfaces.API.Job import Job', DIRAC_OBJECT='Dirac()', JOB_OBJECT='Job()', NAME=mangle_job_name(app), # os.path.basename(exe_script_path), EXE=exe_script_name, # ' '.join([str(arg) for arg in app.args]), EXE_ARG_STR='', EXE_LOG_FILE='Ganga_Executable.log', ENVIRONMENT=None, # app.env, INPUTDATA=input_data, PARAMETRIC_INPUTDATA=parametricinput_data, OUTPUT_SANDBOX=API_nullifier(outputsandbox), OUTPUTFILESSCRIPT=dirac_outputfiles, OUTPUT_PATH="", # job.fqid, OUTPUT_SE=getConfig('DIRAC')['DiracOutputDataSE'], SETTINGS=diracAPI_script_settings(app), DIRAC_OPTS=job.backend.diracOpts, REPLICATE='True' if getConfig('DIRAC')['ReplicateOutputData'] else '', # leave the sandbox for altering later as needs # to be done in backend.submit to combine master. # Note only using 2 #s as auto-remove 3 INPUT_SANDBOX='##INPUT_SANDBOX##' ) return StandardJobConfig(dirac_script, inputbox=unique(inputsandbox), outputbox=unique(outputsandbox))
def prepare(self, app, appsubconfig, appmasterconfig, jobmasterconfig): """ This function prepares the application of the actual job being submitted, master or not Args: app (IApplication): This is the application actually being submitted belonging to the master or sub job being configured appsubconfig (tuple): This is used to prepare the inputsandbox according to the configuration for each subjob if it varies appmasterconfig (tuple): This is also used to prepare the inputsandbox but contains the config of the app for the master job jobmasterconfig (StandardJobConfig): This is the configuration of the master job which may or may not be the same job as owning the app """ # Construct some common objects used in job submission here inputsandbox, outputsandbox = sandbox_prepare(app, appsubconfig, appmasterconfig, jobmasterconfig) input_data, parametricinput_data = dirac_inputdata(app, hasOtherInputData=True) job = app.getJobObject() # Construct the im3shape-script which is used by this job. i.e. the script and full command line to be used in this job exe_script_name = 'im3shape-script.py' output_filename = os.path.basename(job.inputdata[0].lfn) + '.' + str(app.rank) + '.' + str(app.size) im3shape_args = ' '.join([ os.path.basename(job.inputdata[0].lfn), os.path.basename(app.ini_location.namePattern), # input.fz, config.ini app.catalog, output_filename, # catalog, output str(app.rank), str(app.size) ]) full_cmd = app.exe_name + ' ' + im3shape_args outputfiles = [this_file for this_file in job.outputfiles if isinstance(this_file, DiracFile)] inputsandbox.append(FileBuffer( name=exe_script_name, contents=script_generator(Im3Shape_script_template(), ## ARGS for app from job.app RUN_DIR = app.run_dir, BLACKLIST = os.path.basename(app.blacklist.namePattern), COMMAND = full_cmd, ## Stuff for Ganga OUTPUTFILES = repr([this_file.namePattern for this_file in job.outputfiles]), OUTPUTFILESINJECTEDCODE = getWNCodeForOutputPostprocessing(job, ' '), ), executable=True) ) # TODO once there is a common, IApplication.getMeFilesForThisApp function replace this list with a getter ad it shouldn't really be hard-coded app_file_list = [app.im3_location, app.ini_location, app.blacklist] app_file_list = [this_file for this_file in app_file_list if isinstance(this_file, DiracFile)] job.inputfiles.extend(app_file_list) # Slightly mis-using this here but it would be nice to have these files #job.inputfiles.extend(job.inputdata) # NOTE special case for replicas: replicate string must be empty for no # replication dirac_script = script_generator(diracAPI_script_template(), DIRAC_IMPORT = 'from DIRAC.Interfaces.API.Dirac import Dirac', DIRAC_JOB_IMPORT = 'from DIRAC.Interfaces.API.Job import Job', DIRAC_OBJECT = 'Dirac()', JOB_OBJECT = 'Job()', NAME = mangle_job_name(app), EXE = exe_script_name, EXE_ARG_STR = '', EXE_LOG_FILE = 'Ganga_Executable.log', ENVIRONMENT = None, INPUTDATA = input_data, PARAMETRIC_INPUTDATA = parametricinput_data, OUTPUT_SANDBOX = API_nullifier(outputsandbox), OUTPUTFILESSCRIPT = dirac_outputfile_jdl(outputfiles, False), OUTPUT_PATH = "", # job.fqid, SETTINGS = diracAPI_script_settings(app), DIRAC_OPTS = job.backend.diracOpts, REPLICATE = 'True' if getConfig('DIRAC')['ReplicateOutputData'] else '', # leave the sandbox for altering later as needs # to be done in backend.submit to combine master. # Note only using 2 #s as auto-remove 3 INPUT_SANDBOX = '##INPUT_SANDBOX##' ) return StandardJobConfig(dirac_script, inputbox=unique(inputsandbox), outputbox=unique(outputsandbox))