def prepare(self, app, appsubconfig, appmasterconfig, jobmasterconfig):
inputsandbox, outputsandbox = sandbox_prepare(
app, appsubconfig, appmasterconfig, jobmasterconfig)
input_data, parametricinput_data = dirac_inputdata(app)
job = app.getJobObject()
outputfiles = [this_file.namePattern for this_file in job.outputfiles if isinstance(this_file, DiracFile)]
gaudi_script_path = os.path.join(
job.getInputWorkspace().getPath(), "gaudi-script.py")
script_generator(gaudi_script_template(),
#remove_unreplaced = False,
outputfile_path=gaudi_script_path,
PLATFORM=app.platform,
COMMAND='gaudirun.py' # ,
#OUTPUTFILESINJECTEDCODE = getWNCodeForOutputPostprocessing(job, ' ')
)
dirac_script = script_generator(diracAPI_script_template(),
DIRAC_IMPORT='from DIRAC.Interfaces.API.Dirac import Dirac',
DIRAC_JOB_IMPORT='from DIRAC.Interfaces.API.Job import Job',
DIRAC_OBJECT='Dirac()',
JOB_OBJECT='Job()',
NAME=mangle_job_name(app),
EXE=gaudi_script_path,
EXE_ARG_STR=' '.join(
[str(arg) for arg in app.args]),
EXE_LOG_FILE='Ganga_%s_%s.log' % (
app.appname, app.version),
INPUTDATA=input_data,
PARAMETRIC_INPUTDATA=parametricinput_data,
OUTPUT_SANDBOX=outputsandbox,
OUTPUTDATA=list(outputfiles),
OUTPUT_PATH="", # job.fqid,
OUTPUT_SE=getConfig(
'DIRAC')['DiracOutputDataSE'],
SETTINGS=diracAPI_script_settings(app),
DIRAC_OPTS=job.backend.diracOpts,
PLATFORM=app.platform,
# leave the sandbox for altering later as needs
# to be done in backend.submit to combine master.
# Note only using 2 #s as auto-remove 3
INPUT_SANDBOX='##INPUT_SANDBOX##'
)
return StandardJobConfig(dirac_script,
inputbox=unique(inputsandbox),
outputbox=unique(outputsandbox))
def lhcbdiracAPI_script_template():
""" Create a template LHCbDirac LHCbJob by taking the Dirac Job and making some modifications to teh template"""
DiracScript = diracAPI_script_template()
DiracLHCb_Options = """
j.setApplicationScript('###APP_NAME###', '###APP_VERSION###', '###APP_SCRIPT###', logFile='###APP_LOG_FILE###', systemConfig='###PLATFORM###')
j.setAncestorDepth(###ANCESTOR_DEPTH###)
"""
DiracScript = DiracScript.replace('\'###EXE_LOG_FILE###\'', '\'###EXE_LOG_FILE###\', systemConfig=\'###PLATFORM###\'')
DiracScript = DiracScript.replace('j.setPlatform( \'ANY\' )', 'j.setDIRACPlatform()')
setName_str = 'j.setName(\'###NAME###\')'
DiracScript = DiracScript.replace(setName_str, "%s\n%s" % (setName_str, DiracLHCb_Options))
return DiracScript
def lhcbdiracAPI_script_template():
""" Create a template LHCbDirac LHCbJob by taking the Dirac Job and making some modifications to teh template"""
DiracScript = diracAPI_script_template()
DiracLHCb_Options = """
j.setApplicationScript('###APP_NAME###', '###APP_VERSION###', '###APP_SCRIPT###', logFile='###APP_LOG_FILE###', systemConfig='###PLATFORM###')
j.setAncestorDepth(###ANCESTOR_DEPTH###)
"""
DiracScript = DiracScript.replace(
'\'###EXE_LOG_FILE###\'',
'\'###EXE_LOG_FILE###\', systemConfig=\'###PLATFORM###\'')
DiracScript = DiracScript.replace('j.setPlatform( \'ANY\' )',
'j.setDIRACPlatform()')
setName_str = 'j.setName(\'###NAME###\')'
DiracScript = DiracScript.replace(
setName_str, "%s\n%s" % (setName_str, DiracLHCb_Options))
return DiracScript
def lhcbdiracAPI_script_template():
DiracScript = diracAPI_script_template()
## Old Probably deprecated additionsl options
# j.setRootMacro('###ROOT_VERSION###', '###ROOT_MACRO###', ###ROOT_ARGS###, '###ROOT_LOG_FILE###', systemConfig='###PLATFORM###')
# j.setRootPythonScript('###ROOTPY_VERSION###', '###ROOTPY_SCRIPT###', ###ROOTPY_ARGS###, '###ROOTPY_LOG_FILE###', systemConfig='###PLATFORM###')
DiracLHCb_Options = """
j.setApplicationScript('###APP_NAME###', '###APP_VERSION###', '###APP_SCRIPT###', logFile='###APP_LOG_FILE###', systemConfig='###PLATFORM###')
j.setAncestorDepth(###ANCESTOR_DEPTH###)
"""
DiracScript = DiracScript.replace('\'###EXE_LOG_FILE###\'', '\'###EXE_LOG_FILE###\', systemConfig=\'###PLATFORM###\'')
DiracScript = DiracScript.replace('j.setPlatform( \'ANY\' )', 'j.setDIRACPlatform()')
setName_str = 'j.setName(\'###NAME###\')'
DiracScript = DiracScript.replace(setName_str, "%s\n%s" % (setName_str, DiracLHCb_Options))
return DiracScript
def lhcbdiracAPI_script_template():
DiracScript = diracAPI_script_template()
## Old Probably deprecated additionsl options
# j.setRootMacro('###ROOT_VERSION###', '###ROOT_MACRO###', ###ROOT_ARGS###, '###ROOT_LOG_FILE###', systemConfig='###PLATFORM###')
# j.setRootPythonScript('###ROOTPY_VERSION###', '###ROOTPY_SCRIPT###', ###ROOTPY_ARGS###, '###ROOTPY_LOG_FILE###', systemConfig='###PLATFORM###')
DiracLHCb_Options = """
j.setApplicationScript('###APP_NAME###', '###APP_VERSION###', '###APP_SCRIPT###', logFile='###APP_LOG_FILE###', systemConfig='###PLATFORM###')
j.setAncestorDepth(###ANCESTOR_DEPTH###)
"""
DiracScript = DiracScript.replace(
'\'###EXE_LOG_FILE###\'',
'\'###EXE_LOG_FILE###\', systemConfig=\'###PLATFORM###\'')
DiracScript = DiracScript.replace('j.setPlatform( \'ANY\' )',
'j.setDIRACPlatform()')
setName_str = 'j.setName(\'###NAME###\')'
DiracScript = DiracScript.replace(
setName_str, "%s\n%s" % (setName_str, DiracLHCb_Options))
return DiracScript
def prepare(self, app, appsubconfig, appmasterconfig, jobmasterconfig):
"""
This function prepares the application of the actual job being submitted, master or not
Args:
app (IApplication): This is the application actually being submitted belonging to the master or sub job being configured
appsubconfig (tuple): This is used to prepare the inputsandbox according to the configuration for each subjob if it varies
appmasterconfig (tuple): This is also used to prepare the inputsandbox but contains the config of the app for the master job
jobmasterconfig (StandardJobConfig): This is the configuration of the master job which may or may not be the same job as owning the app
"""
# Construct some common objects used in job submission here
inputsandbox, outputsandbox = sandbox_prepare(app, appsubconfig, appmasterconfig, jobmasterconfig)
input_data, parametricinput_data = dirac_inputdata(app, hasOtherInputData=True)
job = app.getJobObject()
# Construct the im3shape-script which is used by this job. i.e. the script and full command line to be used in this job
exe_script_name = "im3shape-script.py"
output_filename = os.path.basename(job.inputdata[0].lfn) + "." + str(app.rank) + "." + str(app.size)
im3shape_args = " ".join(
[
os.path.basename(job.inputdata[0].lfn),
os.path.basename(app.ini_location.namePattern), # input.fz, config.ini
app.catalog,
output_filename, # catalog, output
str(app.rank),
str(app.size),
]
)
full_cmd = app.exe_name + " " + im3shape_args
outputfiles = [this_file for this_file in job.outputfiles if isinstance(this_file, DiracFile)]
inputsandbox.append(
FileBuffer(
name=exe_script_name,
contents=script_generator(
Im3Shape_script_template(),
## ARGS for app from job.app
RUN_DIR=app.run_dir,
BLACKLIST=os.path.basename(app.blacklist.namePattern),
COMMAND=full_cmd,
## Stuff for Ganga
OUTPUTFILES=repr([this_file.namePattern for this_file in job.outputfiles]),
OUTPUTFILESINJECTEDCODE=getWNCodeForOutputPostprocessing(job, " "),
),
executable=True,
)
)
# TODO once there is a common, IApplication.getMeFilesForThisApp function replace this list with a getter ad it shouldn't really be hard-coded
app_file_list = [app.im3_location, app.ini_location, app.blacklist]
app_file_list = [this_file for this_file in app_file_list if isinstance(this_file, DiracFile)]
job.inputfiles.extend(app_file_list)
# Slightly mis-using this here but it would be nice to have these files
# job.inputfiles.extend(job.inputdata)
# NOTE special case for replicas: replicate string must be empty for no
# replication
dirac_script = script_generator(
diracAPI_script_template(),
DIRAC_IMPORT="from DIRAC.Interfaces.API.Dirac import Dirac",
DIRAC_JOB_IMPORT="from DIRAC.Interfaces.API.Job import Job",
DIRAC_OBJECT="Dirac()",
JOB_OBJECT="Job()",
NAME=mangle_job_name(app),
EXE=exe_script_name,
EXE_ARG_STR="",
EXE_LOG_FILE="Ganga_Executable.log",
ENVIRONMENT=None,
INPUTDATA=input_data,
PARAMETRIC_INPUTDATA=parametricinput_data,
OUTPUT_SANDBOX=API_nullifier(outputsandbox),
OUTPUTFILESSCRIPT=dirac_outputfile_jdl(outputfiles, False),
OUTPUT_PATH="", # job.fqid,
SETTINGS=diracAPI_script_settings(app),
DIRAC_OPTS=job.backend.diracOpts,
REPLICATE="True" if getConfig("DIRAC")["ReplicateOutputData"] else "",
# leave the sandbox for altering later as needs
# to be done in backend.submit to combine master.
# Note only using 2 #s as auto-remove 3
INPUT_SANDBOX="##INPUT_SANDBOX##",
)
return StandardJobConfig(dirac_script, inputbox=unique(inputsandbox), outputbox=unique(outputsandbox))
def prepare(self, app, appsubconfig, appmasterconfig, jobmasterconfig):
inputsandbox, outputsandbox = sandbox_prepare(app, appsubconfig,
appmasterconfig,
jobmasterconfig)
input_data, parametricinput_data = dirac_inputdata(app)
# outputdata, outputdata_path = dirac_ouputdata(app)
job = stripProxy(app).getJobObject()
outputfiles = [
this_file for this_file in job.outputfiles
if isType(this_file, DiracFile)
]
commandline = []
commandline.append(app.exe)
if isType(app.exe, File):
#logger.info("app: %s" % str(app.exe.name))
#fileName = os.path.join(get_share_path(app), os.path.basename(app.exe.name))
#logger.info("EXE: %s" % str(fileName))
#inputsandbox.append(File(name=fileName))
inputsandbox.append(app.exe)
commandline[0] = os.path.join('.', os.path.basename(app.exe.name))
commandline.extend([str(arg) for arg in app.args])
logger.debug('Command line: %s: ', commandline)
#exe_script_path = os.path.join(job.getInputWorkspace().getPath(), "exe-script.py")
exe_script_name = 'exe-script.py'
logger.info("Setting Command to be: '%s'" % repr(commandline))
inputsandbox.append(
FileBuffer(
name=exe_script_name,
contents=script_generator(
exe_script_template(),
#remove_unreplaced = False,
# ,
COMMAND=repr(commandline),
OUTPUTFILESINJECTEDCODE=getWNCodeForOutputPostprocessing(
job, ' ')),
executable=True))
contents = script_generator(
exe_script_template(),
COMMAND=repr(commandline),
OUTPUTFILESINJECTEDCODE=getWNCodeForOutputPostprocessing(
job, ' '))
#logger.info("Script is: %s" % str(contents))
from os.path import abspath, expanduser
for this_file in job.inputfiles:
if isinstance(this_file, LocalFile):
for name in this_file.getFilenameList():
inputsandbox.append(File(abspath(expanduser(name))))
elif isinstance(this_file, DiracFile):
name = this_file.lfn
if isinstance(input_data, list):
input_data.append(name)
else:
input_data = [name]
dirac_outputfiles = dirac_outputfile_jdl(outputfiles,
config['RequireDefaultSE'])
# NOTE special case for replicas: replicate string must be empty for no
# replication
dirac_script = script_generator(
diracAPI_script_template(),
DIRAC_IMPORT='from DIRAC.Interfaces.API.Dirac import Dirac',
DIRAC_JOB_IMPORT='from DIRAC.Interfaces.API.Job import Job',
DIRAC_OBJECT='Dirac()',
JOB_OBJECT='Job()',
NAME=mangle_job_name(app),
# os.path.basename(exe_script_path),
EXE=exe_script_name,
# ' '.join([str(arg) for arg in app.args]),
EXE_ARG_STR='',
EXE_LOG_FILE='Ganga_Executable.log',
ENVIRONMENT=None, # app.env,
INPUTDATA=input_data,
PARAMETRIC_INPUTDATA=parametricinput_data,
OUTPUT_SANDBOX=API_nullifier(outputsandbox),
OUTPUTFILESSCRIPT=dirac_outputfiles,
OUTPUT_PATH="", # job.fqid,
SETTINGS=diracAPI_script_settings(app),
DIRAC_OPTS=job.backend.diracOpts,
REPLICATE='True' if config['ReplicateOutputData'] else '',
# leave the sandbox for altering later as needs
# to be done in backend.submit to combine master.
# Note only using 2 #s as auto-remove 3
INPUT_SANDBOX='##INPUT_SANDBOX##')
#logger.info("dirac_script: %s" % dirac_script)
#logger.info("inbox: %s" % str(unique(inputsandbox)))
#logger.info("outbox: %s" % str(unique(outputsandbox)))
return StandardJobConfig(dirac_script,
inputbox=unique(inputsandbox),
outputbox=unique(outputsandbox))
def prepare(self, app, appsubconfig, appmasterconfig, jobmasterconfig):
inputsandbox, outputsandbox = sandbox_prepare(app, appsubconfig, appmasterconfig, jobmasterconfig)
input_data, parametricinput_data = dirac_inputdata(app)
# outputdata, outputdata_path = dirac_ouputdata(app)
job = stripProxy(app).getJobObject()
outputfiles = [this_file for this_file in job.outputfiles if isType(this_file, DiracFile)]
commandline = []
commandline.append(app.exe)
if isType(app.exe, File):
#logger.info("app: %s" % str(app.exe.name))
#fileName = os.path.join(get_share_path(app), os.path.basename(app.exe.name))
#logger.info("EXE: %s" % str(fileName))
#inputsandbox.append(File(name=fileName))
inputsandbox.append(app.exe)
commandline[0]=os.path.join('.', os.path.basename(app.exe.name))
commandline.extend([str(arg) for arg in app.args])
logger.debug('Command line: %s: ', commandline)
#exe_script_path = os.path.join(job.getInputWorkspace().getPath(), "exe-script.py")
exe_script_name = 'exe-script.py'
logger.info("Setting Command to be: '%s'" % repr(commandline))
inputsandbox.append(FileBuffer(name=exe_script_name,
contents=script_generator(exe_script_template(),
#remove_unreplaced = False,
# ,
COMMAND=repr(commandline),
OUTPUTFILESINJECTEDCODE = getWNCodeForOutputPostprocessing(job, ' ')
),
executable=True))
contents=script_generator(exe_script_template(), COMMAND=repr(commandline),
OUTPUTFILESINJECTEDCODE = getWNCodeForOutputPostprocessing(job, ' ')
)
#logger.info("Script is: %s" % str(contents))
from os.path import abspath, expanduser
for this_file in job.inputfiles:
if isinstance(this_file, LocalFile):
for name in this_file.getFilenameList():
inputsandbox.append(File(abspath(expanduser(name))))
elif isinstance(this_file, DiracFile):
name = this_file.lfn
if isinstance(input_data, list):
input_data.append(name)
else:
input_data = [name]
dirac_outputfiles = dirac_outputfile_jdl(outputfiles, config['RequireDefaultSE'])
# NOTE special case for replicas: replicate string must be empty for no
# replication
dirac_script = script_generator(diracAPI_script_template(),
DIRAC_IMPORT='from DIRAC.Interfaces.API.Dirac import Dirac',
DIRAC_JOB_IMPORT='from DIRAC.Interfaces.API.Job import Job',
DIRAC_OBJECT='Dirac()',
JOB_OBJECT='Job()',
NAME=mangle_job_name(app),
# os.path.basename(exe_script_path),
EXE=exe_script_name,
# ' '.join([str(arg) for arg in app.args]),
EXE_ARG_STR='',
EXE_LOG_FILE='Ganga_Executable.log',
ENVIRONMENT=None, # app.env,
INPUTDATA=input_data,
PARAMETRIC_INPUTDATA=parametricinput_data,
OUTPUT_SANDBOX=API_nullifier(outputsandbox),
OUTPUTFILESSCRIPT=dirac_outputfiles,
OUTPUT_PATH="", # job.fqid,
SETTINGS=diracAPI_script_settings(app),
DIRAC_OPTS=job.backend.diracOpts,
REPLICATE='True' if config['ReplicateOutputData'] else '',
# leave the sandbox for altering later as needs
# to be done in backend.submit to combine master.
# Note only using 2 #s as auto-remove 3
INPUT_SANDBOX='##INPUT_SANDBOX##'
)
#logger.info("dirac_script: %s" % dirac_script)
#logger.info("inbox: %s" % str(unique(inputsandbox)))
#logger.info("outbox: %s" % str(unique(outputsandbox)))
return StandardJobConfig(dirac_script,
inputbox=unique(inputsandbox),
outputbox=unique(outputsandbox))
def prepare(self, app, appsubconfig, appmasterconfig, jobmasterconfig):
inputsandbox, outputsandbox = sandbox_prepare(
app, appsubconfig, appmasterconfig, jobmasterconfig)
input_data, parametricinput_data = dirac_inputdata(app)
# outputdata, outputdata_path = dirac_ouputdata(app)
job = app.getJobObject()
outputfiles = [this_file for this_file in job.outputfiles if isType(this_file, DiracFile)]
commandline = app.exe
if type(app.exe) == File:
inputsandbox.append(File(name=os.path.join(get_share_path(app),
os.path.basename(app.exe.name))))
commandline = os.path.basename(app.exe.name)
commandline += ' '
commandline += ' '.join([str(arg) for arg in app.args])
logger.debug('Command line: %s: ', commandline)
#exe_script_path = os.path.join(job.getInputWorkspace().getPath(), "exe-script.py")
exe_script_name = 'exe-script.py'
inputsandbox.append(FileBuffer(name=exe_script_name,
contents=script_generator(exe_script_template(),
#remove_unreplaced = False,
# ,
COMMAND=commandline
#OUTPUTFILESINJECTEDCODE = getWNCodeForOutputPostprocessing(job, ' ')
),
executable=True))
dirac_outputfiles = dirac_outputfile_jdl(outputfiles)
# NOTE special case for replicas: replicate string must be empty for no
# replication
dirac_script = script_generator(diracAPI_script_template(),
DIRAC_IMPORT='from DIRAC.Interfaces.API.Dirac import Dirac',
DIRAC_JOB_IMPORT='from DIRAC.Interfaces.API.Job import Job',
DIRAC_OBJECT='Dirac()',
JOB_OBJECT='Job()',
NAME=mangle_job_name(app),
# os.path.basename(exe_script_path),
EXE=exe_script_name,
# ' '.join([str(arg) for arg in app.args]),
EXE_ARG_STR='',
EXE_LOG_FILE='Ganga_Executable.log',
ENVIRONMENT=None, # app.env,
INPUTDATA=input_data,
PARAMETRIC_INPUTDATA=parametricinput_data,
OUTPUT_SANDBOX=API_nullifier(outputsandbox),
OUTPUTFILESSCRIPT=dirac_outputfiles,
OUTPUT_PATH="", # job.fqid,
OUTPUT_SE=getConfig('DIRAC')['DiracOutputDataSE'],
SETTINGS=diracAPI_script_settings(app),
DIRAC_OPTS=job.backend.diracOpts,
REPLICATE='True' if getConfig('DIRAC')['ReplicateOutputData'] else '',
# leave the sandbox for altering later as needs
# to be done in backend.submit to combine master.
# Note only using 2 #s as auto-remove 3
INPUT_SANDBOX='##INPUT_SANDBOX##'
)
return StandardJobConfig(dirac_script,
inputbox=unique(inputsandbox),
outputbox=unique(outputsandbox))
def prepare(self, app, appsubconfig, appmasterconfig, jobmasterconfig):
"""
This function prepares the application of the actual job being submitted, master or not
Args:
app (IApplication): This is the application actually being submitted belonging to the master or sub job being configured
appsubconfig (tuple): This is used to prepare the inputsandbox according to the configuration for each subjob if it varies
appmasterconfig (tuple): This is also used to prepare the inputsandbox but contains the config of the app for the master job
jobmasterconfig (StandardJobConfig): This is the configuration of the master job which may or may not be the same job as owning the app
"""
# Construct some common objects used in job submission here
inputsandbox, outputsandbox = sandbox_prepare(app, appsubconfig, appmasterconfig, jobmasterconfig)
input_data, parametricinput_data = dirac_inputdata(app, hasOtherInputData=True)
job = app.getJobObject()
# Construct the im3shape-script which is used by this job. i.e. the script and full command line to be used in this job
exe_script_name = 'im3shape-script.py'
output_filename = os.path.basename(job.inputdata[0].lfn) + '.' + str(app.rank) + '.' + str(app.size)
im3shape_args = ' '.join([ os.path.basename(job.inputdata[0].lfn), os.path.basename(app.ini_location.namePattern), # input.fz, config.ini
app.catalog, output_filename, # catalog, output
str(app.rank), str(app.size) ])
full_cmd = app.exe_name + ' ' + im3shape_args
outputfiles = [this_file for this_file in job.outputfiles if isinstance(this_file, DiracFile)]
inputsandbox.append(FileBuffer( name=exe_script_name,
contents=script_generator(Im3Shape_script_template(),
## ARGS for app from job.app
RUN_DIR = app.run_dir,
BLACKLIST = os.path.basename(app.blacklist.namePattern),
COMMAND = full_cmd,
## Stuff for Ganga
OUTPUTFILES = repr([this_file.namePattern for this_file in job.outputfiles]),
OUTPUTFILESINJECTEDCODE = getWNCodeForOutputPostprocessing(job, ' '),
),
executable=True)
)
# TODO once there is a common, IApplication.getMeFilesForThisApp function replace this list with a getter ad it shouldn't really be hard-coded
app_file_list = [app.im3_location, app.ini_location, app.blacklist]
app_file_list = [this_file for this_file in app_file_list if isinstance(this_file, DiracFile)]
job.inputfiles.extend(app_file_list)
# Slightly mis-using this here but it would be nice to have these files
#job.inputfiles.extend(job.inputdata)
# NOTE special case for replicas: replicate string must be empty for no
# replication
dirac_script = script_generator(diracAPI_script_template(),
DIRAC_IMPORT = 'from DIRAC.Interfaces.API.Dirac import Dirac',
DIRAC_JOB_IMPORT = 'from DIRAC.Interfaces.API.Job import Job',
DIRAC_OBJECT = 'Dirac()',
JOB_OBJECT = 'Job()',
NAME = mangle_job_name(app),
EXE = exe_script_name,
EXE_ARG_STR = '',
EXE_LOG_FILE = 'Ganga_Executable.log',
ENVIRONMENT = None,
INPUTDATA = input_data,
PARAMETRIC_INPUTDATA = parametricinput_data,
OUTPUT_SANDBOX = API_nullifier(outputsandbox),
OUTPUTFILESSCRIPT = dirac_outputfile_jdl(outputfiles, False),
OUTPUT_PATH = "", # job.fqid,
SETTINGS = diracAPI_script_settings(app),
DIRAC_OPTS = job.backend.diracOpts,
REPLICATE = 'True' if getConfig('DIRAC')['ReplicateOutputData'] else '',
# leave the sandbox for altering later as needs
# to be done in backend.submit to combine master.
# Note only using 2 #s as auto-remove 3
INPUT_SANDBOX = '##INPUT_SANDBOX##'
)
return StandardJobConfig(dirac_script,
inputbox=unique(inputsandbox),
outputbox=unique(outputsandbox))
