def lhcbdirac_outputfile_jdl(output_files):
DiracScript = dirac_outputfile_jdl(output_files)
DiracScript = DiracScript.replace('###OUTPUT_SE###', '###OUTPUT_SE###,replicate=\'###REPLICATE###\'')
DiracScript = DiracScript.replace('outputPath', 'OutputPath').replace('outputSE', 'OutputSE')
return DiracScript
def lhcbdirac_outputfile_jdl(output_files):
""" Construct the setOutputFile section of the JDL for this job"""
DiracScript = dirac_outputfile_jdl(output_files, False)
DiracScript = DiracScript.replace('###OUTPUT_SE###', '###OUTPUT_SE###,replicate=\'###REPLICATE###\'')
DiracScript = DiracScript.replace('outputPath', 'OutputPath').replace('outputSE', 'OutputSE')
return DiracScript
def lhcbdirac_outputfile_jdl(output_files):
DiracScript = dirac_outputfile_jdl(output_files)
DiracScript = DiracScript.replace(
'###OUTPUT_SE###', '###OUTPUT_SE###,replicate=\'###REPLICATE###\'')
DiracScript = DiracScript.replace('outputPath', 'OutputPath').replace(
'outputSE', 'OutputSE')
return DiracScript
def lhcbdirac_outputfile_jdl(output_files):
""" Construct the setOutputFile section of the JDL for this job"""
DiracScript = dirac_outputfile_jdl(output_files, False)
DiracScript = DiracScript.replace(
'###OUTPUT_SE###', '###OUTPUT_SE###,replicate=\'###REPLICATE###\'')
DiracScript = DiracScript.replace('outputPath', 'OutputPath').replace(
'outputSE', 'OutputSE')
return DiracScript
def lhcbdirac_outputfile_jdl(output_files):
""" Construct the setOutputFile section of the JDL for this job
Args:
output_files (list): A list of DiracFile objects to have their output injected into the dirac submit script
"""
DiracScript = dirac_outputfile_jdl(output_files, False)
DiracScript = DiracScript.replace(
'###OUTPUT_SE###', '###OUTPUT_SE###,replicate=\'###REPLICATE###\'')
DiracScript = DiracScript.replace('outputPath', 'OutputPath').replace(
'outputSE', 'OutputSE')
return DiracScript
def prepare(self, app, appsubconfig, appmasterconfig, jobmasterconfig):
"""
This function prepares the application of the actual job being submitted, master or not
Args:
app (IApplication): This is the application actually being submitted belonging to the master or sub job being configured
appsubconfig (tuple): This is used to prepare the inputsandbox according to the configuration for each subjob if it varies
appmasterconfig (tuple): This is also used to prepare the inputsandbox but contains the config of the app for the master job
jobmasterconfig (StandardJobConfig): This is the configuration of the master job which may or may not be the same job as owning the app
"""
# Construct some common objects used in job submission here
inputsandbox, outputsandbox = sandbox_prepare(app, appsubconfig, appmasterconfig, jobmasterconfig)
input_data, parametricinput_data = dirac_inputdata(app, hasOtherInputData=True)
job = app.getJobObject()
# Construct the im3shape-script which is used by this job. i.e. the script and full command line to be used in this job
exe_script_name = "im3shape-script.py"
output_filename = os.path.basename(job.inputdata[0].lfn) + "." + str(app.rank) + "." + str(app.size)
im3shape_args = " ".join(
[
os.path.basename(job.inputdata[0].lfn),
os.path.basename(app.ini_location.namePattern), # input.fz, config.ini
app.catalog,
output_filename, # catalog, output
str(app.rank),
str(app.size),
]
)
full_cmd = app.exe_name + " " + im3shape_args
outputfiles = [this_file for this_file in job.outputfiles if isinstance(this_file, DiracFile)]
inputsandbox.append(
FileBuffer(
name=exe_script_name,
contents=script_generator(
Im3Shape_script_template(),
## ARGS for app from job.app
RUN_DIR=app.run_dir,
BLACKLIST=os.path.basename(app.blacklist.namePattern),
COMMAND=full_cmd,
## Stuff for Ganga
OUTPUTFILES=repr([this_file.namePattern for this_file in job.outputfiles]),
OUTPUTFILESINJECTEDCODE=getWNCodeForOutputPostprocessing(job, " "),
),
executable=True,
)
)
# TODO once there is a common, IApplication.getMeFilesForThisApp function replace this list with a getter ad it shouldn't really be hard-coded
app_file_list = [app.im3_location, app.ini_location, app.blacklist]
app_file_list = [this_file for this_file in app_file_list if isinstance(this_file, DiracFile)]
job.inputfiles.extend(app_file_list)
# Slightly mis-using this here but it would be nice to have these files
# job.inputfiles.extend(job.inputdata)
# NOTE special case for replicas: replicate string must be empty for no
# replication
dirac_script = script_generator(
diracAPI_script_template(),
DIRAC_IMPORT="from DIRAC.Interfaces.API.Dirac import Dirac",
DIRAC_JOB_IMPORT="from DIRAC.Interfaces.API.Job import Job",
DIRAC_OBJECT="Dirac()",
JOB_OBJECT="Job()",
NAME=mangle_job_name(app),
EXE=exe_script_name,
EXE_ARG_STR="",
EXE_LOG_FILE="Ganga_Executable.log",
ENVIRONMENT=None,
INPUTDATA=input_data,
PARAMETRIC_INPUTDATA=parametricinput_data,
OUTPUT_SANDBOX=API_nullifier(outputsandbox),
OUTPUTFILESSCRIPT=dirac_outputfile_jdl(outputfiles, False),
OUTPUT_PATH="", # job.fqid,
SETTINGS=diracAPI_script_settings(app),
DIRAC_OPTS=job.backend.diracOpts,
REPLICATE="True" if getConfig("DIRAC")["ReplicateOutputData"] else "",
# leave the sandbox for altering later as needs
# to be done in backend.submit to combine master.
# Note only using 2 #s as auto-remove 3
INPUT_SANDBOX="##INPUT_SANDBOX##",
)
return StandardJobConfig(dirac_script, inputbox=unique(inputsandbox), outputbox=unique(outputsandbox))
def prepare(self, app, appsubconfig, appmasterconfig, jobmasterconfig):
inputsandbox, outputsandbox = sandbox_prepare(app, appsubconfig,
appmasterconfig,
jobmasterconfig)
input_data, parametricinput_data = dirac_inputdata(app)
# outputdata, outputdata_path = dirac_ouputdata(app)
job = stripProxy(app).getJobObject()
outputfiles = [
this_file for this_file in job.outputfiles
if isType(this_file, DiracFile)
]
commandline = []
commandline.append(app.exe)
if isType(app.exe, File):
#logger.info("app: %s" % str(app.exe.name))
#fileName = os.path.join(get_share_path(app), os.path.basename(app.exe.name))
#logger.info("EXE: %s" % str(fileName))
#inputsandbox.append(File(name=fileName))
inputsandbox.append(app.exe)
commandline[0] = os.path.join('.', os.path.basename(app.exe.name))
commandline.extend([str(arg) for arg in app.args])
logger.debug('Command line: %s: ', commandline)
#exe_script_path = os.path.join(job.getInputWorkspace().getPath(), "exe-script.py")
exe_script_name = 'exe-script.py'
logger.info("Setting Command to be: '%s'" % repr(commandline))
inputsandbox.append(
FileBuffer(
name=exe_script_name,
contents=script_generator(
exe_script_template(),
#remove_unreplaced = False,
# ,
COMMAND=repr(commandline),
OUTPUTFILESINJECTEDCODE=getWNCodeForOutputPostprocessing(
job, ' ')),
executable=True))
contents = script_generator(
exe_script_template(),
COMMAND=repr(commandline),
OUTPUTFILESINJECTEDCODE=getWNCodeForOutputPostprocessing(
job, ' '))
#logger.info("Script is: %s" % str(contents))
from os.path import abspath, expanduser
for this_file in job.inputfiles:
if isinstance(this_file, LocalFile):
for name in this_file.getFilenameList():
inputsandbox.append(File(abspath(expanduser(name))))
elif isinstance(this_file, DiracFile):
name = this_file.lfn
if isinstance(input_data, list):
input_data.append(name)
else:
input_data = [name]
dirac_outputfiles = dirac_outputfile_jdl(outputfiles,
config['RequireDefaultSE'])
# NOTE special case for replicas: replicate string must be empty for no
# replication
dirac_script = script_generator(
diracAPI_script_template(),
DIRAC_IMPORT='from DIRAC.Interfaces.API.Dirac import Dirac',
DIRAC_JOB_IMPORT='from DIRAC.Interfaces.API.Job import Job',
DIRAC_OBJECT='Dirac()',
JOB_OBJECT='Job()',
NAME=mangle_job_name(app),
# os.path.basename(exe_script_path),
EXE=exe_script_name,
# ' '.join([str(arg) for arg in app.args]),
EXE_ARG_STR='',
EXE_LOG_FILE='Ganga_Executable.log',
ENVIRONMENT=None, # app.env,
INPUTDATA=input_data,
PARAMETRIC_INPUTDATA=parametricinput_data,
OUTPUT_SANDBOX=API_nullifier(outputsandbox),
OUTPUTFILESSCRIPT=dirac_outputfiles,
OUTPUT_PATH="", # job.fqid,
SETTINGS=diracAPI_script_settings(app),
DIRAC_OPTS=job.backend.diracOpts,
REPLICATE='True' if config['ReplicateOutputData'] else '',
# leave the sandbox for altering later as needs
# to be done in backend.submit to combine master.
# Note only using 2 #s as auto-remove 3
INPUT_SANDBOX='##INPUT_SANDBOX##')
#logger.info("dirac_script: %s" % dirac_script)
#logger.info("inbox: %s" % str(unique(inputsandbox)))
#logger.info("outbox: %s" % str(unique(outputsandbox)))
return StandardJobConfig(dirac_script,
inputbox=unique(inputsandbox),
outputbox=unique(outputsandbox))
def prepare(self, app, appsubconfig, appmasterconfig, jobmasterconfig):
inputsandbox, outputsandbox = sandbox_prepare(app, appsubconfig, appmasterconfig, jobmasterconfig)
input_data, parametricinput_data = dirac_inputdata(app)
# outputdata, outputdata_path = dirac_ouputdata(app)
job = stripProxy(app).getJobObject()
outputfiles = [this_file for this_file in job.outputfiles if isType(this_file, DiracFile)]
commandline = []
commandline.append(app.exe)
if isType(app.exe, File):
#logger.info("app: %s" % str(app.exe.name))
#fileName = os.path.join(get_share_path(app), os.path.basename(app.exe.name))
#logger.info("EXE: %s" % str(fileName))
#inputsandbox.append(File(name=fileName))
inputsandbox.append(app.exe)
commandline[0]=os.path.join('.', os.path.basename(app.exe.name))
commandline.extend([str(arg) for arg in app.args])
logger.debug('Command line: %s: ', commandline)
#exe_script_path = os.path.join(job.getInputWorkspace().getPath(), "exe-script.py")
exe_script_name = 'exe-script.py'
logger.info("Setting Command to be: '%s'" % repr(commandline))
inputsandbox.append(FileBuffer(name=exe_script_name,
contents=script_generator(exe_script_template(),
#remove_unreplaced = False,
# ,
COMMAND=repr(commandline),
OUTPUTFILESINJECTEDCODE = getWNCodeForOutputPostprocessing(job, ' ')
),
executable=True))
contents=script_generator(exe_script_template(), COMMAND=repr(commandline),
OUTPUTFILESINJECTEDCODE = getWNCodeForOutputPostprocessing(job, ' ')
)
#logger.info("Script is: %s" % str(contents))
from os.path import abspath, expanduser
for this_file in job.inputfiles:
if isinstance(this_file, LocalFile):
for name in this_file.getFilenameList():
inputsandbox.append(File(abspath(expanduser(name))))
elif isinstance(this_file, DiracFile):
name = this_file.lfn
if isinstance(input_data, list):
input_data.append(name)
else:
input_data = [name]
dirac_outputfiles = dirac_outputfile_jdl(outputfiles, config['RequireDefaultSE'])
# NOTE special case for replicas: replicate string must be empty for no
# replication
dirac_script = script_generator(diracAPI_script_template(),
DIRAC_IMPORT='from DIRAC.Interfaces.API.Dirac import Dirac',
DIRAC_JOB_IMPORT='from DIRAC.Interfaces.API.Job import Job',
DIRAC_OBJECT='Dirac()',
JOB_OBJECT='Job()',
NAME=mangle_job_name(app),
# os.path.basename(exe_script_path),
EXE=exe_script_name,
# ' '.join([str(arg) for arg in app.args]),
EXE_ARG_STR='',
EXE_LOG_FILE='Ganga_Executable.log',
ENVIRONMENT=None, # app.env,
INPUTDATA=input_data,
PARAMETRIC_INPUTDATA=parametricinput_data,
OUTPUT_SANDBOX=API_nullifier(outputsandbox),
OUTPUTFILESSCRIPT=dirac_outputfiles,
OUTPUT_PATH="", # job.fqid,
SETTINGS=diracAPI_script_settings(app),
DIRAC_OPTS=job.backend.diracOpts,
REPLICATE='True' if config['ReplicateOutputData'] else '',
# leave the sandbox for altering later as needs
# to be done in backend.submit to combine master.
# Note only using 2 #s as auto-remove 3
INPUT_SANDBOX='##INPUT_SANDBOX##'
)
#logger.info("dirac_script: %s" % dirac_script)
#logger.info("inbox: %s" % str(unique(inputsandbox)))
#logger.info("outbox: %s" % str(unique(outputsandbox)))
return StandardJobConfig(dirac_script,
inputbox=unique(inputsandbox),
outputbox=unique(outputsandbox))
def prepare(self, app, appsubconfig, appmasterconfig, jobmasterconfig):
inputsandbox, outputsandbox = sandbox_prepare(
app, appsubconfig, appmasterconfig, jobmasterconfig)
input_data, parametricinput_data = dirac_inputdata(app)
# outputdata, outputdata_path = dirac_ouputdata(app)
job = app.getJobObject()
outputfiles = [this_file for this_file in job.outputfiles if isType(this_file, DiracFile)]
commandline = app.exe
if type(app.exe) == File:
inputsandbox.append(File(name=os.path.join(get_share_path(app),
os.path.basename(app.exe.name))))
commandline = os.path.basename(app.exe.name)
commandline += ' '
commandline += ' '.join([str(arg) for arg in app.args])
logger.debug('Command line: %s: ', commandline)
#exe_script_path = os.path.join(job.getInputWorkspace().getPath(), "exe-script.py")
exe_script_name = 'exe-script.py'
inputsandbox.append(FileBuffer(name=exe_script_name,
contents=script_generator(exe_script_template(),
#remove_unreplaced = False,
# ,
COMMAND=commandline
#OUTPUTFILESINJECTEDCODE = getWNCodeForOutputPostprocessing(job, ' ')
),
executable=True))
dirac_outputfiles = dirac_outputfile_jdl(outputfiles)
# NOTE special case for replicas: replicate string must be empty for no
# replication
dirac_script = script_generator(diracAPI_script_template(),
DIRAC_IMPORT='from DIRAC.Interfaces.API.Dirac import Dirac',
DIRAC_JOB_IMPORT='from DIRAC.Interfaces.API.Job import Job',
DIRAC_OBJECT='Dirac()',
JOB_OBJECT='Job()',
NAME=mangle_job_name(app),
# os.path.basename(exe_script_path),
EXE=exe_script_name,
# ' '.join([str(arg) for arg in app.args]),
EXE_ARG_STR='',
EXE_LOG_FILE='Ganga_Executable.log',
ENVIRONMENT=None, # app.env,
INPUTDATA=input_data,
PARAMETRIC_INPUTDATA=parametricinput_data,
OUTPUT_SANDBOX=API_nullifier(outputsandbox),
OUTPUTFILESSCRIPT=dirac_outputfiles,
OUTPUT_PATH="", # job.fqid,
OUTPUT_SE=getConfig('DIRAC')['DiracOutputDataSE'],
SETTINGS=diracAPI_script_settings(app),
DIRAC_OPTS=job.backend.diracOpts,
REPLICATE='True' if getConfig('DIRAC')['ReplicateOutputData'] else '',
# leave the sandbox for altering later as needs
# to be done in backend.submit to combine master.
# Note only using 2 #s as auto-remove 3
INPUT_SANDBOX='##INPUT_SANDBOX##'
)
return StandardJobConfig(dirac_script,
inputbox=unique(inputsandbox),
outputbox=unique(outputsandbox))
def prepare(self, app, appsubconfig, appmasterconfig, jobmasterconfig):
"""
This function prepares the application of the actual job being submitted, master or not
Args:
app (IApplication): This is the application actually being submitted belonging to the master or sub job being configured
appsubconfig (tuple): This is used to prepare the inputsandbox according to the configuration for each subjob if it varies
appmasterconfig (tuple): This is also used to prepare the inputsandbox but contains the config of the app for the master job
jobmasterconfig (StandardJobConfig): This is the configuration of the master job which may or may not be the same job as owning the app
"""
# Construct some common objects used in job submission here
inputsandbox, outputsandbox = sandbox_prepare(app, appsubconfig, appmasterconfig, jobmasterconfig)
input_data, parametricinput_data = dirac_inputdata(app, hasOtherInputData=True)
job = app.getJobObject()
# Construct the im3shape-script which is used by this job. i.e. the script and full command line to be used in this job
exe_script_name = 'im3shape-script.py'
output_filename = os.path.basename(job.inputdata[0].lfn) + '.' + str(app.rank) + '.' + str(app.size)
im3shape_args = ' '.join([ os.path.basename(job.inputdata[0].lfn), os.path.basename(app.ini_location.namePattern), # input.fz, config.ini
app.catalog, output_filename, # catalog, output
str(app.rank), str(app.size) ])
full_cmd = app.exe_name + ' ' + im3shape_args
outputfiles = [this_file for this_file in job.outputfiles if isinstance(this_file, DiracFile)]
inputsandbox.append(FileBuffer( name=exe_script_name,
contents=script_generator(Im3Shape_script_template(),
## ARGS for app from job.app
RUN_DIR = app.run_dir,
BLACKLIST = os.path.basename(app.blacklist.namePattern),
COMMAND = full_cmd,
## Stuff for Ganga
OUTPUTFILES = repr([this_file.namePattern for this_file in job.outputfiles]),
OUTPUTFILESINJECTEDCODE = getWNCodeForOutputPostprocessing(job, ' '),
),
executable=True)
)
# TODO once there is a common, IApplication.getMeFilesForThisApp function replace this list with a getter ad it shouldn't really be hard-coded
app_file_list = [app.im3_location, app.ini_location, app.blacklist]
app_file_list = [this_file for this_file in app_file_list if isinstance(this_file, DiracFile)]
job.inputfiles.extend(app_file_list)
# Slightly mis-using this here but it would be nice to have these files
#job.inputfiles.extend(job.inputdata)
# NOTE special case for replicas: replicate string must be empty for no
# replication
dirac_script = script_generator(diracAPI_script_template(),
DIRAC_IMPORT = 'from DIRAC.Interfaces.API.Dirac import Dirac',
DIRAC_JOB_IMPORT = 'from DIRAC.Interfaces.API.Job import Job',
DIRAC_OBJECT = 'Dirac()',
JOB_OBJECT = 'Job()',
NAME = mangle_job_name(app),
EXE = exe_script_name,
EXE_ARG_STR = '',
EXE_LOG_FILE = 'Ganga_Executable.log',
ENVIRONMENT = None,
INPUTDATA = input_data,
PARAMETRIC_INPUTDATA = parametricinput_data,
OUTPUT_SANDBOX = API_nullifier(outputsandbox),
OUTPUTFILESSCRIPT = dirac_outputfile_jdl(outputfiles, False),
OUTPUT_PATH = "", # job.fqid,
SETTINGS = diracAPI_script_settings(app),
DIRAC_OPTS = job.backend.diracOpts,
REPLICATE = 'True' if getConfig('DIRAC')['ReplicateOutputData'] else '',
# leave the sandbox for altering later as needs
# to be done in backend.submit to combine master.
# Note only using 2 #s as auto-remove 3
INPUT_SANDBOX = '##INPUT_SANDBOX##'
)
return StandardJobConfig(dirac_script,
inputbox=unique(inputsandbox),
outputbox=unique(outputsandbox))