def master_prepare(self, app, appmasterconfig):
"""
Prepare the RTHandler for the master job so that applications to be submitted
Args:
app (GaudiExec): This application is only expected to handle GaudiExec Applications here
appmasterconfig (unknown): Output passed from the application master configuration call
"""
inputsandbox, outputsandbox = master_sandbox_prepare(
app, appmasterconfig)
if isinstance(app.jobScriptArchive, LocalFile):
app.jobScriptArchive = None
generateJobScripts(app, appendJobScripts=True)
scriptArchive = os.path.join(app.jobScriptArchive.localDir,
app.jobScriptArchive.namePattern)
inputsandbox.append(File(name=scriptArchive))
if app.getMetadata:
logger.info("Adding options to make the summary.xml")
inputsandbox.append(
FileBuffer(
'summary.py',
"\nfrom Gaudi.Configuration import *\nfrom Configurables import LHCbApp\nLHCbApp().XMLSummary='summary.xml'"
))
return StandardJobConfig(inputbox=unique(inputsandbox),
outputbox=unique(outputsandbox))
def master_prepare(self, app, appmasterconfig):
"""
Prepare the RTHandler for the master job so that applications to be submitted
Args:
app (GaudiExec): This application is only expected to handle GaudiExec Applications here
appmasterconfig (unknown): Output passed from the application master configuration call
"""
inputsandbox, outputsandbox = master_sandbox_prepare(app, appmasterconfig)
if not isinstance(app.uploadedInput, DiracFile):
generateDiracInput(app)
assert isinstance(app.uploadedInput, DiracFile), "Failed to upload needed file, aborting submit. Tried to upload to: %s\nIf your Ganga installation is not at CERN your username may be trying to create a non-existent LFN. Try setting the 'DIRAC' configuration 'DiracLFNBase' to your grid user path.\n" % DiracFile.diracLFNBase()
rep_data = app.uploadedInput.getReplicas()
assert rep_data != {}, "Failed to find a replica, aborting submit"
if isinstance(app.jobScriptArchive, (DiracFile, LocalFile)):
app.jobScriptArchive = None
generateDiracScripts(app)
assert isinstance(app.jobScriptArchive, DiracFile), "Failed to upload needed file, aborting submit"
rep_data = app.jobScriptArchive.getReplicas()
assert rep_data != {}, "Failed to find a replica, aborting submit"
return StandardJobConfig(inputbox=unique(inputsandbox), outputbox=unique(outputsandbox))
def master_prepare(self, app, appmasterconfig):
"""
Prepare the RTHandler for the master job so that applications to be submitted
Args:
app (GaudiExec): This application is only expected to handle GaudiExec Applications here
appmasterconfig (unknown): Output passed from the application master configuration call
"""
inputsandbox, outputsandbox = master_sandbox_prepare(app, appmasterconfig)
if not isinstance(app.uploadedInput, DiracFile):
generateDiracInput(app)
assert isinstance(app.uploadedInput, DiracFile), "Failed to upload needed file, aborting submit"
rep_data = app.uploadedInput.getReplicas()
assert rep_data != {}, "Failed to find a replica, aborting submit"
if isinstance(app.jobScriptArchive, (DiracFile, LocalFile)):
app.jobScriptArchive = None
generateDiracScripts(app)
assert isinstance(app.jobScriptArchive, DiracFile), "Failed to upload needed file, aborting submit"
rep_data = app.jobScriptArchive.getReplicas()
assert rep_data != {}, "Failed to find a replica, aborting submit"
return StandardJobConfig(inputbox=unique(inputsandbox), outputbox=unique(outputsandbox))
def master_prepare(self, app, appmasterconfig):
"""
Prepare the RTHandler for the master job so that applications to be submitted
Args:
app (GaudiExec): This application is only expected to handle GaudiExec Applications here
appmasterconfig (unknown): Output passed from the application master configuration call
"""
inputsandbox, outputsandbox = master_sandbox_prepare(app, appmasterconfig)
# If we are getting the metadata we need to make sure the summary.xml is added to the output sandbox if not there already.
if app.getMetadata and not 'summary.xml' in outputsandbox:
outputsandbox += ['summary.xml']
if not isinstance(app.uploadedInput, DiracFile):
generateDiracInput(app)
assert isinstance(app.uploadedInput, DiracFile), "Failed to upload needed file, aborting submit. Tried to upload to: %s\nIf your Ganga installation is not at CERN your username may be trying to create a non-existent LFN. Try setting the 'DIRAC' configuration 'DiracLFNBase' to your grid user path.\n" % DiracFile.diracLFNBase()
rep_data = app.uploadedInput.getReplicas()
assert rep_data != {}, "Failed to find a replica, aborting submit"
if isinstance(app.jobScriptArchive, (DiracFile, LocalFile)):
app.jobScriptArchive = None
generateDiracScripts(app)
assert isinstance(app.jobScriptArchive, DiracFile), "Failed to upload needed file, aborting submit"
rep_data = app.jobScriptArchive.getReplicas()
assert rep_data != {}, "Failed to find a replica, aborting submit"
return StandardJobConfig(inputbox=unique(inputsandbox), outputbox=unique(outputsandbox))
def master_prepare(self, app, appmasterconfig):
inputsandbox, outputsandbox = master_sandbox_prepare(app, appmasterconfig, ['inputsandbox'])
# add summary.xml
outputsandbox += ['summary.xml', '__parsedxmlsummary__']
return StandardJobConfig(inputbox=unique(inputsandbox),
outputbox=unique(outputsandbox))
def master_prepare(self, app, appmasterconfig):
inputsandbox, outputsandbox = master_sandbox_prepare(app, appmasterconfig, ['inputsandbox'])
# add summary.xml
outputsandbox += ['summary.xml', '__parsedxmlsummary__']
return StandardJobConfig(inputbox=unique(inputsandbox),
outputbox=unique(outputsandbox))
def master_prepare(self, app, appmasterconfig):
"""
This function prepares the application of a master job during submit. A priori we aren't doing anything with this in Im3ShapeApp but until this is understood I'd rather not remove it
Args:
app (IApplication): This is the application given in the master job
appasterconfig (tuple): This is the configuration which is to prepare the app in the master job # TODO check type and this interface
"""
inputsandbox, outputsandbox = master_sandbox_prepare(app, appmasterconfig)
return StandardJobConfig(inputbox=unique(inputsandbox), outputbox=unique(outputsandbox))
def master_prepare(self, app, appmasterconfig):
"""
Prepare the RTHandler for the master job so that applications to be submitted
Args:
app (GaudiExec): This application is only expected to handle GaudiExec Applications here
appmasterconfig (unknown): Output passed from the application master configuration call
"""
cred_req = app.getJobObject().backend.credential_requirements
check_creds(cred_req)
inputsandbox, outputsandbox = master_sandbox_prepare(app, appmasterconfig)
# If we are getting the metadata we need to make sure the summary.xml is added to the output sandbox if not there already.
if app.getMetadata and not 'summary.xml' in outputsandbox:
outputsandbox += ['summary.xml']
# Check a previously uploaded input is there in case of a job copy
if isinstance(app.uploadedInput, DiracFile):
if app.uploadedInput.getReplicas() == {}:
app.uploadedInput = None
logger.info("Previously uploaded cmake target missing from Dirac. Uploading it again.")
if not isinstance(app.uploadedInput, DiracFile):
generateDiracInput(app)
try:
assert isinstance(app.uploadedInput, DiracFile)
except AssertionError:
raise ApplicationPrepareError("Failed to upload needed file, aborting submit. Tried to upload to: %s\nIf your Ganga installation is not at CERN your username may be trying to create a non-existent LFN. Try setting the 'DIRAC' configuration 'DiracLFNBase' to your grid user path.\n" % DiracFile.diracLFNBase(cred_req))
rep_data = app.uploadedInput.getReplicas()
try:
assert rep_data != {}
except AssertionError:
raise ApplicationPrepareError("Failed to find a replica of uploaded file, aborting submit")
if isinstance(app.jobScriptArchive, (DiracFile, LocalFile)):
app.jobScriptArchive = None
generateDiracScripts(app)
try:
assert isinstance(app.jobScriptArchive, DiracFile)
except AssertionError:
raise ApplicationPrepareError("Failed to upload needed file, aborting submit")
rep_data = app.jobScriptArchive.getReplicas()
try:
assert rep_data != {}
except AssertionError:
raise ApplicationPrepareError("Failed to find a replica, aborting submit")
#Create a replica of the job and scripts files
replicateJobFile(app.jobScriptArchive)
replicateJobFile(app.uploadedInput)
return StandardJobConfig(inputbox=unique(inputsandbox), outputbox=unique(outputsandbox))
def master_prepare(self, app, appmasterconfig):
"""
This function prepares the application of a master job during submit. A priori we aren't doing anything with this in Im3ShapeApp but until this is understood I'd rather not remove it
Args:
app (IApplication): This is the application given in the master job
appasterconfig (tuple): This is the configuration which is to prepare the app in the master job # TODO check type and this interface
"""
inputsandbox, outputsandbox = master_sandbox_prepare(app, appmasterconfig)
return StandardJobConfig(inputbox=unique(inputsandbox),
outputbox=unique(outputsandbox))
def master_prepare(self, app, appmasterconfig):
inputsandbox, outputsandbox = master_sandbox_prepare(app, appmasterconfig, ["inputsandbox"])
# add summary.xml
outputsandbox += ["summary.xml", "__parsedxmlsummary__"]
logger.debug("Master Prepare LHCbGaudiDiracRunTimeHandler")
return StandardJobConfig(inputbox=unique(inputsandbox), outputbox=unique(outputsandbox))
def master_prepare(self, app, appmasterconfig):
inputsandbox, outputsandbox = master_sandbox_prepare(app, appmasterconfig)
if type(app.exe) == File:
input_dir = app.getJobObject().getInputWorkspace().getPath()
exefile = os.path.join(input_dir, os.path.basename(app.exe.name))
if not os.path.exists(exefile):
msg = 'Executable: "%s" must exist!' % str(exefile)
raise ApplicationConfigurationError(None, msg)
os.system('chmod +x %s' % exefile)
return StandardJobConfig(inputbox=unique(inputsandbox),
outputbox=unique(outputsandbox))
def master_prepare(self, app, appmasterconfig):
"""
Prepare the RTHandler for the master job so that applications to be submitted
Args:
app (GaudiRun): This application is only expected to handle GaudiRun Applications here
appmasterconfig (unknown): Output passed from the application master configuration call
"""
WarnUsers()
inputsandbox, outputsandbox = master_sandbox_prepare(app, appmasterconfig)
return StandardJobConfig(inputbox=unique(inputsandbox), outputbox=unique(outputsandbox))
def master_prepare(self, app, appmasterconfig):
inputsandbox, outputsandbox = master_sandbox_prepare(
app, appmasterconfig)
if type(app.exe) == File:
input_dir = app.getJobObject().getInputWorkspace().getPath()
exefile = os.path.join(input_dir, os.path.basename(app.exe.name))
if not os.path.exists(exefile):
msg = 'Executable: "%s" must exist!' % str(exefile)
raise ApplicationConfigurationError(None, msg)
os.system('chmod +x %s' % exefile)
return StandardJobConfig(inputbox=unique(inputsandbox),
outputbox=unique(outputsandbox))
def master_prepare(self, app, appmasterconfig):
logger.debug("Master Prepare")
inputsandbox, outputsandbox = master_sandbox_prepare(
app, appmasterconfig, ['inputsandbox'])
# add summary.xml
outputsandbox += ['summary.xml', '__parsedxmlsummary__']
logger.debug("Master Prepare LHCbGaudiDiracRunTimeHandler")
return StandardJobConfig(inputbox=unique(inputsandbox),
outputbox=unique(outputsandbox))
def master_prepare(self, app, appmasterconfig):
inputsandbox, outputsandbox = master_sandbox_prepare(
app, appmasterconfig)
if type(app.exe) == File:
exefile = os.path.join(get_share_path(app),
os.path.basename(app.exe.name))
if not os.path.exists(exefile):
msg = 'Executable must exist!'
raise ApplicationConfigurationError(None, msg)
os.system('chmod +x %s' % exefile)
return StandardJobConfig(inputbox=unique(inputsandbox),
outputbox=unique(outputsandbox))
def master_prepare(self, app, appmasterconfig):
inputsandbox, outputsandbox = master_sandbox_prepare(
app, appmasterconfig)
# check file is set OK
if not app.script.name:
msg = 'Root.script.name must be set.'
raise ApplicationConfigurationError(msg)
sharedir_scriptpath = os.path.join(get_share_path(app),
os.path.basename(app.script.name))
if not os.path.exists(sharedir_scriptpath):
msg = 'Script must exist!'
raise ApplicationConfigurationError(msg)
return StandardJobConfig(inputbox=unique(inputsandbox),
outputbox=unique(outputsandbox))
def master_prepare(self, app, appmasterconfig):
inputsandbox, outputsandbox = master_sandbox_prepare(
app, appmasterconfig)
# check file is set OK
if not app.script.name:
msg = 'Root.script.name must be set.'
raise ApplicationConfigurationError(None, msg)
sharedir_scriptpath = os.path.join(get_share_path(app),
os.path.basename(app.script.name))
if not os.path.exists(sharedir_scriptpath):
msg = 'Script must exist!'
raise ApplicationConfigurationError(None, msg)
return StandardJobConfig(inputbox=unique(inputsandbox),
outputbox=unique(outputsandbox))
def master_prepare(self, app, appmasterconfig):
"""
Prepare the RTHandler for the master job so that applications to be submitted
Args:
app (GaudiExec): This application is only expected to handle GaudiExec Applications here
appmasterconfig (unknown): Output passed from the application master configuration call
"""
inputsandbox, outputsandbox = master_sandbox_prepare(app, appmasterconfig)
if isinstance(app.jobScriptArchive, LocalFile):
app.jobScriptArchive = None
generateJobScripts(app, appendJobScripts=True)
scriptArchive = os.path.join(app.jobScriptArchive.localDir, app.jobScriptArchive.namePattern)
inputsandbox.append(File(name=scriptArchive))
return StandardJobConfig(inputbox=unique(inputsandbox), outputbox=unique(outputsandbox))
def master_prepare(self, app, appmasterconfig):
"""
Prepare the RTHandler for the master job so that applications to be submitted
Args:
app (GaudiExec): This application is only expected to handle GaudiExec Applications here
appmasterconfig (unknown): Output passed from the application master configuration call
"""
inputsandbox, outputsandbox = master_sandbox_prepare(app, appmasterconfig)
if isinstance(app.jobScriptArchive, LocalFile):
app.jobScriptArchive = None
generateJobScripts(app, appendJobScripts=True)
scriptArchive = os.path.join(app.jobScriptArchive.localDir, app.jobScriptArchive.namePattern)
inputsandbox.append(File(name=scriptArchive))
return StandardJobConfig(inputbox=unique(inputsandbox), outputbox=unique(outputsandbox))
def master_prepare(self, app, appmasterconfig):
"""
Prepare the RTHandler for the master job so that applications to be submitted
Args:
app (GaudiExec): This application is only expected to handle GaudiExec Applications here
appmasterconfig (unknown): Output passed from the application master configuration call
"""
inputsandbox, outputsandbox = master_sandbox_prepare(app, appmasterconfig)
if isinstance(app.jobScriptArchive, LocalFile):
app.jobScriptArchive = None
generateJobScripts(app, appendJobScripts=True)
scriptArchive = os.path.join(app.jobScriptArchive.localDir, app.jobScriptArchive.namePattern)
inputsandbox.append(File(name=scriptArchive))
if app.getMetadata:
logger.info("Adding options to make the summary.xml")
inputsandbox.append(FileBuffer('summary.py', "\nfrom Gaudi.Configuration import *\nfrom Configurables import LHCbApp\nLHCbApp().XMLSummary='summary.xml'"))
return StandardJobConfig(inputbox=unique(inputsandbox), outputbox=unique(outputsandbox))
def master_prepare(self, app, appmasterconfig):
inputsandbox, outputsandbox = master_sandbox_prepare(app, appmasterconfig, ['inputsandbox'])
return StandardJobConfig(inputbox=unique(inputsandbox),
outputbox=unique(outputsandbox))
def master_prepare(self, app, appmasterconfig):
inputsandbox, outputsandbox = master_sandbox_prepare(
app, appmasterconfig, ['inputsandbox'])
return StandardJobConfig(inputbox=unique(inputsandbox),
outputbox=unique(outputsandbox))
def master_prepare(self, app, appmasterconfig):
"""
Prepare the RTHandler for the master job so that applications to be submitted
Args:
app (GaudiExec): This application is only expected to handle GaudiExec Applications here
appmasterconfig (unknown): Output passed from the application master configuration call
"""
cred_req = app.getJobObject().backend.credential_requirements
check_creds(cred_req)
inputsandbox, outputsandbox = master_sandbox_prepare(
app, appmasterconfig)
# If we are getting the metadata we need to make sure the summary.xml is added to the output sandbox if not there already.
if app.getMetadata and not 'summary.xml' in outputsandbox:
outputsandbox += ['summary.xml']
# Check a previously uploaded input is there in case of a job copy
if isinstance(app.uploadedInput, DiracFile):
if app.uploadedInput.getReplicas() == {}:
app.uploadedInput = None
logger.info(
"Previously uploaded cmake target missing from Dirac. Uploading it again."
)
if not isinstance(app.uploadedInput, DiracFile):
generateDiracInput(app)
try:
assert isinstance(app.uploadedInput, DiracFile)
except AssertionError:
raise ApplicationPrepareError(
"Failed to upload needed file, aborting submit. Tried to upload to: %s\nIf your Ganga installation is not at CERN your username may be trying to create a non-existent LFN. Try setting the 'DIRAC' configuration 'DiracLFNBase' to your grid user path.\n"
% DiracFile.diracLFNBase(cred_req))
rep_data = app.uploadedInput.getReplicas()
try:
assert rep_data != {}
except AssertionError:
raise ApplicationPrepareError(
"Failed to find a replica of uploaded file, aborting submit")
if isinstance(app.jobScriptArchive, (DiracFile, LocalFile)):
app.jobScriptArchive = None
generateDiracScripts(app)
try:
assert isinstance(app.jobScriptArchive, DiracFile)
except AssertionError:
raise ApplicationPrepareError(
"Failed to upload needed file, aborting submit")
rep_data = app.jobScriptArchive.getReplicas()
try:
assert rep_data != {}
except AssertionError:
raise ApplicationPrepareError(
"Failed to find a replica, aborting submit")
#Create a replica of the job and scripts files
replicateJobFile(app.jobScriptArchive)
replicateJobFile(app.uploadedInput)
return StandardJobConfig(inputbox=unique(inputsandbox),
outputbox=unique(outputsandbox))
