def master_sandbox_prepare(app, appmasterconfig, sharedir_roots=None):
if sharedir_roots is None:
sharedir_roots = ['']
logger.debug("RTUTils master_sandbox_prepare")
# catch errors from not preparing properly
if not hasattr(stripProxy(app), 'is_prepared') or app.is_prepared is None:
logger.warning('Application is not prepared properly')
if hasattr(stripProxy(app), 'is_prepared'):
logger.warning("app.is_prepared: %s" % str(app.is_prepared))
import traceback
traceback.print_stack()
raise GangaException(None, 'Application not prepared properly')
# Note EITHER the master inputsandbox OR the job.inputsandbox is added to
# the subjob inputsandbox depending if the jobmasterconfig object is present
# or not... Therefore combine the job.inputsandbox with appmasterconfig.
job = stripProxy(app).getJobObject()
# user added items from the interactive GPI
from Ganga.Utility.Config import getConfig
if not getConfig('Output')['ForbidLegacyInput']:
inputsandbox = job.inputsandbox[:]
else:
if len(job.inputsandbox) > 0:
from Ganga.GPIDev.Lib.Job import JobError
raise JobError(
"InputFiles have been requested but there are objects in the inputSandBox... Aborting Job Prepare!"
)
inputsandbox = []
for filepattern in getInputFilesPatterns(job)[0]:
inputsandbox.append(File(filepattern))
if len(inputsandbox) > 100:
logger.warning(
'InputSandbox exceeds maximum size (100) supported by the Dirac backend'
)
raise GangaException(None, 'InputSandbox exceed maximum size')
outputsandbox = getOutputSandboxPatterns(job) # job.outputsandbox[:]
# inputsandbox files stored in share_dir from prepare method
sharedir_handler(app, sharedir_roots, inputsandbox)
# Here add any sandbox files/data coming from the appmasterconfig
# from master_configure. Catch the case where None is passed (as in tests)
if appmasterconfig:
inputsandbox += appmasterconfig.getSandboxFiles()
outputsandbox += appmasterconfig.getOutputSandboxFiles()
return unique(inputsandbox), unique(outputsandbox)
def master_prepare(self, masterjobconfig):
""" Prepare the master job (shared sandbox files). This method is/should be called by master_submit() exactly once.
The input sandbox is created according to self._packed_input_sandbox flag (a class attribute)
"""
from Ganga.GPIDev.Lib.File.OutputFileManager import getInputFilesPatterns
from Ganga.GPIDev.Lib.File.File import File, ShareDir
job = self.getJobObject()
create_sandbox = job.createInputSandbox
if self._packed_input_sandbox:
create_sandbox = job.createPackedInputSandbox
if masterjobconfig:
if hasattr(job.application, "is_prepared") and isType(job.application.is_prepared, ShareDir):
sharedir_pred = lambda f: f.name.find(job.application.is_prepared.name) > -1
sharedir_files = itertools.ifilter(sharedir_pred, masterjobconfig.getSandboxFiles())
nonsharedir_files = itertools.ifilterfalse(sharedir_pred, masterjobconfig.getSandboxFiles())
# ATLAS use bool to bypass the prepare mechanism and some ATLAS
# apps have no is_prepared
else:
sharedir_files = []
nonsharedir_files = masterjobconfig.getSandboxFiles()
inputsandbox = create_sandbox(nonsharedir_files, master=True)
inputsandbox.extend(itertools.imap(lambda f: f.name, sharedir_files))
return inputsandbox
tmpDir = None
files = []
if len(job.inputfiles) > 0 or (
len(job.subjobs) == 0
and job.inputdata
and isType(job.inputdata, GangaDataset)
and job.inputdata.treat_as_inputfiles
):
(fileNames, tmpDir) = getInputFilesPatterns(job)
files = itertools.imap(lambda f: File(f), fileNames)
else:
# RTHandler is not required to produce masterjobconfig, in that
# case just use the inputsandbox
files = job.inputsandbox
result = create_sandbox(files, master=True)
if tmpDir is not None:
import shutil
# remove temp dir
if os.path.exists(tmpDir):
shutil.rmtree(tmpDir)
return result
def master_prepare(self, masterjobconfig):
""" Prepare the master job (shared sandbox files). This method is/should be called by master_submit() exactly once.
The input sandbox is created according to self._packed_input_sandbox flag (a class attribute)
"""
from Ganga.GPIDev.Lib.File.OutputFileManager import getInputFilesPatterns
from Ganga.GPIDev.Lib.File.File import File, ShareDir
job = self.getJobObject()
create_sandbox = job.createInputSandbox
if self._packed_input_sandbox:
create_sandbox = job.createPackedInputSandbox
if masterjobconfig:
if hasattr(job.application, 'is_prepared') and isType(
job.application.is_prepared, ShareDir):
sharedir_pred = lambda f: f.name.find(job.application.
is_prepared.name) > -1
sharedir_files = itertools.ifilter(
sharedir_pred, masterjobconfig.getSandboxFiles())
nonsharedir_files = itertools.ifilterfalse(
sharedir_pred, masterjobconfig.getSandboxFiles())
# ATLAS use bool to bypass the prepare mechanism and some ATLAS
# apps have no is_prepared
else:
sharedir_files = []
nonsharedir_files = masterjobconfig.getSandboxFiles()
inputsandbox = create_sandbox(nonsharedir_files, master=True)
inputsandbox.extend(
itertools.imap(lambda f: f.name, sharedir_files))
return inputsandbox
tmpDir = None
files = []
if len(job.inputfiles) > 0 or (len(job.subjobs) == 0 and job.inputdata and\
isType(job.inputdata, GangaDataset) and job.inputdata.treat_as_inputfiles):
(fileNames, tmpDir) = getInputFilesPatterns(job)
files = itertools.imap(lambda f: File(f), fileNames)
else:
# RTHandler is not required to produce masterjobconfig, in that
# case just use the inputsandbox
files = job.inputsandbox
result = create_sandbox(files, master=True)
if tmpDir is not None:
import shutil
# remove temp dir
if os.path.exists(tmpDir):
shutil.rmtree(tmpDir)
return result
def master_sandbox_prepare(app, appmasterconfig, sharedir_roots=None):
if sharedir_roots is None:
sharedir_roots = ['']
logger.debug("RTUTils master_sandbox_prepare")
# catch errors from not preparing properly
if not hasattr(stripProxy(app), 'is_prepared') or app.is_prepared is None:
logger.warning('Application is not prepared properly')
if hasattr(stripProxy(app), 'is_prepared'):
logger.warning("app.is_prepared: %s" % str(app.is_prepared))
import traceback
traceback.print_stack()
raise GangaException(None, 'Application not prepared properly')
# Note EITHER the master inputsandbox OR the job.inputsandbox is added to
# the subjob inputsandbox depending if the jobmasterconfig object is present
# or not... Therefore combine the job.inputsandbox with appmasterconfig.
job = stripProxy(app).getJobObject()
# user added items from the interactive GPI
from Ganga.Utility.Config import getConfig
if not getConfig('Output')['ForbidLegacyInput']:
inputsandbox = job.inputsandbox[:]
else:
if len(job.inputsandbox) > 0:
from Ganga.GPIDev.Lib.Job import JobError
raise JobError("InputFiles have been requested but there are objects in the inputSandBox... Aborting Job Prepare!")
inputsandbox = []
for filepattern in getInputFilesPatterns(job)[0]:
inputsandbox.append(File(filepattern))
if len(inputsandbox) > 100:
logger.warning('InputSandbox exceeds maximum size (100) supported by the Dirac backend')
raise GangaException(None, 'InputSandbox exceed maximum size')
outputsandbox = getOutputSandboxPatterns(job) # job.outputsandbox[:]
# inputsandbox files stored in share_dir from prepare method
sharedir_handler(app, sharedir_roots, inputsandbox)
# Here add any sandbox files/data coming from the appmasterconfig
# from master_configure. Catch the case where None is passed (as in tests)
if appmasterconfig:
inputsandbox += appmasterconfig.getSandboxFiles()
outputsandbox += appmasterconfig.getOutputSandboxFiles()
return unique(inputsandbox), unique(outputsandbox)