def _check_inputs(self):
"""Checks the validity of some of user's entries for Gaudi schema"""
# Warn user that no optsfiles given
if len(self.optsfile) == 0:
logger.warning("The 'optsfile' is not set. I hope this is OK!")
else:
# Check for non-exting optsfiles defined
nonexistentOptFiles = []
for f in self.optsfile:
from Ganga.GPIDev.Lib.File.File import File
if type(f) is str:
myFile = File(f)
else:
myFile = f
myFile.name = fullpath(myFile.name)
if not os.path.isfile(myFile.name):
nonexistentOptFiles.append(myFile)
if len(nonexistentOptFiles):
tmpmsg = "The 'optsfile' attribute contains non-existent file/s: ["
for _f in nonexistentOptFiles:
tmpmsg += "'%s', " % _f.name
msg = tmpmsg[:-2] + ']'
raise ApplicationConfigurationError(msg)
def _check_inputs(self):
"""Checks the validity of some of user's entries for Gaudi schema"""
# Warn user that no optsfiles given
if len(self.optsfile) == 0:
logger.warning("The 'optsfile' is not set. I hope this is OK!")
else:
# Check for non-exting optsfiles defined
nonexistentOptFiles = []
for f in self.optsfile:
from Ganga.GPIDev.Lib.File.File import File
if type(f) is str:
myFile = File(f)
else:
myFile = f
myFile.name = fullpath(myFile.name)
if not os.path.isfile(myFile.name):
nonexistentOptFiles.append(myFile)
if len(nonexistentOptFiles):
tmpmsg = "The 'optsfile' attribute contains non-existent file/s: ["
for _f in nonexistentOptFiles:
tmpmsg += "'%s', " % _f.name
msg = tmpmsg[:-2] + ']'
raise ApplicationConfigurationError(msg)
def prepare(self, app, appconfig, appmasterconfig, jobmasterconfig):
from Ganga.Lib.gLite import gLiteJobConfig
prepared_exe = app.exe
if app.is_prepared is not None:
shared_path = os.path.join(expandfilename(getConfig('Configuration')['gangadir']),
'shared', getConfig('Configuration')['user'])
if isinstance(app.exe, str):
# we have a file. is it an absolute path?
if os.path.abspath(app.exe) == app.exe:
logger.info("Submitting a prepared application; taking any input files from %s" % (
app.is_prepared.name))
prepared_exe = File(os.path.join(os.path.join(
shared_path, app.is_prepared.name), os.path.basename(File(app.exe).name)))
# else assume it's a system binary, so we don't need to
# transport anything to the sharedir
else:
prepared_exe = app.exe
elif isinstance(app.exe, File):
logger.info("Submitting a prepared application; taking any input files from %s" % (
app.is_prepared.name))
prepared_exe = File(os.path.join(os.path.join(
shared_path, app.is_prepared.name), os.path.basename(File(app.exe).name)))
return gLiteJobConfig(prepared_exe, app._getParent().inputsandbox, convertIntToStringArgs(app.args), app._getParent().outputsandbox, app.env)
def setUp(self):
j = Job(application=DaVinci())
j.prepare()
from Ganga.Utility.Config import getConfig
if getConfig('Output')['ForbidLegacyInput']:
j.inputfiles = [LocalFile(name='dummy.in')]
else:
j.inputsandbox = [File(name='dummy.in')]
self.app = j.application._impl
#self.extra = GaudiExtras()
# self.extra.master_input_buffers['master.buffer'] = '###MASTERBUFFER###'
#self.extra.master_input_files = [File(name='master.in')]
# self.extra.input_buffers['subjob.buffer'] = '###SUBJOBBUFFER###'
self.input_files = [
File(name='subjob.in'),
File(
FileBuffer('subjob.buffer',
'###SUBJOBBUFFER###').create().name)
]
self.appmasterconfig = StandardJobConfig(inputbox=[
File(name='master.in'),
File(
FileBuffer('master.buffer',
'###MASTERBUFFER###').create().name)
])
j.outputfiles = ['dummy1.out', 'dummy2.out', 'dummy3.out']
self.rth = LHCbGaudiRunTimeHandler()
def master_prepare(self, app, appmasterconfig):
"""
Prepare the RTHandler for the master job so that applications to be submitted
Args:
app (GaudiExec): This application is only expected to handle GaudiExec Applications here
appmasterconfig (unknown): Output passed from the application master configuration call
"""
inputsandbox, outputsandbox = master_sandbox_prepare(
app, appmasterconfig)
if isinstance(app.jobScriptArchive, LocalFile):
app.jobScriptArchive = None
generateJobScripts(app, appendJobScripts=True)
scriptArchive = os.path.join(app.jobScriptArchive.localDir,
app.jobScriptArchive.namePattern)
inputsandbox.append(File(name=scriptArchive))
if app.getMetadata:
logger.info("Adding options to make the summary.xml")
inputsandbox.append(
FileBuffer(
'summary.py',
"\nfrom Gaudi.Configuration import *\nfrom Configurables import LHCbApp\nLHCbApp().XMLSummary='summary.xml'"
))
return StandardJobConfig(inputbox=unique(inputsandbox),
outputbox=unique(outputsandbox))
def prepare(self, force=False):
from Ganga.GPIDev.Lib.GangaList.GangaList import GangaList
from Ganga.GPIDev.Lib.File.File import File
if isType(self.optsfile, (list, tuple, GangaList)):
for this_file in self.optsfile:
if type(this_file) is str:
this_file = File(this_file)
else:
continue
elif type(self.optsfile) is str:
self.optsfile = [File(self.optsfile)]
try:
super(Gaudi, self).prepare(force)
except Exception, err:
logger.debug("Super Prepare Error:\n%s" % str(err))
raise err
def master_prepare(self, masterjobconfig):
""" Prepare the master job (shared sandbox files). This method is/should be called by master_submit() exactly once.
The input sandbox is created according to self._packed_input_sandbox flag (a class attribute)
"""
from Ganga.GPIDev.Lib.File.OutputFileManager import getInputFilesPatterns
from Ganga.GPIDev.Lib.File.File import File, ShareDir
job = self.getJobObject()
create_sandbox = job.createInputSandbox
if self._packed_input_sandbox:
create_sandbox = job.createPackedInputSandbox
if masterjobconfig:
if hasattr(job.application, 'is_prepared') and isType(
job.application.is_prepared, ShareDir):
sharedir_pred = lambda f: f.name.find(job.application.
is_prepared.name) > -1
sharedir_files = itertools.ifilter(
sharedir_pred, masterjobconfig.getSandboxFiles())
nonsharedir_files = itertools.ifilterfalse(
sharedir_pred, masterjobconfig.getSandboxFiles())
# ATLAS use bool to bypass the prepare mechanism and some ATLAS
# apps have no is_prepared
else:
sharedir_files = []
nonsharedir_files = masterjobconfig.getSandboxFiles()
inputsandbox = create_sandbox(nonsharedir_files, master=True)
inputsandbox.extend(
itertools.imap(lambda f: f.name, sharedir_files))
return inputsandbox
tmpDir = None
files = []
if len(job.inputfiles) > 0 or (len(job.subjobs) == 0 and job.inputdata and\
isType(job.inputdata, GangaDataset) and job.inputdata.treat_as_inputfiles):
(fileNames, tmpDir) = getInputFilesPatterns(job)
files = itertools.imap(lambda f: File(f), fileNames)
else:
# RTHandler is not required to produce masterjobconfig, in that
# case just use the inputsandbox
files = job.inputsandbox
result = create_sandbox(files, master=True)
if tmpDir is not None:
import shutil
# remove temp dir
if os.path.exists(tmpDir):
shutil.rmtree(tmpDir)
return result
def master_prepare(self, app, appmasterconfig):
"""
Prepare the RTHandler for the master job so that applications to be submitted
Args:
app (GaudiExec): This application is only expected to handle GaudiExec Applications here
appmasterconfig (unknown): Output passed from the application master configuration call
"""
inputsandbox, outputsandbox = master_sandbox_prepare(app, appmasterconfig)
if isinstance(app.jobScriptArchive, LocalFile):
app.jobScriptArchive = None
generateJobScripts(app, appendJobScripts=True)
scriptArchive = os.path.join(app.jobScriptArchive.localDir, app.jobScriptArchive.namePattern)
inputsandbox.append(File(name=scriptArchive))
return StandardJobConfig(inputbox=unique(inputsandbox), outputbox=unique(outputsandbox))
def __populate():
if len(transformDictionary) == 0:
file_obj = File()
transformDictionary[type(file_obj)] = LocalFile