def prepare(self,app,appconfig,appmasterconfig,jobmasterconfig):
from Ganga.GPIDev.Adapters.StandardJobConfig import StandardJobConfig
c = StandardJobConfig(app.exe,app.inputs,app.args,app.outputs,app.envs)
c.monitoring_svc = mc['GangaTutorial']
return c
def prepare(self, app, appconfig, appmasterconfig, jobmasterconfig):
from Ganga.GPIDev.Adapters.StandardJobConfig import StandardJobConfig
c = StandardJobConfig(app.exe, app.inputs, app.args, app.outputs,
app.envs)
c.monitoring_svc = mc['GangaTutorial']
return c
def test_LHCbGaudiDiracRunTimeHandler_prepare(self):
# app = self.app
# app.extra = self.extra
sjc = StandardJobConfig(inputbox=self.input_files, outputbox=["dummy1.out", "dummy2.out", "dummy3.out"])
jobmasterconfig = StandardJobConfig()
jobmasterconfig.outputdata = []
stdjobconfig = self.rth.prepare(self.app, sjc, self.appmasterconfig, jobmasterconfig)
# should have subjob.in(buffer), data.opts and gaudiscript.py
print("sandbox =", stdjobconfig.getSandboxFiles())
print("sandbox =", [file.name for file in stdjobconfig.getSandboxFiles()])
assert len(stdjobconfig.getSandboxFiles()) == 3, "inputsandbox error"
l = len(stdjobconfig.getOutputSandboxFiles())
print("outputsandbox =", stdjobconfig.getOutputSandboxFiles())
assert l == 3, "outputsandbox error"
def prepare(self, app, appconfig, appmasterconfig, jobmasterconfig):
# some extra lines for simulation job options on DIRAC site
opts = 'DatabaseSvc.DbType = "sqlite";\n'
app.extra.input_buffers['data.opts'] += opts
# extra lines for reconstruction and remove empty files
if app.recoptsfile:
app.extra.input_buffers['recdata.opts'] += opts
extraopts = 'MixerAlg.UseNewDataDir = ".";\n'
app.extra.input_buffers['recdata.opts'] += extraopts
else:
app.extra.input_buffers.pop('recdata.opts', None)
# extra lines for analysis and remove empty files
if app.anaoptsfile:
app.extra.input_buffers['anadata.opts'] += opts
else:
app.extra.input_buffers.pop('anadata.opts', None)
if app.extra.inputdata and app.extra.inputdata.hasLFNs():
cat_opts = '\nFileCatalog().Catalogs = ' \
'["xmlcatalog_file:pool_xml_catalog.xml"]\n'
# app.extra.input_buffers['data.py'] += cat_opts
script = self._create_boss_script(app)
sandbox = get_input_sandbox(app.extra)
app.extra.outputsandbox += [
'script.log', 'rantrg.log', 'bosslog', 'bosserr', 'recbosslog',
'recbosserr', 'anabosslog', 'anabosserr'
]
outputsandbox = app.extra.outputsandbox
c = StandardJobConfig(script, sandbox, [], outputsandbox, None)
dirac_script = DiracScript()
dirac_script.job_type = 'Job()'
dirac_script.exe = DiracApplication(app, script)
dirac_script.platform = app.platform
dirac_script.output_sandbox = outputsandbox
dirac_script.default_sites = gConfig.getValue(
'/Resources/Applications/DefaultSites/%s' %
DfcOperation().getGroupName(), [])
if app.extra.inputdata:
dirac_script.inputdata = DiracInputData(app.extra.inputdata)
if app.extra.outputdata:
dirac_script.outputdata = app.extra.outputdata
c.script = dirac_script
return c
def test_GaudiRunTimeHandler_prepare(self):
sjc = StandardJobConfig(
inputbox=self.input_files, outputbox=['dummy1.out', 'dummy2.out', 'dummy3.out'])
jobmasterconfig = StandardJobConfig()
jobmasterconfig.outputdata = []
stdjobconfig = self.rth.prepare(
self.app, sjc, self.appmasterconfig, jobmasterconfig)
# should have subjob.in(buffer), data.opts and gaudiscript.py
print("sandbox =", stdjobconfig.getSandboxFiles())
print(
"sandbox =", [file.name for file in stdjobconfig.getSandboxFiles()])
assert len(stdjobconfig.getSandboxFiles()) == 4, 'inputsandbox error'
l = len(stdjobconfig.getOutputSandboxFiles())
print("outputsandbox =", stdjobconfig.getOutputSandboxFiles())
assert l == 4, 'outputsandbox error'
def master_prepare(self, app, appmasterconfig):
"""
Prepare the RTHandler for the master job so that applications to be submitted
Args:
app (GaudiExec): This application is only expected to handle GaudiExec Applications here
appmasterconfig (unknown): Output passed from the application master configuration call
"""
inputsandbox, outputsandbox = master_sandbox_prepare(
app, appmasterconfig)
if isinstance(app.jobScriptArchive, LocalFile):
app.jobScriptArchive = None
generateJobScripts(app, appendJobScripts=True)
scriptArchive = os.path.join(app.jobScriptArchive.localDir,
app.jobScriptArchive.namePattern)
inputsandbox.append(File(name=scriptArchive))
if app.getMetadata:
logger.info("Adding options to make the summary.xml")
inputsandbox.append(
FileBuffer(
'summary.py',
"\nfrom Gaudi.Configuration import *\nfrom Configurables import LHCbApp\nLHCbApp().XMLSummary='summary.xml'"
))
return StandardJobConfig(inputbox=unique(inputsandbox),
outputbox=unique(outputsandbox))
def master_prepare(self, app, appmasterconfig):
"""
Prepare the RTHandler for the master job so that applications to be submitted
Args:
app (GaudiExec): This application is only expected to handle GaudiExec Applications here
appmasterconfig (unknown): Output passed from the application master configuration call
"""
inputsandbox, outputsandbox = master_sandbox_prepare(app, appmasterconfig)
# If we are getting the metadata we need to make sure the summary.xml is added to the output sandbox if not there already.
if app.getMetadata and not 'summary.xml' in outputsandbox:
outputsandbox += ['summary.xml']
if not isinstance(app.uploadedInput, DiracFile):
generateDiracInput(app)
assert isinstance(app.uploadedInput, DiracFile), "Failed to upload needed file, aborting submit. Tried to upload to: %s\nIf your Ganga installation is not at CERN your username may be trying to create a non-existent LFN. Try setting the 'DIRAC' configuration 'DiracLFNBase' to your grid user path.\n" % DiracFile.diracLFNBase()
rep_data = app.uploadedInput.getReplicas()
assert rep_data != {}, "Failed to find a replica, aborting submit"
if isinstance(app.jobScriptArchive, (DiracFile, LocalFile)):
app.jobScriptArchive = None
generateDiracScripts(app)
assert isinstance(app.jobScriptArchive, DiracFile), "Failed to upload needed file, aborting submit"
rep_data = app.jobScriptArchive.getReplicas()
assert rep_data != {}, "Failed to find a replica, aborting submit"
return StandardJobConfig(inputbox=unique(inputsandbox), outputbox=unique(outputsandbox))
def configure(self, master_appconfig):
## strip leading and trailing blanks from arguments
self.arguments = [ a.strip() for a in self.arguments ]
## strip leading and trailing blanks from the command
self.commands = [ a.strip() for a in self.commands ]
## the script layout
the_script = layout.format (
scripts = [ os.path.join ( f.subdir , os.path.basename ( f.name ) ) for f in self.scripts ] ,
imports = [ os.path.join ( f.subdir , os.path.basename ( f.name ) ) for f in self.imports ] ,
arguments = self.arguments ,
command = self.commands
)
print 'SCRIPT:\n', the_script
# add summary.xml
outputsandbox_temp = XMLPostProcessor._XMLJobFiles()
outputsandbox_temp += unique(self.getJobObject().outputsandbox)
outputsandbox = unique(outputsandbox_temp)
input_files = []
input_files += [ FileBuffer('gaudipython-wrapper.py', the_script ) ]
logger.debug("Returning StandardJobConfig")
return (None, StandardJobConfig(inputbox=input_files,
outputbox=outputsandbox))
def prepare(self,app,appconfig,appmasterconfig,jobmasterconfig):
from Ganga.GPIDev.Adapters.StandardJobConfig import StandardJobConfig
#prepared_exe = File(os.path.join(shared_path,self.is_prepared.name, self.script_name))
prepared_exe = File(os.path.join( os.path.dirname(__file__), "runNA62MC.sh"))
#if app.is_prepared is not None:
# logger.info("Submitting a prepared application; taking any input files from %s" %(app.is_prepared.name))
# prepared_exe = File(os.path.join(os.path.join(shared_path,app.is_prepared.name),os.path.basename(prepared_exe.name)))
inputbox = [ File(os.path.join(os.path.join(shared_path,app.is_prepared.name), app.getMACFileName())) ]
inputbox.append( File(os.path.join(shared_path,app.is_prepared.name, app.script_name)) )
inputbox.append( File(os.path.join(shared_path,app.is_prepared.name, "input_files.tgz")) )
inputbox += app._getParent().inputsandbox
#outputbox = [ 'na62run%d.stderr.err' % app.run_number, 'na62run%d.stdout.out' % app.run_number ]
outputbox = []
outputbox += app._getParent().outputsandbox
env = {'NA62SCRIPT':app.script_name,
'NA62STDOUT':'na62run%d.out' % app.run_number,
'NA62STDERR':'na62run%d.err' % app.run_number}
args = [app.getMACFileName(), "%s/r%d/%s" % (app.job_type, app.getRevision(), app.getDecayString())]
c = StandardJobConfig(prepared_exe,inputbox,args,outputbox,env)
return c
def master_prepare(self, app, appconfig):
inputsandbox = app._getParent().inputsandbox[:]
if type(app.exe) == File:
os.system('chmod +x %s' % app.exe.name)
inputsandbox.append(app.exe)
c = StandardJobConfig('', inputsandbox, [], [], None)
return c
def setUp(self):
j = Job(application=DaVinci())
j.prepare()
from Ganga.Utility.Config import getConfig
if getConfig('Output')['ForbidLegacyInput']:
j.inputfiles = [LocalFile(name='dummy.in')]
else:
j.inputsandbox = [File(name='dummy.in')]
self.app = j.application._impl
#self.extra = GaudiExtras()
# self.extra.master_input_buffers['master.buffer'] = '###MASTERBUFFER###'
#self.extra.master_input_files = [File(name='master.in')]
# self.extra.input_buffers['subjob.buffer'] = '###SUBJOBBUFFER###'
self.input_files = [
File(name='subjob.in'),
File(
FileBuffer('subjob.buffer',
'###SUBJOBBUFFER###').create().name)
]
self.appmasterconfig = StandardJobConfig(inputbox=[
File(name='master.in'),
File(
FileBuffer('master.buffer',
'###MASTERBUFFER###').create().name)
])
j.outputfiles = ['dummy1.out', 'dummy2.out', 'dummy3.out']
self.rth = LHCbGaudiRunTimeHandler()
def test_GaudiRunTimeHandler_prepare(self):
sjc = StandardJobConfig(
inputbox=self.input_files,
outputbox=['dummy1.out', 'dummy2.out', 'dummy3.out'])
jobmasterconfig = StandardJobConfig()
jobmasterconfig.outputdata = []
stdjobconfig = self.rth.prepare(self.app, sjc, self.appmasterconfig,
jobmasterconfig)
# should have subjob.in(buffer), data.opts and gaudiscript.py
print("sandbox =", stdjobconfig.getSandboxFiles())
print("sandbox =",
[file.name for file in stdjobconfig.getSandboxFiles()])
assert len(stdjobconfig.getSandboxFiles()) == 4, 'inputsandbox error'
l = len(stdjobconfig.getOutputSandboxFiles())
print("outputsandbox =", stdjobconfig.getOutputSandboxFiles())
assert l == 4, 'outputsandbox error'
def master_prepare(self, app, appconfig):
# check file is set OK
if not app.script.name:
msg = 'Root.script.name must be set.'
raise ApplicationConfigurationError(None, msg)
if not os.path.exists(app.script.name):
msg = 'Script must exist!'
raise ApplicationConfigurationError(None, msg)
# check root version
global rootVersions
if not rootVersions:
from Dirac import Dirac
result = Dirac.execAPI('result = DiracCommands.getRootVersions()')
if not result_ok(result):
logger.error('Could not obtain available ROOT versions: %s' \
% str(result))
logger.error('ROOT version will not be validated.')
else:
rootVersions = result['Value']
if rootVersions:
found = False
versions = []
for v in rootVersions:
versions.append(v)
if app.version.find(v) >= 0:
found = True
break
if not found:
msg = 'Invalid ROOT version: %s. Valid versions: %s' \
% (app.version, str(versions))
raise ApplicationConfigurationError(None, msg)
inputsandbox = app._getParent().inputsandbox[:]
c = StandardJobConfig('', inputsandbox, [], [], None)
return c
def prepare(self, app, appconfig, appmasterconfig, jobmasterconfig):
from Ganga.GPIDev.Adapters.StandardJobConfig import StandardJobConfig
c = StandardJobConfig(app._scriptname,
app._getParent().inputsandbox, [],
app._getParent().outputsandbox, app.env)
return c
def prepare(self, app, appconfig, appmasterconfig, jobmasterconfig):
"""
Prepare the job in order to submit to the Local backend
Args:
app (GaudiExec): This application is only expected to handle GaudiExec Applications here
appconfig (unknown): Output passed from the application configuration call
appmasterconfig (unknown): Output passed from the application master_configure call
jobmasterconfig (tuple): Output from the master job prepare step
"""
job = app.getJobObject()
# Setup the command which to be run and the input and output
input_sand = genDataFiles(job)
output_sand = []
# NB with inputfiles the mechanics of getting the inputfiled to the input of the Localhost backend is taken care of for us
# We don't have to do anything to get our files when we start running
# Also we don't manage the outputfiles here!
job_command = prepareCommand(app)
# Generate the script which is to be executed for us on the WN
scriptToRun = generateWNScript(job_command, app)
input_sand.append(scriptToRun)
logger.debug("input_sand: %s" % input_sand)
# It's this authors opinion that the script should be in the PATH on the WN
# As it stands policy is that is isn't so we have to call it in a relative way, hence "./"
c = StandardJobConfig('./'+os.path.join(scriptToRun.subdir, scriptToRun.name), input_sand, [], output_sand)
return c
def _preparePyRootJobConfig(self, app, appconfig, appmasterconfig,
jobmasterconfig):
"""JobConfig for executing a Root script using CINT."""
from Ganga.GPIDev.Adapters.StandardJobConfig import StandardJobConfig
from os.path import join, split
from Ganga.GPIDev.Lib.File import getSharedPath
script = app.script
if script == File():
script = File(defaultPyRootScript())
else:
script = File(
os.path.join(
os.path.join(Ganga.GPIDev.Lib.File.getSharedPath(),
app.is_prepared.name),
os.path.basename(app.script.name)))
arguments = [join('.', script.subdir, split(script.name)[1])]
arguments.extend([str(s) for s in app.args])
arguments.append('-b')
inputsandbox = app._getParent().inputsandbox + [script]
(rootenv, _) = self._getRootEnvSys(app.version, usepython=True)
logger.debug("PyRoot environment:\n %s: ", str(rootenv))
return StandardJobConfig('python', inputsandbox, arguments,
app._getParent().outputsandbox)
def prepare(self, app, appconfig, appmasterconfig, jobmasterconfig):
logger.debug('RTHandler prepare called.')
from Ganga.GPIDev.Adapters.StandardJobConfig import StandardJobConfig
return StandardJobConfig(app.exe,
app._getParent().inputsandbox,
convertIntToStringArgs(app.args),
app._getParent().outputsandbox, app.env)
def prepare(self, app, appconfig, appmasterconfig, jobmasterconfig):
j = app.getJobObject()
script = self._create_exe_script(app)
c = StandardJobConfig(script, [File(script)], app.args,
j.outputsandbox, app.env)
dirac_script = DiracScript()
dirac_script.job_type = 'Job()'
#dirac_script.exe = DiracExe(app.exe,app.args)
dirac_script.exe = DiracExe('exe-script.py', [])
dirac_script.output_sandbox = j.outputsandbox[:]
if j.inputdata: dirac_script.inputdata = DiracInputData(j.inputdata)
if j.outputdata: dirac_script.outputdata = j.outputdata
c.script = dirac_script
return c
def master_prepare(self, app, extra):
if extra.inputdata and extra.inputdata.hasLFNs():
xml_catalog_str = extra.inputdata.getCatalog()
extra.master_input_buffers['catalog.xml'] = xml_catalog_str
sandbox = get_master_input_sandbox(app.getJobObject(), extra)
return StandardJobConfig('', inputbox=sandbox, args=[])
def prepare(self,app,appconfig,appmasterconfig,jobmasterconfig):
j = app.getJobObject()
script = self._create_exe_script(app)
c = StandardJobConfig(script,[File(script)],app.args,
j.outputsandbox,app.env)
dirac_script = DiracScript()
dirac_script.job_type = 'Job()'
#dirac_script.exe = DiracExe(app.exe,app.args)
dirac_script.exe = DiracExe('exe-script.py',[])
dirac_script.output_sandbox = j.outputsandbox[:]
if j.inputdata: dirac_script.inputdata = DiracInputData(j.inputdata)
if j.outputdata: dirac_script.outputdata = j.outputdata
c.script = dirac_script
return c
def master_prepare(self, app, appmasterconfig):
inputsandbox, outputsandbox = master_sandbox_prepare(app, appmasterconfig, ['inputsandbox'])
# add summary.xml
outputsandbox += ['summary.xml', '__parsedxmlsummary__']
return StandardJobConfig(inputbox=unique(inputsandbox),
outputbox=unique(outputsandbox))
def prepare(self, app, appconfig, appmasterconfig, jobmasterconfig):
from Ganga.GPIDev.Adapters.StandardJobConfig import StandardJobConfig
runScript, inputsandbox, rootenv = downloadWrapper(app)
# propage command line args
argList = [str(s) for s in app.args]
return StandardJobConfig(runScript, inputsandbox, argList,
app._getParent().outputsandbox)
def master_prepare(self, app, appmasterconfig):
"""
This function prepares the application of a master job during submit. A priori we aren't doing anything with this in Im3ShapeApp but until this is understood I'd rather not remove it
Args:
app (IApplication): This is the application given in the master job
appasterconfig (tuple): This is the configuration which is to prepare the app in the master job # TODO check type and this interface
"""
inputsandbox, outputsandbox = master_sandbox_prepare(app, appmasterconfig)
return StandardJobConfig(inputbox=unique(inputsandbox),
outputbox=unique(outputsandbox))
def prepare(self, app, appconfig, appmasterconfig, jobmasterconfig):
j = app.getJobObject()
input_dir = j.getInputWorkspace().getPath()
script_name = '%s/%s' % (input_dir, os.path.basename(app.script.name))
c = StandardJobConfig(script_name, [], [], j.outputsandbox, {})
# copy script to input dir
os.system('cp %s %s/.' % (app.script.name, input_dir))
dirac_script = DiracScript()
dirac_script.job_type = 'Job()'
dirac_script.exe = DiracRoot(app, script_name)
dirac_script.platform = getConfig('ROOT')['arch']
dirac_script.output_sandbox = j.outputsandbox[:]
if j.inputdata: dirac_script.inputdata = DiracInputData(j.inputdata)
if j.outputdata: dirac_script.outputdata = j.outputdata
c.script = dirac_script
return c
def prepare(self,app,appconfig,appmasterconfig,jobmasterconfig):
j = app.getJobObject()
input_dir = j.getInputWorkspace().getPath()
script_name = '%s/%s' % (input_dir,os.path.basename(app.script.name))
c = StandardJobConfig(script_name,[],[],j.outputsandbox,{})
# copy script to input dir
os.system('cp %s %s/.' % (app.script.name,input_dir))
dirac_script = DiracScript()
dirac_script.job_type = 'Job()'
dirac_script.exe = DiracRoot(app,script_name)
dirac_script.platform = getConfig('ROOT')['arch']
dirac_script.output_sandbox = j.outputsandbox[:]
if j.inputdata: dirac_script.inputdata = DiracInputData(j.inputdata)
if j.outputdata: dirac_script.outputdata = j.outputdata
c.script = dirac_script
return c
def master_prepare(self, app, appmasterconfig):
inputsandbox, outputsandbox = master_sandbox_prepare(
app, appmasterconfig)
if type(app.exe) == File:
input_dir = app.getJobObject().getInputWorkspace().getPath()
exefile = os.path.join(input_dir, os.path.basename(app.exe.name))
if not os.path.exists(exefile):
msg = 'Executable: "%s" must exist!' % str(exefile)
raise ApplicationConfigurationError(None, msg)
os.system('chmod +x %s' % exefile)
return StandardJobConfig(inputbox=unique(inputsandbox),
outputbox=unique(outputsandbox))
def master_prepare(self, app, appmasterconfig):
logger.debug("Master Prepare")
inputsandbox, outputsandbox = master_sandbox_prepare(
app, appmasterconfig, ['inputsandbox'])
# add summary.xml
outputsandbox += ['summary.xml', '__parsedxmlsummary__']
logger.debug("Master Prepare LHCbGaudiDiracRunTimeHandler")
return StandardJobConfig(inputbox=unique(inputsandbox),
outputbox=unique(outputsandbox))
def prepare(self, app, appsubconfig, appmasterconfig, jobmasterconfig):
inputsandbox, outputsandbox = sandbox_prepare(app, appsubconfig,
appmasterconfig,
jobmasterconfig)
run_script = self.__create_run_script(app, appsubconfig,
appmasterconfig, jobmasterconfig,
inputsandbox, outputsandbox)
return StandardJobConfig(FileBuffer('gaudi-script.py',
run_script,
executable=1),
inputbox=unique(inputsandbox),
outputbox=unique(outputsandbox))
def prepare(self, app, appsubconfig, appmasterconfig, jobmasterconfig):
inputsandbox, outputsandbox = sandbox_prepare(
app, appsubconfig, appmasterconfig, jobmasterconfig)
input_data, parametricinput_data = dirac_inputdata(app)
job = app.getJobObject()
outputfiles = [this_file.namePattern for this_file in job.outputfiles if isinstance(this_file, DiracFile)]
gaudi_script_path = os.path.join(
job.getInputWorkspace().getPath(), "gaudi-script.py")
script_generator(gaudi_script_template(),
#remove_unreplaced = False,
outputfile_path=gaudi_script_path,
PLATFORM=app.platform,
COMMAND='gaudirun.py' # ,
#OUTPUTFILESINJECTEDCODE = getWNCodeForOutputPostprocessing(job, ' ')
)
dirac_script = script_generator(diracAPI_script_template(),
DIRAC_IMPORT='from DIRAC.Interfaces.API.Dirac import Dirac',
DIRAC_JOB_IMPORT='from DIRAC.Interfaces.API.Job import Job',
DIRAC_OBJECT='Dirac()',
JOB_OBJECT='Job()',
NAME=mangle_job_name(app),
EXE=gaudi_script_path,
EXE_ARG_STR=' '.join(
[str(arg) for arg in app.args]),
EXE_LOG_FILE='Ganga_%s_%s.log' % (
app.appname, app.version),
INPUTDATA=input_data,
PARAMETRIC_INPUTDATA=parametricinput_data,
OUTPUT_SANDBOX=outputsandbox,
OUTPUTDATA=list(outputfiles),
OUTPUT_PATH="", # job.fqid,
OUTPUT_SE=getConfig(
'DIRAC')['DiracOutputDataSE'],
SETTINGS=diracAPI_script_settings(app),
DIRAC_OPTS=job.backend.diracOpts,
PLATFORM=app.platform,
# leave the sandbox for altering later as needs
# to be done in backend.submit to combine master.
# Note only using 2 #s as auto-remove 3
INPUT_SANDBOX='##INPUT_SANDBOX##'
)
return StandardJobConfig(dirac_script,
inputbox=unique(inputsandbox),
outputbox=unique(outputsandbox))
def _prepareCintJobConfig(self, app, appconfig, appmasterconfig,
jobmasterconfig):
"""JobConfig for executing a Root script using CINT."""
from Ganga.GPIDev.Adapters.StandardJobConfig import StandardJobConfig
from os.path import join, split
import string
from Ganga.GPIDev.Lib.File import getSharedPath
# Arguments to the ROOT script needs to be a comma separated list
# enclosed in (). Strings should be enclosed in escaped double quotes.
arglist = []
for arg in app.args:
if isinstance(arg, str):
arglist.append(self.quoteCintArgString(arg))
else:
arglist.append(arg)
rootarg = '\(' + string.join([repr(s) for s in arglist], ',') + '\)'
script = app.script
if script == File():
script = File(defaultScript())
else:
script = File(
os.path.join(
os.path.join(Ganga.GPIDev.Lib.File.getSharedPath(),
app.is_prepared.name),
os.path.basename(app.script.name)))
# Start ROOT with the -b and -q options to run without a
# terminal attached.
arguments = [
'-b', '-q',
os.path.relpath(join('.', script.subdir,
split(script.name)[1])) + rootarg
]
inputsandbox = app._getParent().inputsandbox + [script]
(rootenv, _) = self._getRootEnvSys(app.version)
logger.debug("ROOT environment:\n %s: ", str(rootenv))
returnable = StandardJobConfig('root.exe', inputsandbox, arguments,
app._getParent().outputsandbox)
logger.debug("root jobconfig: %s" % str(returnable))
return returnable
def master_prepare(self, app, appmasterconfig):
inputsandbox, outputsandbox = master_sandbox_prepare(
app, appmasterconfig)
# check file is set OK
if not app.script.name:
msg = 'Root.script.name must be set.'
raise ApplicationConfigurationError(msg)
sharedir_scriptpath = os.path.join(get_share_path(app),
os.path.basename(app.script.name))
if not os.path.exists(sharedir_scriptpath):
msg = 'Script must exist!'
raise ApplicationConfigurationError(msg)
return StandardJobConfig(inputbox=unique(inputsandbox),
outputbox=unique(outputsandbox))
def prepare(self, app, extra, appmasterconfig, jobmasterconfig):
if extra.inputdata and extra.inputdata.hasLFNs():
s = '\nFileCatalog().Catalogs = ["xmlcatalog_file:catalog.xml"]\n'
extra.input_buffers['data.py'] += s
sandbox = get_input_sandbox(extra)
outdata = extra.outputdata
if not outdata: outdata = app.getJobObject().outputdata
script = create_runscript(app, outdata, app.getJobObject())
#logger.error("zhangxm log: gaudiscript.py:\n %s " % script)
return StandardJobConfig(FileBuffer('gaudiscript.py',
script,
executable=1),
inputbox=sandbox,
args=[],
outputbox=extra.outputsandbox)
def test_submit(self):
j = Job(backend=self.db)
self.db._parent = j._impl
file1 = tempfile.NamedTemporaryFile('w')
file2 = tempfile.NamedTemporaryFile('w')
file3 = tempfile.NamedTemporaryFile('w')
sjc = StandardJobConfig(exe=self.script,
inputbox=[File(file1.name)._impl,
File(file2.name)._impl,
File(file3.name)._impl],
outputbox=['d', 'e', 'f'])
def _addition_sandbox_content(subjobconfig):
self.assertEqual(subjobconfig, sjc, 'config objects not equal')
return ['g']
def _common_submit(dirac_script):
# this needs to change to workerpool
#from GangaDirac.Lib.Server.DiracClient import DiracClient
#self.assertTrue(isinstance(server, DiracClient),'not a dirac client')
f = open(dirac_script, 'r')
script = f.read()
f.close()
self.assertNotEqual(script, self.script, 'script not changed')
self.assertEqual(self.script.replace('##INPUT_SANDBOX##',
str(['a', 'b', 'c'] +
[os.path.join(j._impl.getInputWorkspace().getPath(),
'_input_sandbox_0.tgz')] +
['g'])),
script, 'script not what it should be')
return True
setattr(
self.db, '_addition_sandbox_content', _addition_sandbox_content)
setattr(self.db, '_common_submit', _common_submit)
self.assertTrue(self.db.submit(sjc, ['a', 'b', 'c']), 'didnt run')
file1.close()
file2.close()
file3.close()
def master_prepare(self, app, appmasterconfig):
"""
Prepare the RTHandler for the master job so that applications to be submitted
Args:
app (GaudiExec): This application is only expected to handle GaudiExec Applications here
appmasterconfig (unknown): Output passed from the application master configuration call
"""
inputsandbox, outputsandbox = master_sandbox_prepare(app, appmasterconfig)
if isinstance(app.jobScriptArchive, LocalFile):
app.jobScriptArchive = None
generateJobScripts(app, appendJobScripts=True)
scriptArchive = os.path.join(app.jobScriptArchive.localDir, app.jobScriptArchive.namePattern)
inputsandbox.append(File(name=scriptArchive))
return StandardJobConfig(inputbox=unique(inputsandbox), outputbox=unique(outputsandbox))
def configure(self, master_appconfig):
# self._configure()
name = join('.', self.script[0].subdir, split(self.script[0].name)[-1])
script = "from Gaudi.Configuration import *\n"
if self.args:
script += 'import sys\nsys.argv += %s\n' % str(self.args)
script += "importOptions('data.py')\n"
script += "execfile(\'%s\')\n" % name
# add summary.xml
outputsandbox_temp = XMLPostProcessor._XMLJobFiles()
outputsandbox_temp += unique(self.getJobObject().outputsandbox)
outputsandbox = unique(outputsandbox_temp)
input_files = []
input_files += [FileBuffer('gaudipython-wrapper.py', script)]
logger.debug("Returning Job Configuration")
return (None,
StandardJobConfig(inputbox=input_files,
outputbox=outputsandbox))
def __init__(self, exe=None, inputbox=[], outputbox=[], args=[], env=None):
if len(env) != 0:
logger.error("Clobi backend currently does not support exe env")
env = {}
StandardJobConfig.__init__(self,exe,inputbox,args,outputbox,env)
