def prepare(self, app, appconfig, appmasterconfig, jobmasterconfig):
from Ganga.Lib.gLite import gLiteJobConfig
prepared_exe = app.exe
if app.is_prepared is not None:
shared_path = os.path.join(expandfilename(getConfig('Configuration')['gangadir']),
'shared', getConfig('Configuration')['user'])
if isinstance(app.exe, str):
# we have a file. is it an absolute path?
if os.path.abspath(app.exe) == app.exe:
logger.info("Submitting a prepared application; taking any input files from %s" % (
app.is_prepared.name))
prepared_exe = File(os.path.join(os.path.join(
shared_path, app.is_prepared.name), os.path.basename(File(app.exe).name)))
# else assume it's a system binary, so we don't need to
# transport anything to the sharedir
else:
prepared_exe = app.exe
elif isType(app.exe, File):
logger.info("Submitting a prepared application; taking any input files from %s" % (
app.is_prepared.name))
prepared_exe = File(os.path.join(os.path.join(
shared_path, app.is_prepared.name), os.path.basename(File(app.exe).name)))
return gLiteJobConfig(prepared_exe, app._getParent().inputsandbox, convertIntToStringArgs(app.args), app._getParent().outputsandbox, app.env)
def _preparePyRootJobConfig(self, app, appconfig, appmasterconfig,
jobmasterconfig):
"""JobConfig for executing a Root script using CINT."""
from Ganga.GPIDev.Adapters.StandardJobConfig import StandardJobConfig
from os.path import join, split
from Ganga.GPIDev.Lib.File import getSharedPath
script = app.script
if script == File():
script = File(defaultPyRootScript())
else:
script = File(
os.path.join(
os.path.join(Ganga.GPIDev.Lib.File.getSharedPath(),
app.is_prepared.name),
os.path.basename(app.script.name)))
arguments = [join('.', script.subdir, split(script.name)[1])]
arguments.extend([str(s) for s in app.args])
arguments.append('-b')
inputsandbox = app._getParent().inputsandbox + [script]
(rootenv, _) = self._getRootEnvSys(app.version, usepython=True)
logger.debug("PyRoot environment:\n %s: ", str(rootenv))
return StandardJobConfig('python', inputsandbox, arguments,
app._getParent().outputsandbox)
def downloadWrapper(app):
from os.path import join, split
from Ganga.GPIDev.Lib.File import FileBuffer
import string
from Ganga.GPIDev.Lib.File import getSharedPath
rootsys = join('.', 'root')
rootenv = {'ROOTSYS': rootsys}
script = app.script
if script == File():
if not app.usepython:
script = File(defaultScript())
else:
script = File(defaultPyRootScript())
else:
script = File(os.path.join(os.path.join(Ganga.GPIDev.Lib.File.getSharedPath(), app.is_prepared.name), os.path.basename(app.script.name)))
commandline = ''
scriptPath = join('.', script.subdir, split(script.name)[1])
if not app.usepython:
# Arguments to the ROOT script needs to be a comma separated list
# enclosed in (). Strings should be enclosed in escaped double quotes.
arglist = []
for arg in app.args:
if isinstance(arg, str):
arglist.append('\\\'' + arg + '\\\'')
else:
arglist.append(arg)
rootarg = '\(\"' + string.join([str(s) for s in arglist], ',') + '\"\)'
# use root
commandline = 'root.exe -b -q ' + scriptPath + rootarg + ''
else:
# use python
pyarg = string.join([str(s) for s in app.args], ' ')
commandline = '\'%(PYTHONCMD)s ' + scriptPath + ' ' + pyarg + ' -b \''
logger.debug("Command line: %s: ", commandline)
# Write a wrapper script that installs ROOT and runs script
script_location = os.path.join(os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))),
'wrapperScriptTemplate.py')
from Ganga.GPIDev.Lib.File import FileUtils
wrapperscript = FileUtils.loadScript(script_location, '')
wrapperscript = wrapperscript.replace('###COMMANDLINE###', commandline)
wrapperscript = wrapperscript.replace('###ROOTVERSION###', app.version)
wrapperscript = wrapperscript.replace('###SCRIPTPATH###', scriptPath)
wrapperscript = wrapperscript.replace('###USEPYTHON###', str(app.usepython))
logger.debug('Script to run on worker node\n' + wrapperscript)
scriptName = "rootwrapper_generated_%s.py" % randomString()
runScript = FileBuffer(scriptName, wrapperscript, executable=1)
inputsandbox = app._getParent().inputsandbox + [script]
return runScript, inputsandbox, rootenv
def _check_inputs(self):
"""Checks the validity of user's entries for BenderScript schema"""
if not self.scripts and not self.imports :
raise ApplicationConfigurationError("Application scripts/imports are not defined")
if isinstance ( self.scripts , str ) : self.scripts = [ File ( self.scripts ) ]
if isinstance ( self.imports , str ) : self.imports = [ File ( self.imports ) ]
for f in self.scripts : f.name = fullpath ( f.name )
for f in self.imports : f.name = fullpath ( f.name )
def get_dataset(self):
from urlparse import urlparse
tables = []
## switching lower and upper if different order is given
if self.table_id_lower > self.table_id_upper:
tmp = self.table_id_lower
self.table_id_lower = self.table_id_upper
self.table_id_upper = tmp
## restrict maximum table id to self.table_id_max
if self.table_id_upper > self.table_id_max:
logger.warning('table_id_upper is restricted to up to %d' %
self.table_id_max)
self.table_id_upper = self.table_id_max
for i in range(self.table_id_lower, self.table_id_upper + 1):
location = '%s/primes%d.zip' % (self.table_location, i)
url = urlparse(location)
if url[0] in ['file', '']:
# create new File object if the location is local
tables.append(File(location))
else:
tables.append(location)
return tables
def sharedir_handler(app, root_dir_names, output):
share_path = get_share_path(app)
if '' in root_dir_names:
# get the '' entry and only that entry so dont waste time walking
# others before root.
root_dir_names = ['']
for share_dir in root_dir_names:
if share_dir == '':
share_dir = share_path
else:
share_dir = os.path.join(share_path, share_dir)
for root, dirs, files in os.walk(share_dir):
# [1:] removes the preceeding /
subdir = root.replace(share_dir, '')[1:]
if isType(output, (list, tuple, GangaList)):
output += [File(name=os.path.join(root, f), subdir=subdir) for f in files]
# for f in files:
## output += [File(name=os.path.join(root,f),subdir=subdir)]
elif type(output) is type(''):
for d in dirs:
if not os.path.isdir(d):
os.makedirs(d)
for f in files:
shutil.copy(os.path.join(root, f), os.path.join(output, subdir, f))
else:
raise GangaException('output must be either a list to append to or a path string to copy to')
def _prepareCintJobConfig(self, app, appconfig, appmasterconfig,
jobmasterconfig):
"""JobConfig for executing a Root script using CINT."""
from Ganga.GPIDev.Adapters.StandardJobConfig import StandardJobConfig
from os.path import join, split
import string
from Ganga.GPIDev.Lib.File import getSharedPath
# Arguments to the ROOT script needs to be a comma separated list
# enclosed in (). Strings should be enclosed in escaped double quotes.
arglist = []
for arg in app.args:
if isinstance(arg, str):
arglist.append(self.quoteCintArgString(arg))
else:
arglist.append(arg)
rootarg = '\(' + string.join([repr(s) for s in arglist], ',') + '\)'
script = app.script
if script == File():
script = File(defaultScript())
else:
script = File(
os.path.join(
os.path.join(Ganga.GPIDev.Lib.File.getSharedPath(),
app.is_prepared.name),
os.path.basename(app.script.name)))
# Start ROOT with the -b and -q options to run without a
# terminal attached.
arguments = [
'-b', '-q',
os.path.relpath(join('.', script.subdir,
split(script.name)[1])) + rootarg
]
inputsandbox = app._getParent().inputsandbox + [script]
(rootenv, _) = self._getRootEnvSys(app.version)
logger.debug("ROOT environment:\n %s: ", str(rootenv))
returnable = StandardJobConfig('root.exe', inputsandbox, arguments,
app._getParent().outputsandbox)
logger.debug("root jobconfig: %s" % str(returnable))
return returnable
def __init__(self):
super(Root, self).__init__()
from Ganga.GPIDev.Lib.File import getSharedPath
self.shared_path = Ganga.GPIDev.Lib.File.getSharedPath()
if self.script is None or self.script == File():
self.script = getDefaultScript()
def _check_inputs(self):
"""Checks the validity of user's entries for Ostap schema"""
if not self.scripts and not self.commands and not self.arguments:
raise ApplicationConfigurationError(
None, "Application scripts are not defined")
if isinstance(self.scripts, str): self.scripts = [File(self.scripts)]
for f in self.scripts:
f.name = fullpath(f.name)
def test_submit(db):
j = Job()
j.id = 0
j.backend = db
db._parent = j
file1 = tempfile.NamedTemporaryFile('w')
file2 = tempfile.NamedTemporaryFile('w')
file3 = tempfile.NamedTemporaryFile('w')
sjc = StandardJobConfig(
exe=script_template,
inputbox=[File(file1.name),
File(file2.name),
File(file3.name)],
outputbox=['d', 'e', 'f'])
def fake_common_submit(dirac_script):
with open(dirac_script, 'r') as f:
script = f.read()
assert script != script_template, 'script not changed'
assert script_template.replace(
'##INPUT_SANDBOX##',
str(['a', 'b', 'c'] + [
os.path.join(j.getInputWorkspace().getPath(),
'_input_sandbox_0.tgz')
] + ['g'])) == script, 'script not what it should be'
return True
with patch.object(db, '_addition_sandbox_content',
return_value=['g']) as _addition_sandbox_content:
with patch.object(db, '_common_submit',
Mock(fake_common_submit)) as _common_submit:
assert db.submit(sjc, ['a', 'b', 'c'])
_addition_sandbox_content.assert_called_once_with(sjc)
_common_submit.assert_called_once()
file1.close()
file2.close()
file3.close()
def bodyTagFile(self, tagFile=""):
lineList = []
removeList = []
inbox = [File(name=tagFile)]
lineList.extend\
( [
"JOB_TAGFILE=\"%s\"" % os.path.basename( tagFile ),
"TAGFILE_DIR=\"${WORKDIR}/tag_data\"",
"rm -rf ${TAGFILE_DIR}",
"mkdir -p ${TAGFILE_DIR}",
"split -l 6 -a 7 ${JOB_TAGFILE} ${TAGFILE_DIR}/tagfile_",
"TAGFILE_LIST=$(ls ${TAGFILE_DIR})",
"",
"for TAGFILE in ${TAGFILE_LIST}; do",
" TAGFILE_PATH=${TAGFILE_DIR}/${TAGFILE}",
" IMAGE_URL=\"\"",
" while read TAG_LINE; do",
" TAG=${TAG_LINE%%:*}",
" if [ \"${TAG}\" == \"URLImage\" ]; then",
" IMAGE_URL=${TAG_LINE#\"URLImage: \"}",
" IMAGE_FILE=${IMAGE_URL##*/}",
" IMAGE_SUFFIX=${IMAGE_FILE##*\\.}",
" break",
" fi",
" done < ${TAGFILE_PATH}",
"",
" if [ \"${IMAGE_URL}\" != \"\" ]; then"
" wget -q ${IMAGE_URL} -O ${IMAGE_FILE}",
" if [ -f ${IMAGE_FILE} ]; then",
" N_DOWNLOAD_SUCCESS=$((${N_DOWNLOAD_SUCCESS}+1))",
" IMAGE_MD5SUM=$(md5sum ${IMAGE_FILE})",
" IMAGE_MD5NAME=$(echo ${IMAGE_MD5SUM} | cut -d ' ' -f 1)",
" SUBDIR1=$(echo ${IMAGE_MD5NAME} | cut -c 1-3)",
" SUBDIR2=$(echo ${IMAGE_MD5NAME} | cut -c 4-6)",
" IMAGE_REMOTE_DIR=\"${WORKDIR}/images/${SUBDIR1}/${SUBDIR2}\"",
" IMAGE_PATH_NO_SUFFIX=\"${IMAGE_REMOTE_DIR}/${IMAGE_MD5NAME}\"",
" IMAGE_REMOTE_PATH=\"${IMAGE_PATH_NO_SUFFIX}.${IMAGE_SUFFIX}\"",
" TAGFILE_REMOTE_PATH=\"${IMAGE_PATH_NO_SUFFIX}.txt\"",
" mkdir -p ${IMAGE_REMOTE_DIR}/vseg",
" mv -f ${IMAGE_FILE} ${IMAGE_REMOTE_PATH}",
" mv -f ${TAGFILE_PATH} ${TAGFILE_REMOTE_PATH}",
" else",
" N_DOWNLOAD_FAILURE=$((${N_DOWNLOAD_FAILURE}+1))",
" fi",
" fi",
"done",
""
])
return (lineList, inbox, removeList)
def prepare(self, app, appsubconfig, appmasterconfig, jobmasterconfig):
local = VansegLocal.prepare\
( self, app, appsubconfig, appmasterconfig, jobmasterconfig )
inbox = []
inbox.extend(local.inputbox)
exeFile = File(fullpath(app.exe))
if not exeFile in inbox:
inbox.append(exeFile)
requirements = LCGRequirements()
lcg = LCGJobConfig( exe = local.exe, inputbox = inbox,\
outputbox = local.outputbox, requirements = requirements )
return lcg
def _check_inputs(self):
"""Checks the validity of user's entries for GaudiPython schema"""
self._check_gaudi_inputs(self.script, self.project)
if len(self.script) == 0:
logger.warning("No script defined. Will use a default " \
'script which is probably not what you want.')
self.script = [
File(
os.path.join(
os.path.dirname(inspect.getsourcefile(GaudiPython)),
'options/GaudiPythonExample.py'))
]
return
def master_sandbox_prepare(app, appmasterconfig, sharedir_roots=None):
if sharedir_roots is None:
sharedir_roots = ['']
logger.debug("RTUTils master_sandbox_prepare")
# catch errors from not preparing properly
if not hasattr(stripProxy(app), 'is_prepared') or app.is_prepared is None:
logger.warning('Application is not prepared properly')
if hasattr(stripProxy(app), 'is_prepared'):
logger.warning("app.is_prepared: %s" % str(app.is_prepared))
import traceback
traceback.print_stack()
raise GangaException(None, 'Application not prepared properly')
# Note EITHER the master inputsandbox OR the job.inputsandbox is added to
# the subjob inputsandbox depending if the jobmasterconfig object is present
# or not... Therefore combine the job.inputsandbox with appmasterconfig.
job = stripProxy(app).getJobObject()
# user added items from the interactive GPI
from Ganga.Utility.Config import getConfig
if not getConfig('Output')['ForbidLegacyInput']:
inputsandbox = job.inputsandbox[:]
else:
if len(job.inputsandbox) > 0:
from Ganga.GPIDev.Lib.Job import JobError
raise JobError(
"InputFiles have been requested but there are objects in the inputSandBox... Aborting Job Prepare!"
)
inputsandbox = []
for filepattern in getInputFilesPatterns(job)[0]:
inputsandbox.append(File(filepattern))
if len(inputsandbox) > 100:
logger.warning(
'InputSandbox exceeds maximum size (100) supported by the Dirac backend'
)
raise GangaException(None, 'InputSandbox exceed maximum size')
outputsandbox = getOutputSandboxPatterns(job) # job.outputsandbox[:]
# inputsandbox files stored in share_dir from prepare method
sharedir_handler(app, sharedir_roots, inputsandbox)
# Here add any sandbox files/data coming from the appmasterconfig
# from master_configure. Catch the case where None is passed (as in tests)
if appmasterconfig:
inputsandbox += appmasterconfig.getSandboxFiles()
outputsandbox += appmasterconfig.getOutputSandboxFiles()
return unique(inputsandbox), unique(outputsandbox)
def prepare(self, app, appconfig, appmasterconfig, jobmasterconfig):
j = app.getJobObject()
script = self._create_exe_script(app)
c = StandardJobConfig(script, [File(script)], app.args,
j.outputsandbox, app.env)
dirac_script = DiracScript()
dirac_script.job_type = 'Job()'
#dirac_script.exe = DiracExe(app.exe,app.args)
dirac_script.exe = DiracExe('exe-script.py', [])
dirac_script.output_sandbox = j.outputsandbox[:]
if j.inputdata: dirac_script.inputdata = DiracInputData(j.inputdata)
if j.outputdata: dirac_script.outputdata = j.outputdata
c.script = dirac_script
return c
def getGangaModulesAsSandboxFiles(modules):
""" This returns a list of sandbox files corresponding to specified Ganga modules.
Ganga modules are placed in a well-known location in the sandbox.
"""
import inspect
import sys
from Ganga.Utility.files import remove_prefix
from Ganga.GPIDev.Lib.File import File
files = []
for m in modules:
fullpath = os.path.realpath(inspect.getsourcefile(m))
dir, fn = os.path.split(remove_prefix(fullpath, sys.path))
if os.path.join(dir, fn) == fullpath:
raise GangaIOError('Cannot find the prefix for %s' % fullpath)
files.append(File(fullpath, subdir=os.path.join(PYTHON_DIR, dir)))
return files
def _check_inputs(self):
"""Checks the validity of user's entries for GaudiPython schema"""
# Always check for None OR empty
#logger.info("self.module: %s" % str(self.module))
if isType(self.module, str):
self.module = File(self.module)
if self.module.name == None:
raise ApplicationConfigurationError(
"Application Module not requested")
elif self.module.name == "":
raise ApplicationConfigurationError(
"Application Module not requested")
else:
# Always check we've been given a FILE!
self.module.name = fullpath(self.module.name)
if not os.path.isfile(self.module.name):
msg = 'Module file %s not found.' % self.module.name
raise ApplicationConfigurationError(msg)
class PandaPilot(IApplication):
"""
PandaPilot application -- start a Panda Pilot on a remote CE
"""
_schema = Schema(
Version(1, 0), {
'exe':
SimpleItem(
defvalue=File(
os.path.join(os.path.dirname(__file__),
'runpilot3-script-stub.sh')),
typelist=['Ganga.GPIDev.Lib.File.File.File'],
doc=
'File object containing the pilot startup script. Do not modify unless you know what you are doing.'
),
'arg_pattern':
SimpleItem(
defvalue='-s %%QUEUE%% -h %%QUEUE%% -j false -u self',
typelist=['str'],
doc=
"Pilot argument pattern. Do not modify unless you know what you are doing."
),
'queue':
SimpleItem(
defvalue='',
typelist=['str'],
doc=
'The analysis queue to register the pilot on e.g. ANALY_TRIUMF'
)
})
_category = 'applications'
_name = 'PandaPilot'
_GUIPrefs = []
_GUIAdvancedPrefs = []
def __init__(self):
super(PandaPilot, self).__init__()
def configure(self, masterappconfig):
return (None, None)
def configure(self, masterappconfig):
from Ganga.GPI import TUTDIR
from Ganga.Core.exceptions import ApplicationConfigurationError
import os.path
## the prime number factorizer executable
self.exe = File(TUTDIR + '/Lib/primes/prime_factor.py')
prime_tables = []
try:
job = self.getJobObject()
prime_tables = job.inputdata.get_dataset()
except:
pass
## the arguments of the executable
## - arg1: the number
## - arg2: the SURL of the prime number lookup table
self.args = ['%s' % self.number] + prime_tables
self.inputs = []
self.outputs = ['*.dat']
self.envs = {}
return (None, None)
class Bender(GaudiBase):
"""The Bender application handler
The user specifies a module file (via Bender.module) which contains a
Bender python module and the number of events they want to run on
(via Bender.events). The user's module is then run on the data by
calling:
USERMODULE.configure(EventSelectorInput,FileCatalogCatalogs)
USERMODULE.run(NUMEVENTS)
"""
_name = 'Bender'
_category = 'applications'
_exportmethods = GaudiBase._exportmethods[:]
_exportmethods += ['prepare', 'unprepare']
_schema = GaudiBase._schema.inherit_copy()
docstr = 'The package the application belongs to (e.g. "Sim", "Phys")'
_schema.datadict['package'] = SimpleItem(defvalue=None,
typelist=['str', 'type(None)'],
doc=docstr)
docstr = 'The package where your top level requirements file is read ' \
'from. Can be written either as a path ' \
'\"Tutorial/Analysis/v6r0\" or in traditional notation ' \
'\"Analysis v6r0 Tutorial\"'
_schema.datadict['masterpackage'] = SimpleItem(
defvalue=None, typelist=['str', 'type(None)'], doc=docstr)
docstr = 'Extra options to be passed onto the SetupProject command '\
'used for configuring the environment. As an example '\
'setting it to \'--dev\' will give access to the DEV area. '\
'For full documentation of the available options see '\
'https://twiki.cern.ch/twiki/bin/view/LHCb/SetupProject'
_schema.datadict['setupProjectOptions'] = SimpleItem(
defvalue='', typelist=['str', 'type(None)'], doc=docstr)
docstr = 'The name of the module to import. A copy will be made ' \
'at submission time'
_schema.datadict['module'] = FileItem(preparable=1,
defvalue=File(),
doc=docstr)
docstr = 'The name of the Gaudi application (Bender)'
_schema.datadict['project'] = SimpleItem(preparable=1,
defvalue='Bender',
hidden=1,
protected=1,
typelist=['str'],
doc=docstr)
docstr = 'The number of events '
_schema.datadict['events'] = SimpleItem(defvalue=-1,
typelist=['int'],
doc=docstr)
docstr = 'Parameres for module '
_schema.datadict['params'] = SimpleItem(
defvalue={},
typelist=['dict', 'str', 'int', 'bool', 'float'],
doc=docstr)
_schema.version.major += 2
_schema.version.minor += 0
#def __init__(self):
# super(Bender, self).__init__()
def _get_default_version(self, gaudi_app):
return guess_version(self, gaudi_app)
def _auto__init__(self):
if (not self.appname) and (not self.project):
self.project = 'Bender' # default
if (not self.appname):
self.appname = self.project
self._init()
def _getshell(self):
import EnvironFunctions
return EnvironFunctions._getshell(self)
def prepare(self, force=False):
super(Bender, self).prepare(force)
self._check_inputs()
share_dir = os.path.join(
expandfilename(getConfig('Configuration')['gangadir']), 'shared',
getConfig('Configuration')['user'], self.is_prepared.name)
fillPackedSandbox([self.module],
os.path.join(
share_dir, 'inputsandbox',
'_input_sandbox_%s.tar' % self.is_prepared.name))
gzipFile(
os.path.join(share_dir, 'inputsandbox',
'_input_sandbox_%s.tar' % self.is_prepared.name),
os.path.join(share_dir, 'inputsandbox',
'_input_sandbox_%s.tgz' % self.is_prepared.name),
True)
# add the newly created shared directory into the metadata system if
# the app is associated with a persisted object
self.checkPreparedHasParent(self)
self.post_prepare()
logger.debug("Finished Preparing Application in %s" % share_dir)
def master_configure(self):
return (None, StandardJobConfig())
def configure(self, master_appconfig):
# self._configure()
modulename = split(self.module.name)[-1].split('.')[0]
script = """
from copy import deepcopy
from Gaudi.Configuration import *
importOptions('data.py')
import %s as USERMODULE
EventSelectorInput = deepcopy(EventSelector().Input)
FileCatalogCatalogs = deepcopy(FileCatalog().Catalogs)
EventSelector().Input=[]
FileCatalog().Catalogs=[]\n""" % modulename
script_configure = "USERMODULE.configure(EventSelectorInput,FileCatalogCatalogs%s)\n"
if self.params:
param_string = ",params=%s" % self.params
else:
param_string = ""
script_configure = script_configure % param_string
script += script_configure
script += "USERMODULE.run(%d)\n" % self.events
script += getXMLSummaryScript()
# add summary.xml
outputsandbox_temp = XMLPostProcessor._XMLJobFiles()
outputsandbox_temp += unique(self.getJobObject().outputsandbox)
outputsandbox = unique(outputsandbox_temp)
input_files = []
input_files += [FileBuffer('gaudipython-wrapper.py', script)]
logger.debug("Returning StandardJobConfig")
return (None,
StandardJobConfig(inputbox=input_files,
outputbox=outputsandbox))
def _check_inputs(self):
"""Checks the validity of user's entries for GaudiPython schema"""
# Always check for None OR empty
#logger.info("self.module: %s" % str(self.module))
if isType(self.module, str):
self.module = File(self.module)
if self.module.name == None:
raise ApplicationConfigurationError(
"Application Module not requested")
elif self.module.name == "":
raise ApplicationConfigurationError(
"Application Module not requested")
else:
# Always check we've been given a FILE!
self.module.name = fullpath(self.module.name)
if not os.path.isfile(self.module.name):
msg = 'Module file %s not found.' % self.module.name
raise ApplicationConfigurationError(msg)
def postprocess(self):
XMLPostProcessor.postprocess(self, logger)
def mergefiles(self, file_list, output_file):
#if no opts file is specified, then use version from installation
if self.merge_opts is None or not self.merge_opts.name:
self.merge_opts = File(self.selectOptionsFile(self.version))
if not os.path.exists(self.merge_opts.name):
msg = "The options file '%s' needed for merging does not exist."
raise MergerError(msg % self.merge_opts.name)
logger.info("Using the options file '%s'.", self.merge_opts.name)
#this is the bit specifing the files
output_opts = """
// the output file from the merge
InputCopyStream.Output = "DATAFILE='PFN:%s' TYP='POOL_ROOTTREE' OPT='REC'";
//the files to merge
EventSelector.Input = {""" % output_file
file_sep = ','
file_len = len(file_list)
for i in xrange(file_len):
file_name = file_list[i]
if i == (file_len - 1):
file_sep = '' #i.e. last entry does not have a comma
output_opts += """
"DATAFILE='PFN:%s' TYP='POOL_ROOTTREE' OPT='READ'"%s""" % (file_name, file_sep)
output_opts += """
};"""
#write this out to a file
opts_file_name = tempfile.mktemp('.opts')
try:
opts_file = file(opts_file_name,'w')
opts_file.write(output_opts)
finally:
opts_file.close()
if not os.path.exists(opts_file_name):
msg = "Failed to write tempory options file '%s' during merge"
raise MergerError(msg % opts_file_name)
#now run gaudirun via a script
shell_script = """#!/bin/sh
SP=`which SetupProject.sh`
if [ -n $SP ]; then
. SetupProject.sh --ignore-missing DaVinci %s
else
echo "Could not find the SetupProject.sh script. Your job will probably fail"
fi
gaudirun.py %s %s
exit $?
""" % (self.version, self.merge_opts.name, opts_file_name)
script_file_name = tempfile.mktemp('.sh')
try:
script_file = file(script_file_name,'w')
script_file.write(shell_script)
finally:
script_file.close()
return_code = subprocess.call(['/bin/sh',script_file_name])
if return_code != 0:
msg = 'The DSTMerger returned %i when calling gaudirun'
logger.warning(msg % return_code)
#finally clean up
os.unlink(script_file_name)
os.unlink(opts_file_name)
if not os.path.exists(output_file):
msg = "The output file '%s' was not created"
raise MergerError(msg % output_file)
class TestApplication(IPrepareApp):
_schema = Schema(
Version(1, 0),
{
'exe':
SimpleItem(defvalue='/usr/bin/env'),
'derived_value':
SimpleItem(defvalue='', typelist=['str']),
'sequence':
SimpleItem([], sequence=1, typelist=['str']),
'file_sequence':
FileItem(defvalue=[], sequence=1),
'optsfile':
FileItem(),
'modified':
SimpleItem(defvalue=0),
'nodefault_file_item':
FileItem(defvalue=File('a')),
'args':
SimpleItem(
defvalue=["Hello World"],
typelist=['str', 'Ganga.GPIDev.Lib.File.File'],
sequence=1,
doc=
"List of arguments for the executable. Arguments may be strings or File objects."
), # introduced for RTHandler compatibility
'env':
SimpleItem(
defvalue={},
doc='Environment'), # introduced for RTHandler compatibility
'fail':
SimpleItem(
defvalue='',
doc=
'Define the artificial runtime failures: "config", "prepare"'),
'postprocess_mark_as_failed':
SimpleItem(
defvalue=False,
doc=
'Update teh status of the job as failed in the postprocess step'
),
'is_prepared':
SimpleItem(
defvalue=None,
strict_sequence=0,
visitable=1,
copyable=1,
typelist=['type(None)', 'str', 'bool'],
protected=0,
comparable=1,
doc=
'Location of shared resources. Presence of this attribute implies the application has been prepared.'
),
'hash':
SimpleItem(defvalue=None, typelist=['type(None)', 'str'], hidden=1)
})
_name = 'TestApplication'
_exportmethods = ['modify', 'prepare']
def _auto__init__(self):
self.derived_value = 'This is an example of the derived property: ' + self.exe
# this is to test manually (visually) the logging level mechanism
#print("raw print: Hello from TestApplication")
#logger.debug('Hello from TestApplication')
#logger.info('Hello from TestApplication')
#logger.warning('Hello from TestApplication')
#logger.error('Hello from TestApplication')
#logger.critical('Hello from TestApplication')
def configure(self, masterappconfig):
if self.fail == 'config':
x = 'triggered failure during config'
raise Exception(x)
appcfg = ''
if self.fail == 'prepare':
appcfg += 'error'
return (None, appcfg)
def modify(self):
self.modified = 1
self._setDirty()
def postprocess(self):
if self.postprocess_mark_as_failed:
logger.info('postprocess: trying to mark the job as failed')
from Ganga.GPIDev.Adapters.IApplication import PostprocessStatusUpdate
raise PostprocessStatusUpdate("failed")
def postprocess_failed(self):
logger.info('postprocess_failed() called')
def preparejob(self, jobconfig, master_input_sandbox):
job = self.getJobObject()
# print str(job.backend_output_postprocess)
mon = job.getMonitoringService()
import Ganga.Core.Sandbox as Sandbox
from Ganga.GPIDev.Lib.File import File
from Ganga.Core.Sandbox.WNSandbox import PYTHON_DIR
import inspect
fileutils = File(inspect.getsourcefile(Ganga.Utility.files),
subdir=PYTHON_DIR)
sharedfiles = jobconfig.getSharedFiles()
subjob_input_sandbox = job.createPackedInputSandbox(
jobconfig.getSandboxFiles() + [fileutils])
appscriptpath = [jobconfig.getExeString()] + jobconfig.getArgStrings()
if self.nice:
appscriptpath = ['nice', '-n %d' % self.nice] + appscriptpath
if self.nice < 0:
logger.warning(
'increasing process priority is often not allowed, your job may fail due to this'
)
sharedoutputpath = job.getOutputWorkspace().getPath()
## FIXME DON'T just use the blind list here, request the list of files to be in the output from a method.
outputpatterns = jobconfig.outputbox
environment = dict() if jobconfig.env is None else jobconfig.env
import tempfile
workdir = tempfile.mkdtemp(dir=config['location'])
import inspect
script_location = os.path.join(
os.path.dirname(
os.path.abspath(inspect.getfile(inspect.currentframe()))),
'LocalHostExec.py.template')
from Ganga.GPIDev.Lib.File import FileUtils
script = FileUtils.loadScript(script_location, '')
script = script.replace('###INLINEMODULES###',
inspect.getsource(Sandbox.WNSandbox))
from Ganga.GPIDev.Lib.File.OutputFileManager import getWNCodeForOutputSandbox, getWNCodeForOutputPostprocessing, getWNCodeForDownloadingInputFiles, getWNCodeForInputdataListCreation
from Ganga.Utility.Config import getConfig
jobidRepr = repr(job.getFQID('.'))
script = script.replace(
'###OUTPUTSANDBOXPOSTPROCESSING###',
getWNCodeForOutputSandbox(job, ['stdout', 'stderr', '__syslog__'],
jobidRepr))
script = script.replace('###OUTPUTUPLOADSPOSTPROCESSING###',
getWNCodeForOutputPostprocessing(job, ''))
script = script.replace('###DOWNLOADINPUTFILES###',
getWNCodeForDownloadingInputFiles(job, ''))
script = script.replace('###CREATEINPUTDATALIST###',
getWNCodeForInputdataListCreation(job, ''))
script = script.replace('###APPLICATION_NAME###',
repr(getName(job.application)))
script = script.replace(
'###INPUT_SANDBOX###',
repr(subjob_input_sandbox + master_input_sandbox + sharedfiles))
script = script.replace('###SHAREDOUTPUTPATH###',
repr(sharedoutputpath))
script = script.replace('###APPSCRIPTPATH###', repr(appscriptpath))
script = script.replace('###OUTPUTPATTERNS###', str(outputpatterns))
script = script.replace('###JOBID###', jobidRepr)
script = script.replace('###ENVIRONMENT###', repr(environment))
script = script.replace('###WORKDIR###', repr(workdir))
script = script.replace('###INPUT_DIR###',
repr(job.getStringInputDir()))
self.workdir = workdir
script = script.replace('###GANGADIR###',
repr(getConfig('System')['GANGA_PYTHONPATH']))
wrkspace = job.getInputWorkspace()
scriptPath = wrkspace.writefile(FileBuffer('__jobscript__', script),
executable=1)
return scriptPath
def configure(self, masterappconfig):
'''It creates the SuperB job wrapper config file and manages application
string in Executable() method to be ready to launch the job wrapper.
It also configures the input and output sandbox. Such a method is called one
time per master job.'''
logger.debug('SBApp configure called.')
job = self._getParent()
masterjob = job._getParent()
# creating the configuration file needed by the job wrapper
config = ConfigParser.RawConfigParser()
config.add_section('REST')
config.set('REST', 'PROXYVAR', 'X509_USER_PROXY')
config.set('REST', 'NAME', 'restsbk5')
config.set('REST', 'HOSTNAME', 'bbr-serv09.cr.cnaf.infn.it')
config.set('REST', 'PORT', '8443')
config.add_section('EXPORT_VARS')
config.set('EXPORT_VARS', 'LCG_GFAL_INFOSYS',
'egee-bdii.cnaf.infn.it:2170')
config.add_section('OPTIONS')
config.set('OPTIONS', 'MODULENAME', 'Analysis')
# setting PRODSERIES and REQUESTNAME as workaround (this parameters are useless for analysis)
# TODO: fix asap modifying the job wrapper
config.set('OPTIONS', 'PRODSERIES', '2011_Full_grid_test')
config.set('OPTIONS', 'REQUESTNAME',
'6b88b34759fab9e2d7e8efe73636dc35')
config.set('OPTIONS', 'OS_ARCH', self.os_arch)
config.add_section('SOFTWARE')
config.set('SOFTWARE', 'EXEPATH', self.executable)
(head, tail) = os.path.split(self.sw_archive)
config.set('SOFTWARE', 'SWPATH', tail)
config.add_section('INPUT')
config.set('INPUT', 'INPUTMODE', job.inputdata.input_mode)
if job.inputdata.input_mode == 'none':
path = ''
elif job.inputdata.input_mode == 'list':
(head, path) = os.path.split(job.inputsandbox[0].name)
elif job.inputdata.input_mode == 'dir':
path = job.inputdata.input_path[0]
else:
raise ApplicationConfigurationError('input_mode not recognized.')
config.set('INPUT', 'INPUTPATH', path)
config.add_section('ANALYSIS')
config.set('ANALYSIS', 'SUBJOB_ID', job.id)
config.set('ANALYSIS', 'DATASET', self.temp_dataset)
if isinstance(job.inputdata, SBInputDataset.SBInputPersonalProduction):
config.set('ANALYSIS', 'TAG', job.inputdata.soft_version)
config.set('ANALYSIS', 'SIMULATION', job.inputdata.session)
config.add_section('SITE')
config.set('SITE', 'USELOCALNAME', 'true')
config.add_section('OUTPUT')
# <identity_certificato>/<dataset_id>/<date_idjob_usertaballsenzaestensione_jobname>/output/subjobs.id_nomefile.root
dirTree = ("%s/%%DATASET_ID%%/%s_%s_%s" %
(self.user_id, self.now, masterjob.id, self.filename))
if masterjob.name != '':
dirTree += ("_%s" % masterjob.name)
dirTree += '/output'
config.set('OUTPUT', 'LFN',
'lfn:/grid/superbvo.org/analysis/' + dirTree)
config.set('OUTPUT', 'OUTPUTPATH', dirTree)
config.set('OUTPUT', 'LOG', '')
config.add_section('OUTPUT_DATASET')
if isinstance(job.outputdata, SBOutputDataset.SBOutputDataset):
for key, value in job.outputdata.pairs.items():
config.set('OUTPUT_DATASET', key, value)
config.add_section('TARGETSITE')
config.set('TARGETSITE', 'SITENAME',
getConfig('SuperB')['submission_site'])
config.set('TARGETSITE', 'USELOCALNAME', 'false')
fileConf = os.path.join(job.inputdir, "file_%d.conf" % job.id)
f = open(fileConf, "w")
config.write(f)
f.close()
severus_dir = getConfig('SuperB')['severus_dir']
severus = os.path.join(severus_dir, 'severus.tgz')
severus_wrapper = os.path.join(severus_dir, 'severus_wrapper.sh')
# updating the information of application, inputsandbox and outputsandbox objects
# (the job wrapper, the configuration file, the user software, run number and
# verbosity flag as command line parameters, the output log and a list of output files)
job.application = Executable()
job.application.exe = File(severus_wrapper)
# severus verbose (the string 'test' as third parameter)
job.application.args = [File(fileConf), 10000000, 'test']
# severus quiet (missing third parameter)
#job.application.args = [File(fileConf), 10000000]
job.inputsandbox.append(File(severus))
job.inputsandbox.append(File(self.sw_archive))
job.outputsandbox.append('severus.log')
job.outputsandbox.append('output_files.txt')
return (None, None)
def getDefaultScript():
name = os.path.join(
os.path.dirname(
os.path.abspath(inspect.getfile(inspect.currentframe()))),
'defaultRootScript.C')
return File(name=name)
def prestagesandbox(self, jobconfig):
job = self.getJobObject()
mon = job.getMonitoringService()
inw = job.getInputWorkspace()
sandbox_files = jobconfig.getSandboxFiles()
logger.info("pre-staging files for saga job with id: %s", job.id)
## Compression is ENABLED. All input sandbox files are archived
## into an uncompressed tarball which is then transfered to the
## execution host. This speeds up things a lot in many scenarios.
##
if self.enable_compression:
logger.info(" * adding %s user defined files to input sandbox",
len(sandbox_files))
import Ganga.Core.Sandbox as Sandbox
from Ganga.GPIDev.Lib.File import File
from Ganga.Core.Sandbox.WNSandbox import PYTHON_DIR
import inspect
fileutils = File(inspect.getsourcefile(Ganga.Utility.files),
subdir=PYTHON_DIR)
compressed_input_sandbox = job.createPackedInputSandbox(
jobconfig.getSandboxFiles() + [fileutils])
try:
for f in compressed_input_sandbox:
archive_url = saga.url("file://localhost/" + f)
logger.info(
" * copying %s -> %s ", archive_url.url,
self.filesystem_url + "/" + self.workdir_uuid + "/")
sf = saga.filesystem.file(archive_url.url)
sf.copy(
self.filesystem_url + "/" + self.workdir_uuid + "/",
saga.filesystem.Overwrite)
except saga.exception as e:
logger.error('exception caught while transfering file: %s',
e.get_all_exceptions())
job.updateStatus("failed")
return False
## Compression is DISABLED. All input sandbox files are transfered
## one-by-one to the execution host. This can be very slow, especially
## if SAGA is using the Globus GridFTP adaptor.
else:
logger.info("prestaging %s input sandbox files",
len(sandbox_files))
for f in sandbox_files:
try:
source = saga.url(f.name)
if (len(source.scheme) == 0):
source.scheme = "file"
if (len(source.host) == 0):
source.host = "localhost"
if (f.subdir != '.'): # create subdirectory & copy
target = saga.url(self.filesystem_url + "/" +
self.workdir_uuid + "/" + f.subdir +
"/")
sd = saga.filesystem.directory(target,
saga.filesystem.Create)
sf = saga.filesystem.file(source)
logger.info(" * copying %s -> %s ", source, target)
sf.copy(target,
saga.filesystem.Overwrite) # copy the file
else: # copy to remote work dir
logger.info(
" * copying %s -> %s ", source,
self.filesystem_url + "/" + self.workdir_uuid +
"/")
sf = saga.filesystem.file(source)
sf.copy(
self.filesystem_url + "/" + self.workdir_uuid +
"/", saga.filesystem.Overwrite)
except saga.exception as e:
logger.error('exception caught while transfering file: %s',
e.get_all_exceptions())
job.updateStatus("failed")
return False
##
## copy the job script
##
try:
jobscript_path = saga.url("file://localhost/" + inw.getPath() +
"/__jobscript__")
if os.path.exists(jobscript_path.path):
logger.info(
" * copying %s -> %s ", jobscript_path.url,
self.filesystem_url + "/" + self.workdir_uuid + "/")
sf = saga.filesystem.file(jobscript_path.url)
sf.copy(self.filesystem_url + "/" + self.workdir_uuid + "/",
saga.filesystem.Overwrite)
except saga.exception as e:
logger.error('exception caught while transfering file: %s',
e.get_all_exceptions())
job.updateStatus("failed")
return False
return True
def preparejob(self, jobconfig, master_job_sandbox):
'''Prepare the JDL'''
script = self.__jobWrapperTemplate__()
job = self.getJobObject()
inpw = job.getInputWorkspace()
wrapperlog = '__jobscript__.log'
import Ganga.Core.Sandbox as Sandbox
# FIXME: check what happens if 'stdout','stderr' are specified here
script = script.replace('###OUTPUTSANDBOX###',
repr(jobconfig.outputbox))
script = script.replace('###APPLICATION_NAME###',
getName(job.application))
script = script.replace('###APPLICATIONEXEC###',
repr(jobconfig.getExeString()))
script = script.replace('###APPLICATIONARGS###',
repr(jobconfig.getArguments()))
from Ganga.GPIDev.Lib.File.OutputFileManager import getWNCodeForOutputPostprocessing, getWNCodeForDownloadingInputFiles
script = script.replace('###OUTPUTUPLOADSPOSTPROCESSING###',
getWNCodeForOutputPostprocessing(job, ' '))
script = script.replace('###DOWNLOADINPUTFILES###',
getWNCodeForDownloadingInputFiles(job, ' '))
if jobconfig.env:
script = script.replace('###APPLICATIONENVS###',
repr(jobconfig.env))
else:
script = script.replace('###APPLICATIONENVS###', repr({}))
script = script.replace('###WRAPPERLOG###', repr(wrapperlog))
import inspect
script = script.replace('###INLINEMODULES###',
inspect.getsource(Sandbox.WNSandbox))
mon = job.getMonitoringService()
self.monInfo = None
# set the monitoring file by default to the stdout
if isinstance(self.monInfo, dict):
self.monInfo['remotefile'] = 'stdout'
# try to print out the monitoring service information in debug mode
try:
logger.debug('job info of monitoring service: %s' %
str(self.monInfo))
except:
pass
# prepare input/output sandboxes
import Ganga.Utility.files
from Ganga.GPIDev.Lib.File import File
from Ganga.Core.Sandbox.WNSandbox import PYTHON_DIR
import inspect
fileutils = File(inspect.getsourcefile(Ganga.Utility.files),
subdir=PYTHON_DIR)
packed_files = jobconfig.getSandboxFiles() + [fileutils]
sandbox_files = job.createPackedInputSandbox(packed_files)
# sandbox of child jobs should include master's sandbox
sandbox_files.extend(master_job_sandbox)
# check the input file size and pre-upload larger inputs to the iocache
lfc_host = ''
input_sandbox_uris = []
input_sandbox_names = []
ick = True
max_prestaged_fsize = 0
for f in sandbox_files:
idx = self.__check_and_prestage_inputfile__(f)
if not idx:
logger.error('input sandbox preparation failed: %s' % f)
ick = False
break
else:
if idx['lfc_host']:
lfc_host = idx['lfc_host']
if idx['remote']:
abspath = os.path.abspath(f)
fsize = os.path.getsize(abspath)
if fsize > max_prestaged_fsize:
max_prestaged_fsize = fsize
input_sandbox_uris.append(
idx['remote'][os.path.basename(f)])
input_sandbox_names.append(os.path.basename(
urlparse(f)[2]))
if idx['local']:
input_sandbox_uris += idx['local']
input_sandbox_names.append(os.path.basename(f))
if not ick:
logger.error('stop job submission')
return None
# determin the lcg-cp timeout according to the max_prestaged_fsize
# - using the assumption of 1 MB/sec.
max_prestaged_fsize = 0
lfc_host = ''
transfer_timeout = config['SandboxTransferTimeout']
predict_timeout = int(math.ceil(max_prestaged_fsize / 1000000.0))
if predict_timeout > transfer_timeout:
transfer_timeout = predict_timeout
if transfer_timeout < 60:
transfer_timeout = 60
script = script.replace('###TRANSFERTIMEOUT###',
'%d' % transfer_timeout)
# update the job wrapper with the inputsandbox list
script = script.replace(
'###INPUTSANDBOX###',
repr({
'remote': {},
'local': input_sandbox_names
}))
# write out the job wrapper and put job wrapper into job's inputsandbox
scriptPath = inpw.writefile(FileBuffer(
'__jobscript_%s__' % job.getFQID('.'), script),
executable=1)
input_sandbox = input_sandbox_uris + [scriptPath]
for isb in input_sandbox:
logger.debug('ISB URI: %s' % isb)
# compose output sandbox to include by default the following files:
# - gzipped stdout (transferred only when the JobLogHandler is WMS)
# - gzipped stderr (transferred only when the JobLogHandler is WMS)
# - __jobscript__.log (job wrapper's log)
output_sandbox = [wrapperlog]
from Ganga.GPIDev.Lib.File.OutputFileManager import getOutputSandboxPatterns
for outputSandboxPattern in getOutputSandboxPatterns(job):
output_sandbox.append(outputSandboxPattern)
if config['JobLogHandler'] in ['WMS']:
output_sandbox += ['stdout.gz', 'stderr.gz']
if len(jobconfig.outputbox):
output_sandbox += [Sandbox.OUTPUT_TARBALL_NAME]
# compose ARC XRSL
xrsl = {
#'VirtualOrganisation' : config['VirtualOrganisation'],
'executable': os.path.basename(scriptPath),
'environment': {
'GANGA_LCG_VO': config['VirtualOrganisation'],
'GANGA_LOG_HANDLER': config['JobLogHandler'],
'LFC_HOST': lfc_host
},
#'stdout' : 'stdout',
#'stderr' : 'stderr',
'inputFiles': input_sandbox,
'outputFiles': output_sandbox,
#'OutputSandboxBaseDestURI': 'gsiftp://localhost'
}
xrsl['environment'].update({'GANGA_LCG_CE': self.CE})
#xrsl['Requirements'] = self.requirements.merge(jobconfig.requirements).convert()
# if self.jobtype.upper() in ['NORMAL','MPICH']:
#xrsl['JobType'] = self.jobtype.upper()
# if self.jobtype.upper() == 'MPICH':
#xrsl['Requirements'].append('(other.GlueCEInfoTotalCPUs >= NodeNumber)')
# xrsl['Requirements'].append('Member("MPICH",other.GlueHostApplicationSoftwareRunTimeEnvironment)')
#xrsl['NodeNumber'] = self.requirements.nodenumber
# else:
# logger.warning('JobType "%s" not supported' % self.jobtype)
# return
# additional settings from the job
if jobconfig.env:
xrsl['environment'].update(jobconfig.env)
xrslText = Grid.expandxrsl(xrsl)
# append any additional requirements from the requirements object
xrslText += '\n'.join(self.requirements.other)
logger.debug('subjob XRSL: %s' % xrslText)
return inpw.writefile(FileBuffer('__xrslfile__', xrslText))
def prepare(self, app, appsubconfig, appmasterconfig, jobmasterconfig):
inputsandbox, outputsandbox = sandbox_prepare(app, appsubconfig,
appmasterconfig,
jobmasterconfig)
input_data, parametricinput_data = dirac_inputdata(app)
# outputdata, outputdata_path = dirac_ouputdata(app)
job = stripProxy(app).getJobObject()
outputfiles = [
this_file for this_file in job.outputfiles
if isType(this_file, DiracFile)
]
commandline = []
commandline.append(app.exe)
if isType(app.exe, File):
#logger.info("app: %s" % str(app.exe.name))
#fileName = os.path.join(get_share_path(app), os.path.basename(app.exe.name))
#logger.info("EXE: %s" % str(fileName))
#inputsandbox.append(File(name=fileName))
inputsandbox.append(app.exe)
commandline[0] = os.path.join('.', os.path.basename(app.exe.name))
commandline.extend([str(arg) for arg in app.args])
logger.debug('Command line: %s: ', commandline)
#exe_script_path = os.path.join(job.getInputWorkspace().getPath(), "exe-script.py")
exe_script_name = 'exe-script.py'
logger.info("Setting Command to be: '%s'" % repr(commandline))
inputsandbox.append(
FileBuffer(
name=exe_script_name,
contents=script_generator(
exe_script_template(),
#remove_unreplaced = False,
# ,
COMMAND=repr(commandline),
OUTPUTFILESINJECTEDCODE=getWNCodeForOutputPostprocessing(
job, ' ')),
executable=True))
contents = script_generator(
exe_script_template(),
COMMAND=repr(commandline),
OUTPUTFILESINJECTEDCODE=getWNCodeForOutputPostprocessing(
job, ' '))
#logger.info("Script is: %s" % str(contents))
from os.path import abspath, expanduser
for this_file in job.inputfiles:
if isinstance(this_file, LocalFile):
for name in this_file.getFilenameList():
inputsandbox.append(File(abspath(expanduser(name))))
elif isinstance(this_file, DiracFile):
name = this_file.lfn
if isinstance(input_data, list):
input_data.append(name)
else:
input_data = [name]
dirac_outputfiles = dirac_outputfile_jdl(outputfiles,
config['RequireDefaultSE'])
# NOTE special case for replicas: replicate string must be empty for no
# replication
dirac_script = script_generator(
diracAPI_script_template(),
DIRAC_IMPORT='from DIRAC.Interfaces.API.Dirac import Dirac',
DIRAC_JOB_IMPORT='from DIRAC.Interfaces.API.Job import Job',
DIRAC_OBJECT='Dirac()',
JOB_OBJECT='Job()',
NAME=mangle_job_name(app),
# os.path.basename(exe_script_path),
EXE=exe_script_name,
# ' '.join([str(arg) for arg in app.args]),
EXE_ARG_STR='',
EXE_LOG_FILE='Ganga_Executable.log',
ENVIRONMENT=None, # app.env,
INPUTDATA=input_data,
PARAMETRIC_INPUTDATA=parametricinput_data,
OUTPUT_SANDBOX=API_nullifier(outputsandbox),
OUTPUTFILESSCRIPT=dirac_outputfiles,
OUTPUT_PATH="", # job.fqid,
SETTINGS=diracAPI_script_settings(app),
DIRAC_OPTS=job.backend.diracOpts,
REPLICATE='True' if config['ReplicateOutputData'] else '',
# leave the sandbox for altering later as needs
# to be done in backend.submit to combine master.
# Note only using 2 #s as auto-remove 3
INPUT_SANDBOX='##INPUT_SANDBOX##')
#logger.info("dirac_script: %s" % dirac_script)
#logger.info("inbox: %s" % str(unique(inputsandbox)))
#logger.info("outbox: %s" % str(unique(outputsandbox)))
return StandardJobConfig(dirac_script,
inputbox=unique(inputsandbox),
outputbox=unique(outputsandbox))
def preparejob(self, jobconfig, master_input_sandbox):
"""Method for preparing job script"""
job = self.getJobObject()
from Ganga.GPIDev.Lib.File import File
from Ganga.Core.Sandbox.WNSandbox import PYTHON_DIR
import Ganga.Utility.files
import inspect
fileutils = File(inspect.getsourcefile(Ganga.Utility.files),
subdir=PYTHON_DIR)
inputfiles = jobconfig.getSandboxFiles() + [fileutils]
inbox = job.createPackedInputSandbox(inputfiles)
inbox.extend(master_input_sandbox)
inpDir = job.getInputWorkspace(create=True).getPath()
outDir = job.getOutputWorkspace(create=True).getPath()
workdir = tempfile.mkdtemp()
self.workdir = workdir
exeString = jobconfig.getExeString()
argList = jobconfig.getArgStrings()
argString = " ".join(map(lambda x: " %s " % x, argList))
outputSandboxPatterns = jobconfig.outputbox
patternsToZip = []
wnCodeForPostprocessing = ''
wnCodeToDownloadInputFiles = ''
if (len(job.outputfiles) > 0):
from Ganga.GPIDev.Lib.File.OutputFileManager import getOutputSandboxPatternsForInteractive, getWNCodeForOutputPostprocessing
(outputSandboxPatterns,
patternsToZip) = getOutputSandboxPatternsForInteractive(job)
wnCodeForPostprocessing = 'def printError(message):pass\ndef printInfo(message):pass' + \
getWNCodeForOutputPostprocessing(job, '')
all_inputfiles = [this_file for this_file in job.inputfiles]
if job.master is not None:
all_inputfiles.extend(
[this_file for this_file in job.master.inputfiles])
wnCodeToDownloadInputFiles = ''
if (len(all_inputfiles) > 0):
from Ganga.GPIDev.Lib.File.OutputFileManager import outputFilePostProcessingOnWN
for inputFile in all_inputfiles:
inputfileClassName = getName(inputFile)
logger.debug("name: %s" % inputfileClassName)
logger.debug(
"result: %s" %
str(outputFilePostProcessingOnWN(job, inputfileClassName)))
if outputFilePostProcessingOnWN(job, inputfileClassName):
inputFile.processWildcardMatches()
if inputFile.subfiles:
getfromFile = False
for subfile in inputFile.subfiles:
wnCodeToDownloadInputFiles += subfile.getWNScriptDownloadCommand(
'')
else:
getfromFile = True
else:
getFromFile = True
if getFromFile:
wnCodeToDownloadInputFiles += inputFile.getWNScriptDownloadCommand(
'')
wnCodeToDownloadInputData = ''
if job.inputdata and (len(job.inputdata) > 0):
from Ganga.GPIDev.Lib.File.OutputFileManager import getWNCodeForDownloadingInputFiles
wnCodeToDownloadInputData = getWNCodeForDownloadingInputFiles(
job, '')
import inspect
replace_dict = {
'###CONSTRUCT_TIME###': (time.strftime("%c")),
'###WNSANDBOX_SOURCE###':
inspect.getsource(Sandbox.WNSandbox),
'###GANGA_PYTHONPATH###':
getConfig("System")["GANGA_PYTHONPATH"],
'###OUTPUTDIR###':
outDir,
'###WORKDIR###':
workdir,
'###IN_BOX###':
inbox,
'###WN_INPUTFILES###':
wnCodeToDownloadInputFiles,
'###WN_INPUTDATA###':
wnCodeToDownloadInputData,
'###JOBCONFIG_ENV###':
jobconfig.env if jobconfig.env is not None else dict(),
'###EXE_STRING###':
exeString,
'###ARG_STRING###':
argString,
'###WN_POSTPROCESSING###':
wnCodeForPostprocessing,
'###PATTERNS_TO_ZIP###':
patternsToZip,
'###OUTPUT_SANDBOX_PATTERNS###':
outputSandboxPatterns
}
script_location = os.path.join(
os.path.dirname(
os.path.abspath(inspect.getfile(inspect.currentframe()))),
'InteractiveScriptTemplate.py.template')
from Ganga.GPIDev.Lib.File import FileUtils
commandString = FileUtils.loadScript(script_location, '')
for k, v in replace_dict.iteritems():
commandString = commandString.replace(str(k), str(v))
return job.getInputWorkspace().writefile(FileBuffer(
"__jobscript__", commandString),
executable=1)
def preparejob(self, jobconfig, master_input_sandbox):
job = self.getJobObject()
mon = job.getMonitoringService()
import Ganga.Core.Sandbox as Sandbox
from Ganga.GPIDev.Lib.File import File
from Ganga.Core.Sandbox.WNSandbox import PYTHON_DIR
import inspect
fileutils = File( inspect.getsourcefile(Ganga.Utility.files), subdir=PYTHON_DIR )
subjob_input_sandbox = job.createPackedInputSandbox(jobconfig.getSandboxFiles() + [ fileutils ] )
appscriptpath = [jobconfig.getExeString()] + jobconfig.getArgStrings()
sharedoutputpath = job.getOutputWorkspace().getPath()
## FIXME Check this isn't a GangaList
outputpatterns = jobconfig.outputbox
environment = jobconfig.env if not jobconfig.env is None else {}
import inspect
script_location = os.path.join(os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))),
'BatchScriptTemplate.py')
from Ganga.GPIDev.Lib.File import FileUtils
text = FileUtils.loadScript(script_location, '')
import Ganga.Core.Sandbox as Sandbox
import Ganga.Utility as Utility
from Ganga.Utility.Config import getConfig
from Ganga.GPIDev.Lib.File.OutputFileManager import getWNCodeForOutputSandbox, getWNCodeForOutputPostprocessing, getWNCodeForDownloadingInputFiles
jobidRepr = repr(self.getJobObject().getFQID('.'))
replace_dict = {
'###OUTPUTSANDBOXPOSTPROCESSING###' : getWNCodeForOutputSandbox(job, ['__syslog__'], jobidRepr),
'###OUTPUTUPLOADSPOSTPROCESSING###' : getWNCodeForOutputPostprocessing(job, ''),
'###DOWNLOADINPUTFILES###' : getWNCodeForDownloadingInputFiles(job, ''),
'###INLINEMODULES###' : inspect.getsource(Sandbox.WNSandbox),
'###INLINEHOSTNAMEFUNCTION###' : inspect.getsource(Utility.util.hostname),
'###APPSCRIPTPATH###' : repr(appscriptpath),
#'###SHAREDINPUTPATH###' : repr(sharedinputpath)),
'###INPUT_SANDBOX###' : repr(subjob_input_sandbox + master_input_sandbox),
'###SHAREDOUTPUTPATH###' : repr(sharedoutputpath),
'###OUTPUTPATTERNS###' : repr(outputpatterns),
'###JOBID###' : jobidRepr,
'###ENVIRONMENT###' : repr(environment),
'###PREEXECUTE###' : self.config['preexecute'],
'###POSTEXECUTE###' : self.config['postexecute'],
'###JOBIDNAME###' : self.config['jobid_name'],
'###QUEUENAME###' : self.config['queue_name'],
'###HEARTBEATFREQUENCE###' : self.config['heartbeat_frequency'],
'###INPUT_DIR###' : repr(job.getStringInputDir()),
'###GANGADIR###' : repr(getConfig('System')['GANGA_PYTHONPATH'])
}
for k, v in replace_dict.iteritems():
text = text.replace(str(k), str(v))
logger.debug('subjob input sandbox %s ', subjob_input_sandbox)
logger.debug('master input sandbox %s ', master_input_sandbox)
from Ganga.GPIDev.Lib.File import FileBuffer
return job.getInputWorkspace().writefile(FileBuffer('__jobscript__', text), executable=1)