def getWNInjectedScript(self, outputFiles, indent, patternsToZip, postProcessLocationsFP):
"""
Returns script that have to be injected in the jobscript for postprocessing on the WN
"""
import inspect
script_path = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
script_location = os.path.join( script_path, 'uploadScript.py')
from Ganga.GPIDev.Lib.File import FileUtils
upload_script = FileUtils.loadScript(script_location, '')
WNscript_location = os.path.join( script_path, 'WNInjectTemplate.py' )
script = FileUtils.loadScript(WNscript_location, '###INDENT###')
selfConstructedLFNs = False
if self.remoteDir == '' and self.lfn == '':
import datetime
t = datetime.datetime.now()
this_date = t.strftime("%H.%M_%A_%d_%B_%Y")
self.lfn = os.path.join(configDirac['DiracLFNBase'], 'GangaFiles_%s' % this_date)
selfConstructedLFNs = True
if self.remoteDir == '' and self.lfn != '':
self.remoteDir = configDirac['DiracLFNBase']
if self.remoteDir[:4] == 'LFN:':
lfn_base = self.remoteDir[4:]
else:
lfn_base = self.remoteDir
for this_file in outputFiles:
isCompressed = this_file.namePattern in patternsToZip
if not regex.search(this_file.namePattern) is None:
script += self._WN_wildcard_script(this_file.namePattern, lfn_base, str(isCompressed))
else:
script += '###INDENT###print("Uploading: %s as: %s")\n' % (this_file.namePattern, str(os.path.join(lfn_base, this_file.namePattern)))
script += '###INDENT###processes.append(uploadFile("%s", "%s", %s))\n' % (this_file.namePattern, lfn_base, str(isCompressed))
if stripProxy(self)._parent is not None and stripProxy(self).getJobObject() and getName(stripProxy(self).getJobObject().backend) != 'Dirac':
script_env = self._getDiracEnvStr()
else:
script_env = str(None)
script = '\n'.join([str('###INDENT###' + str(line)) for line in script.split('\n')])
replace_dict = {'###UPLOAD_SCRIPT###' : upload_script,
'###STORAGE_ELEMENTS###' : str(configDirac['allDiracSE']),
'###INDENT###' : indent,
'###LOCATIONSFILE###' : postProcessLocationsFP,
'###DIRAC_ENV###' : script_env}
for k, v in replace_dict.iteritems():
script = script.replace(str(k), str(v))
return script
def getWNInjectedScript(self, outputFiles, indent, patternsToZip, postProcessLocationsFP):
"""
Returns script that have to be injected in the jobscript for postprocessing on the WN
"""
script_path = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
script_location = os.path.join( script_path, 'uploadScript.py')
from Ganga.GPIDev.Lib.File import FileUtils
upload_script = FileUtils.loadScript(script_location, '')
WNscript_location = os.path.join( script_path, 'WNInjectTemplate.py' )
script = FileUtils.loadScript(WNscript_location, '')
if not self.remoteDir:
try:
job = self.getJobObject()
lfn_folder = os.path.join("GangaUploadedFiles", "GangaJob_%s" % job.getFQID('.'))
except AssertionError:
t = datetime.datetime.now()
this_date = t.strftime("%H.%M_%A_%d_%B_%Y")
lfn_folder = os.path.join("GangaUploadedFiles", 'GangaFiles_%s' % this_date)
self.lfn = os.path.join(DiracFile.diracLFNBase(), lfn_folder, self.namePattern)
if self.remoteDir == '':
self.remoteDir = DiracFile.diracLFNBase()
if self.remoteDir[:4] == 'LFN:':
lfn_base = self.remoteDir[4:]
else:
lfn_base = self.remoteDir
for this_file in outputFiles:
isCompressed = this_file.namePattern in patternsToZip
if not regex.search(this_file.namePattern) is None:
script += self._WN_wildcard_script(this_file.namePattern, lfn_base, str(isCompressed))
else:
script += '###INDENT###print("Uploading: %s as: %s")\n' % (this_file.namePattern, str(os.path.join(lfn_base, this_file.namePattern)))
script += '###INDENT###processes.append(uploadFile("%s", "%s", %s))\n' % (this_file.namePattern, lfn_base, str(isCompressed))
if stripProxy(self)._parent is not None and stripProxy(self).getJobObject() and getName(stripProxy(self).getJobObject().backend) != 'Dirac':
script_env = self._getDiracEnvStr()
else:
script_env = str(None)
script = '\n'.join([str('###INDENT###' + str(line)) for line in script.split('\n')])
replace_dict = {'###UPLOAD_SCRIPT###' : upload_script,
'###STORAGE_ELEMENTS###' : str(configDirac['allDiracSE']),
'###INDENT###' : indent,
'###LOCATIONSFILE###' : postProcessLocationsFP,
'###DIRAC_ENV###' : script_env}
for k, v in replace_dict.iteritems():
script = script.replace(str(k), str(v))
return script
def getWNInjectedScript(self, outputFiles, indent, patternsToZip, postProcessLocationsFP):
"""
Returns script that have to be injected in the jobscript for postprocessing on the WN
"""
script_path = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
script_location = os.path.join( script_path, 'uploadScript.py.template')
upload_script = FileUtils.loadScript(script_location, '')
WNscript_location = os.path.join( script_path, 'WNInjectTemplate.py.template' )
script = FileUtils.loadScript(WNscript_location, '')
if not self.remoteDir:
try:
job = self.getJobObject()
lfn_folder = os.path.join("GangaJob_%s" % job.getFQID('.'), "OutputFiles")
except AssertionError:
t = datetime.datetime.now()
this_date = t.strftime("%H.%M_%A_%d_%B_%Y")
lfn_folder = os.path.join('GangaFiles_%s' % this_date)
lfn_base = os.path.join(DiracFile.diracLFNBase(self.credential_requirements), lfn_folder)
else:
lfn_base = oa.path.join(DiracFile.diracLFNBase(self.credential_requirements), self.remoteDir)
for this_file in outputFiles:
isCompressed = this_file.namePattern in patternsToZip
if not regex.search(this_file.namePattern) is None:
script += self._WN_wildcard_script(this_file.namePattern, lfn_base, str(isCompressed))
else:
script += '###INDENT###print("Uploading: %s as: %s")\n' % (this_file.namePattern, str(os.path.join(lfn_base, this_file.namePattern)))
script += '###INDENT###processes.append(uploadFile("%s", "%s", %s))\n' % (this_file.namePattern, lfn_base, str(isCompressed))
if stripProxy(self)._parent is not None and stripProxy(self).getJobObject() and getName(stripProxy(self).getJobObject().backend) != 'Dirac':
script_env = self._getDiracEnvStr()
else:
script_env = str(None)
script = '\n'.join([str('###INDENT###' + str(line)) for line in script.split('\n')])
replace_dict = {'###UPLOAD_SCRIPT###' : upload_script,
'###STORAGE_ELEMENTS###' : str(configDirac['allDiracSE']),
'###INDENT###' : indent,
'###LOCATIONSFILE###' : postProcessLocationsFP,
'###DIRAC_ENV###' : script_env}
for k, v in replace_dict.iteritems():
script = script.replace(str(k), str(v))
return script
def getWNInjectedScript(self, outputFiles, indent, patternsToZip,
postProcessLocationsFP):
"""
Returns script that have to be injected in the jobscript for postprocessing on the WN
"""
lcgCommands = []
for outputFile in outputFiles:
lcgCommands.append('lcgse %s %s %s' %
(outputFile.namePattern, outputFile.lfc_host,
outputFile.getUploadCmd()))
logger.debug("OutputFile (%s) cmd for WN script is: %s" %
(outputFile.namePattern, outputFile.getUploadCmd()))
import inspect
script_location = os.path.join(
os.path.dirname(
os.path.abspath(inspect.getfile(inspect.currentframe()))),
'scripts/LCGSEFileWNScript.py')
from Ganga.GPIDev.Lib.File import FileUtils
script = FileUtils.loadScript(script_location, '###INDENT###')
script = script.replace('###LCGCOMMANDS###', str(lcgCommands))
script = script.replace('###PATTERNSTOZIP###', str(patternsToZip))
script = script.replace('###INDENT###', indent)
script = script.replace('###POSTPROCESSLOCATIONSFP###',
postProcessLocationsFP)
return script
def getWNScriptDownloadCommand(self, indent):
script_location = os.path.join(
os.path.dirname(
os.path.abspath(inspect.getfile(inspect.currentframe()))),
'downloadScript.py.template')
download_script = FileUtils.loadScript(script_location, '')
script = """\n
download_script='''\n###DOWNLOAD_SCRIPT###'''
import subprocess
dirac_env=###DIRAC_ENV###
subprocess.Popen('''python -c "import sys\nexec(sys.stdin.read())"''', shell=True, env=dirac_env, stdin=subprocess.PIPE).communicate(download_script)
"""
script = '\n'.join(
[str(indent + str(line)) for line in script.split('\n')])
replace_dict = {
'###DOWNLOAD_SCRIPT###': download_script,
'###DIRAC_ENV###': self._getDiracEnvStr(),
'###LFN###': self.lfn
}
for k, v in replace_dict.iteritems():
script = script.replace(str(k), str(v))
return script
def getWNInjectedScript(self, outputFiles, indent, patternsToZip, postProcessLocationsFP):
"""
Returns script that have to be injected in the jobscript for postprocessing on the WN
"""
lcgCommands = []
for outputFile in outputFiles:
lcgCommands.append('lcgse %s %s %s' % (
outputFile.namePattern, outputFile.lfc_host, outputFile.getUploadCmd()))
logger.debug("OutputFile (%s) cmd for WN script is: %s" %
(outputFile.namePattern, outputFile.getUploadCmd()))
import inspect
script_location = os.path.join(os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))),
'scripts/LCGSEFileWNScript.py')
from Ganga.GPIDev.Lib.File import FileUtils
script = FileUtils.loadScript(script_location, '###INDENT###')
script = script.replace('###LCGCOMMANDS###', str(lcgCommands))
script = script.replace('###PATTERNSTOZIP###', str(patternsToZip))
script = script.replace('###INDENT###', indent)
script = script.replace('###POSTPROCESSLOCATIONSFP###', postProcessLocationsFP)
return script
def getXMLSummaryScript(indent=''):
'''Returns the necessary script to parse and make sense of the XMLSummary data'''
import inspect
from GangaLHCb.Lib.Applications.AppsBaseUtils import activeSummaryItems
script = "###INDENT#### Parsed XMLSummary data extraction methods\n"
for summaryItem in activeSummaryItems().values():
script += ''.join([
'###INDENT###' + line
for line in inspect.getsourcelines(summaryItem)[0]
])
script += ''.join([
'###INDENT###' + line
for line in inspect.getsourcelines(activeSummaryItems)[0]
])
import inspect
script_location = os.path.join(
os.path.dirname(
os.path.abspath(inspect.getfile(inspect.currentframe()))),
'XMLWorkerScript.py')
from Ganga.GPIDev.Lib.File import FileUtils
xml_script = FileUtils.loadScript(script_location, '###INDENT###')
script += xml_script
return script.replace('###INDENT###', indent)
def getWNScriptDownloadCommand(self, indent):
import inspect
script_location = os.path.join(
os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))), "downloadScript.py"
)
from Ganga.GPIDev.Lib.File import FileUtils
download_script = FileUtils.loadScript(script_location, "")
script = """\n
download_script='''\n###DOWNLOAD_SCRIPT###'''
import subprocess
dirac_env=###DIRAC_ENV###
subprocess.Popen('''python -c "import sys\nexec(sys.stdin.read())"''', shell=True, env=dirac_env, stdin=subprocess.PIPE).communicate(download_script)
"""
script = "\n".join([str(indent + str(line)) for line in script.split("\n")])
replace_dict = {
"###DOWNLOAD_SCRIPT###": download_script,
"###DIRAC_ENV###": self._getDiracEnvStr(),
"###LFN###": self.lfn,
}
for k, v in replace_dict.iteritems():
script = script.replace(str(k), str(v))
return script
def preparejob(self, jobconfig, master_input_sandbox):
job = self.getJobObject()
# print str(job.backend_output_postprocess)
mon = job.getMonitoringService()
import Ganga.Core.Sandbox as Sandbox
subjob_input_sandbox = job.createPackedInputSandbox(jobconfig.getSandboxFiles() + Sandbox.getGangaModulesAsSandboxFiles(Sandbox.getDefaultModules()))
appscriptpath = [jobconfig.getExeString()] + jobconfig.getArgStrings()
if self.nice:
appscriptpath = ['nice', '-n %d' % self.nice] + appscriptpath
if self.nice < 0:
logger.warning('increasing process priority is often not allowed, your job may fail due to this')
sharedoutputpath = job.getOutputWorkspace().getPath()
## FIXME DON'T just use the blind list here, request the list of files to be in the output from a method.
outputpatterns = jobconfig.outputbox
environment = dict() if jobconfig.env is None else jobconfig.env
import tempfile
workdir = tempfile.mkdtemp(dir=config['location'])
import inspect
script_location = os.path.join(os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))),
'LocalHostExec.py')
from Ganga.GPIDev.Lib.File import FileUtils
script = FileUtils.loadScript(script_location, '')
script = script.replace('###INLINEMODULES###', inspect.getsource(Sandbox.WNSandbox))
from Ganga.GPIDev.Lib.File.OutputFileManager import getWNCodeForOutputSandbox, getWNCodeForOutputPostprocessing, getWNCodeForDownloadingInputFiles, getWNCodeForInputdataListCreation
from Ganga.Utility.Config import getConfig
jobidRepr = repr(job.getFQID('.'))
script = script.replace('###OUTPUTSANDBOXPOSTPROCESSING###', getWNCodeForOutputSandbox(job, ['stdout', 'stderr', '__syslog__'], jobidRepr))
script = script.replace('###OUTPUTUPLOADSPOSTPROCESSING###', getWNCodeForOutputPostprocessing(job, ''))
script = script.replace('###DOWNLOADINPUTFILES###', getWNCodeForDownloadingInputFiles(job, ''))
script = script.replace('###CREATEINPUTDATALIST###', getWNCodeForInputdataListCreation(job, ''))
script = script.replace('###APPLICATION_NAME###', repr(getName(job.application)))
script = script.replace('###INPUT_SANDBOX###', repr(subjob_input_sandbox + master_input_sandbox))
script = script.replace('###SHAREDOUTPUTPATH###', repr(sharedoutputpath))
script = script.replace('###APPSCRIPTPATH###', repr(appscriptpath))
script = script.replace('###OUTPUTPATTERNS###', str(outputpatterns))
script = script.replace('###JOBID###', jobidRepr)
script = script.replace('###ENVIRONMENT###', repr(environment))
script = script.replace('###WORKDIR###', repr(workdir))
script = script.replace('###INPUT_DIR###', repr(job.getStringInputDir()))
self.workdir = workdir
script = script.replace('###GANGADIR###', repr(getConfig('System')['GANGA_PYTHONPATH']))
wrkspace = job.getInputWorkspace()
scriptPath = wrkspace.writefile(FileBuffer('__jobscript__', script), executable=1)
return scriptPath
def preparejob(self, jobconfig, master_input_sandbox):
job = self.getJobObject()
# print str(job.backend_output_postprocess)
mon = job.getMonitoringService()
import Ganga.Core.Sandbox as Sandbox
subjob_input_sandbox = job.createPackedInputSandbox(jobconfig.getSandboxFiles() + Sandbox.getGangaModulesAsSandboxFiles(Sandbox.getDefaultModules()))
appscriptpath = [jobconfig.getExeString()] + jobconfig.getArgStrings()
if self.nice:
appscriptpath = ['nice', '-n %d' % self.nice] + appscriptpath
if self.nice < 0:
logger.warning('increasing process priority is often not allowed, your job may fail due to this')
sharedoutputpath = job.getOutputWorkspace().getPath()
## FIXME DON'T just use the blind list here, request the list of files to be in the output from a method.
outputpatterns = jobconfig.outputbox
environment = dict() if jobconfig.env is None else jobconfig.env
import tempfile
workdir = tempfile.mkdtemp(dir=config['location'])
import inspect
script_location = os.path.join(os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))),
'LocalHostExec.py')
from Ganga.GPIDev.Lib.File import FileUtils
script = FileUtils.loadScript(script_location, '')
script = script.replace('###INLINEMODULES###', inspect.getsource(Sandbox.WNSandbox))
from Ganga.GPIDev.Lib.File.OutputFileManager import getWNCodeForOutputSandbox, getWNCodeForOutputPostprocessing, getWNCodeForDownloadingInputFiles, getWNCodeForInputdataListCreation
from Ganga.Utility.Config import getConfig
jobidRepr = repr(job.getFQID('.'))
script = script.replace('###OUTPUTSANDBOXPOSTPROCESSING###', getWNCodeForOutputSandbox(job, ['stdout', 'stderr', '__syslog__'], jobidRepr))
script = script.replace('###OUTPUTUPLOADSPOSTPROCESSING###', getWNCodeForOutputPostprocessing(job, ''))
script = script.replace('###DOWNLOADINPUTFILES###', getWNCodeForDownloadingInputFiles(job, ''))
script = script.replace('###CREATEINPUTDATALIST###', getWNCodeForInputdataListCreation(job, ''))
script = script.replace('###APPLICATION_NAME###', repr(job.application._name))
script = script.replace('###INPUT_SANDBOX###', repr(subjob_input_sandbox + master_input_sandbox))
script = script.replace('###SHAREDOUTPUTPATH###', repr(sharedoutputpath))
script = script.replace('###APPSCRIPTPATH###', repr(appscriptpath))
script = script.replace('###OUTPUTPATTERNS###', str(outputpatterns))
script = script.replace('###JOBID###', jobidRepr)
script = script.replace('###ENVIRONMENT###', repr(environment))
script = script.replace('###WORKDIR###', repr(workdir))
script = script.replace('###INPUT_DIR###', repr(job.getStringInputDir()))
self.workdir = workdir
script = script.replace('###GANGADIR###', repr(getConfig('System')['GANGA_PYTHONPATH']))
wrkspace = job.getInputWorkspace()
scriptPath = wrkspace.writefile(FileBuffer('__jobscript__', script), executable=1)
return scriptPath
def downloadWrapper(app):
from os.path import join, split
from Ganga.GPIDev.Lib.File import FileBuffer
import string
from Ganga.GPIDev.Lib.File import getSharedPath
rootsys = join('.', 'root')
rootenv = {'ROOTSYS': rootsys}
script = app.script
if script == File():
if not app.usepython:
script = File(defaultScript())
else:
script = File(defaultPyRootScript())
else:
script = File(os.path.join(os.path.join(Ganga.GPIDev.Lib.File.getSharedPath(), app.is_prepared.name), os.path.basename(app.script.name)))
commandline = ''
scriptPath = join('.', script.subdir, split(script.name)[1])
if not app.usepython:
# Arguments to the ROOT script needs to be a comma separated list
# enclosed in (). Strings should be enclosed in escaped double quotes.
arglist = []
for arg in app.args:
if isinstance(arg, str):
arglist.append('\\\'' + arg + '\\\'')
else:
arglist.append(arg)
rootarg = '\(\"' + string.join([str(s) for s in arglist], ',') + '\"\)'
# use root
commandline = 'root.exe -b -q ' + scriptPath + rootarg + ''
else:
# use python
pyarg = string.join([str(s) for s in app.args], ' ')
commandline = '\'%(PYTHONCMD)s ' + scriptPath + ' ' + pyarg + ' -b \''
logger.debug("Command line: %s: ", commandline)
# Write a wrapper script that installs ROOT and runs script
script_location = os.path.join(os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))),
'wrapperScriptTemplate.py')
from Ganga.GPIDev.Lib.File import FileUtils
wrapperscript = FileUtils.loadScript(script_location, '')
wrapperscript = wrapperscript.replace('###COMMANDLINE###', commandline)
wrapperscript = wrapperscript.replace('###ROOTVERSION###', app.version)
wrapperscript = wrapperscript.replace('###SCRIPTPATH###', scriptPath)
wrapperscript = wrapperscript.replace('###USEPYTHON###', str(app.usepython))
logger.debug('Script to run on worker node\n' + wrapperscript)
scriptName = "rootwrapper_generated_%s.py" % randomString()
runScript = FileBuffer(scriptName, wrapperscript, executable=1)
inputsandbox = app._getParent().inputsandbox + [script]
return runScript, inputsandbox, rootenv
def defaultScript():
tmpdir = tempfile.mktemp()
os.mkdir(tmpdir)
fname = os.path.join(tmpdir, 'test.C')
script_location = os.path.join(os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))),
'defaultRootScript.C')
from Ganga.GPIDev.Lib.File import FileUtils
script = FileUtils.loadScript(script_location, '')
with open(fname, 'w') as f:
f.write(script)
return fname
def getWNInjectedScript(self, outputFiles, indent, patternsToZip,
postProcessLocationsFP):
"""
Returns script that have to be injected in the jobscript for postprocessing on the WN
"""
massStorageCommands = []
massStorageConfig = getConfig(
'Output')['MassStorageFile']['uploadOptions']
for outputFile in outputFiles:
outputfilenameformat = 'None'
if outputFile.outputfilenameformat != None and outputFile.outputfilenameformat != '':
outputfilenameformat = outputFile.outputfilenameformat
massStorageCommands.append([
'massstorage', outputFile.namePattern, outputfilenameformat,
massStorageConfig['mkdir_cmd'], massStorageConfig['cp_cmd'],
massStorageConfig['ls_cmd'], massStorageConfig['path']
])
import inspect
script_location = os.path.join(
os.path.dirname(
os.path.abspath(inspect.getfile(inspect.currentframe()))),
'scripts/MassStorageFileWNScript.py')
from Ganga.GPIDev.Lib.File import FileUtils
script = FileUtils.loadScript(script_location, '###INDENT###')
jobfqid = self.getJobObject().fqid
jobid = jobfqid
subjobid = ''
if (jobfqid.find('.') > -1):
jobid = jobfqid.split('.')[0]
subjobid = jobfqid.split('.')[1]
replace_dict = {
'###MASSSTORAGECOMMANDS###': repr(massStorageCommands),
'###PATTERNSTOZIP###': str(patternsToZip),
'###INDENT###': indent,
'###POSTPROCESSLOCATIONSFP###': postProcessLocationsFP,
'###FULLJOBDIR###': str(jobfqid.replace('.', os.path.sep)),
'###JOBDIR###': str(jobid),
'###SUBJOBDIR###': str(subjobid)
}
for k, v in replace_dict.iteritems():
script = script.replace(str(k), str(v))
return script
def gaudi_script_template():
'''Creates the script that will be executed by DIRAC job. '''
import inspect
script_location = os.path.join(os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))),
'GaudiTemplate.py')
from Ganga.GPIDev.Lib.File import FileUtils
script_template = FileUtils.loadScript(script_location, '')
return script_template
def diracAPI_script_template():
""" Generate and return the DiracAPI job submission template """
import inspect
import os.path
script_location = os.path.join(os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))),
'DiracRTHScript.py')
from Ganga.GPIDev.Lib.File import FileUtils
script_template = FileUtils.loadScript(script_location, '')
return script_template
def diracAPI_script_template():
# NOTE setOutputData(replicate) replicate keyword only for LHCbDirac. must
# move there when get a chance.
import inspect
import os.path
script_location = os.path.join(os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))),
'DiracRTHScript.py')
from Ganga.GPIDev.Lib.File import FileUtils
script_template = FileUtils.loadScript(script_location, '')
return script_template
def gaudi_script_template():
'''Creates the script that will be executed by DIRAC job. '''
import inspect
script_location = os.path.join(
os.path.dirname(
os.path.abspath(inspect.getfile(inspect.currentframe()))),
'GaudiTemplate.py')
from Ganga.GPIDev.Lib.File import FileUtils
script_template = FileUtils.loadScript(script_location, '')
return script_template
def diracAPI_script_template():
""" Generate and return the DiracAPI job submission template """
import inspect
import os.path
script_location = os.path.join(
os.path.dirname(
os.path.abspath(inspect.getfile(inspect.currentframe()))),
'DiracRTHScript.py.template')
from Ganga.GPIDev.Lib.File import FileUtils
script_template = FileUtils.loadScript(script_location, '')
return script_template
def create_runscript(useCmake=False):
""" Returna run script for Gaudi Apps """
from GangaLHCb.Lib.Applications.EnvironFunctions import construct_run_environ
environ_script = construct_run_environ(useCmake)
import inspect
script_location = os.path.join(os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))),
'WorkerScript.py.template')
from Ganga.GPIDev.Lib.File import FileUtils
worker_script = FileUtils.loadScript(script_location, '')
worker_script = worker_script.replace('###CONSTRUCT_ENVIRON###', environ_script)
return worker_script
def diracAPI_script_template():
# NOTE setOutputData(replicate) replicate keyword only for LHCbDirac. must
# move there when get a chance.
import inspect
import os.path
script_location = os.path.join(
os.path.dirname(
os.path.abspath(inspect.getfile(inspect.currentframe()))),
'DiracRTHScript.py')
from Ganga.GPIDev.Lib.File import FileUtils
script_template = FileUtils.loadScript(script_location, '')
return script_template
def wrapper(self, regexp, version, timeout, kernel):
"""Write a wrapper Python script that executes the notebooks"""
wrapperscript = FileUtils.loadScript(self.templatelocation(), '')
wrapperscript = wrapperscript.replace('###NBFILES###', str(regexp))
wrapperscript = wrapperscript.replace('###VERSION###', str(version))
wrapperscript = wrapperscript.replace('###TIMEOUT###', str(timeout))
wrapperscript = wrapperscript.replace('###KERNEL###', str(kernel))
wrapperscript = wrapperscript.replace('###UUID###', str(uuid.uuid4()))
logger.debug('Script to run on worker node\n' + wrapperscript)
scriptName = "notebook_wrapper_generated.py"
runScript = FileBuffer(scriptName, wrapperscript, executable=1)
return runScript
def defaultPyRootScript():
tmpdir = tempfile.mktemp()
os.mkdir(tmpdir)
fname = os.path.join(tmpdir, "test.py")
script_location = os.path.join(
os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))), "defaultPyRootScript.py"
)
from Ganga.GPIDev.Lib.File import FileUtils
default_script = FileUtils.loadScript(script_location, "")
with open(fname, "w") as f:
f.write(default_script)
return fname
def getWNInjectedScript(self, outputFiles, indent, patternsToZip, postProcessLocationsFP):
"""
Returns script that have to be injected in the jobscript for postprocessing on the WN
"""
massStorageCommands = []
massStorageConfig = getConfig('Output')['MassStorageFile']['uploadOptions']
for outputFile in outputFiles:
outputfilenameformat = 'None'
if outputFile.outputfilenameformat != None and outputFile.outputfilenameformat != '':
outputfilenameformat = outputFile.outputfilenameformat
massStorageCommands.append(['massstorage', outputFile.namePattern, outputfilenameformat,
massStorageConfig['mkdir_cmd'], massStorageConfig['cp_cmd'],
massStorageConfig['ls_cmd'], massStorageConfig['path']])
import inspect
script_location = os.path.join(os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))),
'scripts/MassStorageFileWNScript.py')
from Ganga.GPIDev.Lib.File import FileUtils
script = FileUtils.loadScript(script_location, '###INDENT###')
jobfqid = self.getJobObject().fqid
jobid = jobfqid
subjobid = ''
if (jobfqid.find('.') > -1):
jobid = jobfqid.split('.')[0]
subjobid = jobfqid.split('.')[1]
replace_dict = {'###MASSSTORAGECOMMANDS###' : repr(massStorageCommands),
'###PATTERNSTOZIP###' : str(patternsToZip),
'###INDENT###' : indent,
'###POSTPROCESSLOCATIONSFP###' : postProcessLocationsFP,
'###FULLJOBDIR###' : str(jobfqid.replace('.', os.path.sep)),
'###JOBDIR###' : str(jobid),
'###SUBJOBDIR###' : str(subjobid)}
for k, v in replace_dict.iteritems():
script = script.replace(str(k), str(v))
return script
def create_runscript(useCmake=False):
""" Returna run script for Gaudi Apps """
from GangaLHCb.Lib.Applications.EnvironFunctions import construct_run_environ
environ_script = construct_run_environ(useCmake)
import inspect
script_location = os.path.join(
os.path.dirname(
os.path.abspath(inspect.getfile(inspect.currentframe()))),
'WorkerScript.py')
from Ganga.GPIDev.Lib.File import FileUtils
worker_script = FileUtils.loadScript(script_location, '')
worker_script = worker_script.replace('###CONSTRUCT_ENVIRON###',
environ_script)
return worker_script
def getXMLSummaryScript(indent=''):
'''Returns the necessary script to parse and make sense of the XMLSummary data'''
import inspect
from GangaLHCb.Lib.Applications.AppsBaseUtils import activeSummaryItems
script = "###INDENT#### Parsed XMLSummary data extraction methods\n"
for summaryItem in activeSummaryItems().values():
script += ''.join(['###INDENT###' + line for line in inspect.getsourcelines(summaryItem)[0]])
script += ''.join(['###INDENT###' + line for line in inspect.getsourcelines(activeSummaryItems)[0]])
import inspect
script_location = os.path.join(os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))),
'XMLWorkerScript.py.template')
from Ganga.GPIDev.Lib.File import FileUtils
xml_script = FileUtils.loadScript(script_location, '###INDENT###')
script += xml_script
return script.replace('###INDENT###', indent)
def preparejob(self, jobconfig, master_input_sandbox):
"""Method for preparing job script"""
job = self.getJobObject()
from Ganga.GPIDev.Lib.File import File
from Ganga.Core.Sandbox.WNSandbox import PYTHON_DIR
import Ganga.Utility.files
import inspect
fileutils = File(inspect.getsourcefile(Ganga.Utility.files),
subdir=PYTHON_DIR)
inputfiles = jobconfig.getSandboxFiles() + [fileutils]
inbox = job.createPackedInputSandbox(inputfiles)
inbox.extend(master_input_sandbox)
inpDir = job.getInputWorkspace(create=True).getPath()
outDir = job.getOutputWorkspace(create=True).getPath()
workdir = tempfile.mkdtemp()
self.workdir = workdir
exeString = jobconfig.getExeString()
argList = jobconfig.getArgStrings()
argString = " ".join(map(lambda x: " %s " % x, argList))
outputSandboxPatterns = jobconfig.outputbox
patternsToZip = []
wnCodeForPostprocessing = ''
wnCodeToDownloadInputFiles = ''
if (len(job.outputfiles) > 0):
from Ganga.GPIDev.Lib.File.OutputFileManager import getOutputSandboxPatternsForInteractive, getWNCodeForOutputPostprocessing
(outputSandboxPatterns,
patternsToZip) = getOutputSandboxPatternsForInteractive(job)
wnCodeForPostprocessing = 'def printError(message):pass\ndef printInfo(message):pass' + \
getWNCodeForOutputPostprocessing(job, '')
all_inputfiles = [this_file for this_file in job.inputfiles]
if job.master is not None:
all_inputfiles.extend(
[this_file for this_file in job.master.inputfiles])
wnCodeToDownloadInputFiles = ''
if (len(all_inputfiles) > 0):
from Ganga.GPIDev.Lib.File.OutputFileManager import outputFilePostProcessingOnWN
for inputFile in all_inputfiles:
inputfileClassName = getName(inputFile)
logger.debug("name: %s" % inputfileClassName)
logger.debug(
"result: %s" %
str(outputFilePostProcessingOnWN(job, inputfileClassName)))
if outputFilePostProcessingOnWN(job, inputfileClassName):
inputFile.processWildcardMatches()
if inputFile.subfiles:
getfromFile = False
for subfile in inputFile.subfiles:
wnCodeToDownloadInputFiles += subfile.getWNScriptDownloadCommand(
'')
else:
getfromFile = True
else:
getFromFile = True
if getFromFile:
wnCodeToDownloadInputFiles += inputFile.getWNScriptDownloadCommand(
'')
wnCodeToDownloadInputData = ''
if job.inputdata and (len(job.inputdata) > 0):
from Ganga.GPIDev.Lib.File.OutputFileManager import getWNCodeForDownloadingInputFiles
wnCodeToDownloadInputData = getWNCodeForDownloadingInputFiles(
job, '')
import inspect
replace_dict = {
'###CONSTRUCT_TIME###': (time.strftime("%c")),
'###WNSANDBOX_SOURCE###':
inspect.getsource(Sandbox.WNSandbox),
'###GANGA_PYTHONPATH###':
getConfig("System")["GANGA_PYTHONPATH"],
'###OUTPUTDIR###':
outDir,
'###WORKDIR###':
workdir,
'###IN_BOX###':
inbox,
'###WN_INPUTFILES###':
wnCodeToDownloadInputFiles,
'###WN_INPUTDATA###':
wnCodeToDownloadInputData,
'###JOBCONFIG_ENV###':
jobconfig.env if jobconfig.env is not None else dict(),
'###EXE_STRING###':
exeString,
'###ARG_STRING###':
argString,
'###WN_POSTPROCESSING###':
wnCodeForPostprocessing,
'###PATTERNS_TO_ZIP###':
patternsToZip,
'###OUTPUT_SANDBOX_PATTERNS###':
outputSandboxPatterns
}
script_location = os.path.join(
os.path.dirname(
os.path.abspath(inspect.getfile(inspect.currentframe()))),
'InteractiveScriptTemplate.py.template')
from Ganga.GPIDev.Lib.File import FileUtils
commandString = FileUtils.loadScript(script_location, '')
for k, v in replace_dict.iteritems():
commandString = commandString.replace(str(k), str(v))
return job.getInputWorkspace().writefile(FileBuffer(
"__jobscript__", commandString),
executable=1)
def preparejob(self, jobconfig, master_input_sandbox):
job = self.getJobObject()
mon = job.getMonitoringService()
import Ganga.Core.Sandbox as Sandbox
from Ganga.GPIDev.Lib.File import File
from Ganga.Core.Sandbox.WNSandbox import PYTHON_DIR
import inspect
fileutils = File( inspect.getsourcefile(Ganga.Utility.files), subdir=PYTHON_DIR )
subjob_input_sandbox = job.createPackedInputSandbox(jobconfig.getSandboxFiles() + [ fileutils ] )
appscriptpath = [jobconfig.getExeString()] + jobconfig.getArgStrings()
sharedoutputpath = job.getOutputWorkspace().getPath()
## FIXME Check this isn't a GangaList
outputpatterns = jobconfig.outputbox
environment = jobconfig.env if not jobconfig.env is None else {}
import inspect
script_location = os.path.join(os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))),
'BatchScriptTemplate.py')
from Ganga.GPIDev.Lib.File import FileUtils
text = FileUtils.loadScript(script_location, '')
import Ganga.Core.Sandbox as Sandbox
import Ganga.Utility as Utility
from Ganga.Utility.Config import getConfig
from Ganga.GPIDev.Lib.File.OutputFileManager import getWNCodeForOutputSandbox, getWNCodeForOutputPostprocessing, getWNCodeForDownloadingInputFiles
jobidRepr = repr(self.getJobObject().getFQID('.'))
replace_dict = {
'###OUTPUTSANDBOXPOSTPROCESSING###' : getWNCodeForOutputSandbox(job, ['__syslog__'], jobidRepr),
'###OUTPUTUPLOADSPOSTPROCESSING###' : getWNCodeForOutputPostprocessing(job, ''),
'###DOWNLOADINPUTFILES###' : getWNCodeForDownloadingInputFiles(job, ''),
'###INLINEMODULES###' : inspect.getsource(Sandbox.WNSandbox),
'###INLINEHOSTNAMEFUNCTION###' : inspect.getsource(Utility.util.hostname),
'###APPSCRIPTPATH###' : repr(appscriptpath),
#'###SHAREDINPUTPATH###' : repr(sharedinputpath)),
'###INPUT_SANDBOX###' : repr(subjob_input_sandbox + master_input_sandbox),
'###SHAREDOUTPUTPATH###' : repr(sharedoutputpath),
'###OUTPUTPATTERNS###' : repr(outputpatterns),
'###JOBID###' : jobidRepr,
'###ENVIRONMENT###' : repr(environment),
'###PREEXECUTE###' : self.config['preexecute'],
'###POSTEXECUTE###' : self.config['postexecute'],
'###JOBIDNAME###' : self.config['jobid_name'],
'###QUEUENAME###' : self.config['queue_name'],
'###HEARTBEATFREQUENCE###' : self.config['heartbeat_frequency'],
'###INPUT_DIR###' : repr(job.getStringInputDir()),
'###GANGADIR###' : repr(getConfig('System')['GANGA_PYTHONPATH'])
}
for k, v in replace_dict.iteritems():
text = text.replace(str(k), str(v))
logger.debug('subjob input sandbox %s ', subjob_input_sandbox)
logger.debug('master input sandbox %s ', master_input_sandbox)
from Ganga.GPIDev.Lib.File import FileBuffer
return job.getInputWorkspace().writefile(FileBuffer('__jobscript__', text), executable=1)
def preparejob(self, jobconfig, master_input_sandbox):
job = self.getJobObject()
# print str(job.backend_output_postprocess)
mon = job.getMonitoringService()
import Ganga.Core.Sandbox as Sandbox
subjob_input_sandbox = job.createPackedInputSandbox(
jobconfig.getSandboxFiles()
+ Sandbox.getGangaModulesAsSandboxFiles(Sandbox.getDefaultModules())
+ Sandbox.getGangaModulesAsSandboxFiles(mon.getSandboxModules())
)
appscriptpath = [jobconfig.getExeString()] + jobconfig.getArgStrings()
if self.nice:
appscriptpath = ["nice", "-n %d" % self.nice] + appscriptpath
if self.nice < 0:
logger.warning("increasing process priority is often not allowed, your job may fail due to this")
sharedoutputpath = job.getOutputWorkspace().getPath()
outputpatterns = jobconfig.outputbox
environment = dict() if jobconfig.env is None else jobconfig.env
import tempfile
workdir = tempfile.mkdtemp(dir=config["location"])
import inspect
script_location = os.path.join(
os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))), "LocalHostExec.py"
)
from Ganga.GPIDev.Lib.File import FileUtils
script = FileUtils.loadScript(script_location, "")
script = script.replace("###INLINEMODULES###", inspect.getsource(Sandbox.WNSandbox))
from Ganga.GPIDev.Lib.File.OutputFileManager import (
getWNCodeForOutputSandbox,
getWNCodeForOutputPostprocessing,
getWNCodeForDownloadingInputFiles,
getWNCodeForInputdataListCreation,
)
from Ganga.Utility.Config import getConfig
jobidRepr = repr(job.getFQID("."))
script = script.replace(
"###OUTPUTSANDBOXPOSTPROCESSING###",
getWNCodeForOutputSandbox(job, ["stdout", "stderr", "__syslog__"], jobidRepr),
)
script = script.replace("###OUTPUTUPLOADSPOSTPROCESSING###", getWNCodeForOutputPostprocessing(job, ""))
script = script.replace("###DOWNLOADINPUTFILES###", getWNCodeForDownloadingInputFiles(job, ""))
script = script.replace("###CREATEINPUTDATALIST###", getWNCodeForInputdataListCreation(job, ""))
script = script.replace("###APPLICATION_NAME###", repr(job.application._name))
script = script.replace("###INPUT_SANDBOX###", repr(subjob_input_sandbox + master_input_sandbox))
script = script.replace("###SHAREDOUTPUTPATH###", repr(sharedoutputpath))
script = script.replace("###APPSCRIPTPATH###", repr(appscriptpath))
script = script.replace("###OUTPUTPATTERNS###", repr(outputpatterns))
script = script.replace("###JOBID###", jobidRepr)
script = script.replace("###ENVIRONMENT###", repr(environment))
script = script.replace("###WORKDIR###", repr(workdir))
script = script.replace("###INPUT_DIR###", repr(job.getStringInputDir()))
script = script.replace(
"###MONITORING_SERVICE###", job.getMonitoringService().getWrapperScriptConstructorText()
)
self.workdir = workdir
script = script.replace("###GANGADIR###", repr(getConfig("System")["GANGA_PYTHONPATH"]))
wrkspace = job.getInputWorkspace()
scriptPath = wrkspace.writefile(FileBuffer("__jobscript__", script), executable=1)
return scriptPath
def preparejob(self, jobconfig, master_input_sandbox):
"""Method for preparing job script"""
job = self.getJobObject()
inputfiles = jobconfig.getSandboxFiles()
inbox = job.createPackedInputSandbox(inputfiles)
inbox.extend(master_input_sandbox)
inpDir = job.getInputWorkspace(create=True).getPath()
outDir = job.getOutputWorkspace(create=True).getPath()
workdir = tempfile.mkdtemp()
self.workdir = workdir
exeString = jobconfig.getExeString()
argList = jobconfig.getArgStrings()
argString = " ".join(map(lambda x: " %s " % x, argList))
outputSandboxPatterns = jobconfig.outputbox
patternsToZip = []
wnCodeForPostprocessing = ''
wnCodeToDownloadInputFiles = ''
if (len(job.outputfiles) > 0):
from Ganga.GPIDev.Lib.File.OutputFileManager import getOutputSandboxPatternsForInteractive, getWNCodeForOutputPostprocessing
(outputSandboxPatterns,
patternsToZip) = getOutputSandboxPatternsForInteractive(job)
wnCodeForPostprocessing = 'def printError(message):pass\ndef printInfo(message):pass' + \
getWNCodeForOutputPostprocessing(job, '')
all_inputfiles = [this_file for this_file in job.inputfiles]
if job.master: all_inputfiles.extend([this_file for this_file in job.master.inputfiles])
wnCodeToDownloadInputFiles = ''
if(len(all_inputfiles) > 0):
from Ganga.GPIDev.Lib.File.OutputFileManager import outputFilePostProcessingOnWN
for inputFile in all_inputfiles:
inputfileClassName = getName(inputFile)
logger.debug("name: %s" % inputfileClassName)
logger.debug("result: %s" % str(outputFilePostProcessingOnWN(job, inputfileClassName)))
if outputFilePostProcessingOnWN(job, inputfileClassName):
inputFile.processWildcardMatches()
if inputFile.subfiles:
getfromFile = False
for subfile in inputFile.subfiles:
wnCodeToDownloadInputFiles += subfile.getWNScriptDownloadCommand('')
else:
getfromFile = True
else:
getFromFile = True
if getFromFile:
wnCodeToDownloadInputFiles += inputFile.getWNScriptDownloadCommand('')
wnCodeToDownloadInputData = ''
if job.inputdata and (len(job.inputdata) > 0):
from Ganga.GPIDev.Lib.File.OutputFileManager import getWNCodeForDownloadingInputFiles
wnCodeToDownloadInputData = getWNCodeForDownloadingInputFiles(job, '')
import inspect
replace_dict = {
'###CONSTRUCT_TIME###' : (time.strftime("%c")),
'###WNSANDBOX_SOURCE###' : inspect.getsource(Sandbox.WNSandbox),
'###GANGA_PYTHONPATH###' : getConfig("System")["GANGA_PYTHONPATH"],
'###OUTPUTDIR###' : outDir,
'###WORKDIR###' : workdir,
'###IN_BOX###' : inbox,
'###WN_INPUTFILES###' : wnCodeToDownloadInputFiles,
'###WN_INPUTDATA###' : wnCodeToDownloadInputData,
'###JOBCONFIG_ENV###' : jobconfig.env if jobconfig.env is not None else dict(),
'###EXE_STRING###' : exeString,
'###ARG_STRING###' : argString,
'###WN_POSTPROCESSING###' : wnCodeForPostprocessing,
'###PATTERNS_TO_ZIP###' : patternsToZip,
'###OUTPUT_SANDBOX_PATTERNS###' : outputSandboxPatterns
}
script_location = os.path.join(os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))),
'InteractiveScriptTemplate.py')
from Ganga.GPIDev.Lib.File import FileUtils
commandString = FileUtils.loadScript(script_location, '')
for k, v in replace_dict.iteritems():
commandString = commandString.replace(str(k), str(v))
return job.getInputWorkspace().writefile(FileBuffer("__jobscript__", commandString), executable=1)
