def preparejob(self, jobconfig, master_input_sandbox):
job = self.getJobObject()
# print str(job.backend_output_postprocess)
mon = job.getMonitoringService()
import Ganga.Core.Sandbox as Sandbox
subjob_input_sandbox = job.createPackedInputSandbox(jobconfig.getSandboxFiles() + Sandbox.getGangaModulesAsSandboxFiles(Sandbox.getDefaultModules()))
appscriptpath = [jobconfig.getExeString()] + jobconfig.getArgStrings()
if self.nice:
appscriptpath = ['nice', '-n %d' % self.nice] + appscriptpath
if self.nice < 0:
logger.warning('increasing process priority is often not allowed, your job may fail due to this')
sharedoutputpath = job.getOutputWorkspace().getPath()
## FIXME DON'T just use the blind list here, request the list of files to be in the output from a method.
outputpatterns = jobconfig.outputbox
environment = dict() if jobconfig.env is None else jobconfig.env
import tempfile
workdir = tempfile.mkdtemp(dir=config['location'])
import inspect
script_location = os.path.join(os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))),
'LocalHostExec.py')
from Ganga.GPIDev.Lib.File import FileUtils
script = FileUtils.loadScript(script_location, '')
script = script.replace('###INLINEMODULES###', inspect.getsource(Sandbox.WNSandbox))
from Ganga.GPIDev.Lib.File.OutputFileManager import getWNCodeForOutputSandbox, getWNCodeForOutputPostprocessing, getWNCodeForDownloadingInputFiles, getWNCodeForInputdataListCreation
from Ganga.Utility.Config import getConfig
jobidRepr = repr(job.getFQID('.'))
script = script.replace('###OUTPUTSANDBOXPOSTPROCESSING###', getWNCodeForOutputSandbox(job, ['stdout', 'stderr', '__syslog__'], jobidRepr))
script = script.replace('###OUTPUTUPLOADSPOSTPROCESSING###', getWNCodeForOutputPostprocessing(job, ''))
script = script.replace('###DOWNLOADINPUTFILES###', getWNCodeForDownloadingInputFiles(job, ''))
script = script.replace('###CREATEINPUTDATALIST###', getWNCodeForInputdataListCreation(job, ''))
script = script.replace('###APPLICATION_NAME###', repr(getName(job.application)))
script = script.replace('###INPUT_SANDBOX###', repr(subjob_input_sandbox + master_input_sandbox))
script = script.replace('###SHAREDOUTPUTPATH###', repr(sharedoutputpath))
script = script.replace('###APPSCRIPTPATH###', repr(appscriptpath))
script = script.replace('###OUTPUTPATTERNS###', str(outputpatterns))
script = script.replace('###JOBID###', jobidRepr)
script = script.replace('###ENVIRONMENT###', repr(environment))
script = script.replace('###WORKDIR###', repr(workdir))
script = script.replace('###INPUT_DIR###', repr(job.getStringInputDir()))
self.workdir = workdir
script = script.replace('###GANGADIR###', repr(getConfig('System')['GANGA_PYTHONPATH']))
wrkspace = job.getInputWorkspace()
scriptPath = wrkspace.writefile(FileBuffer('__jobscript__', script), executable=1)
return scriptPath
def preparejob(self, jobconfig, master_input_sandbox):
job = self.getJobObject()
# print str(job.backend_output_postprocess)
mon = job.getMonitoringService()
import Ganga.Core.Sandbox as Sandbox
subjob_input_sandbox = job.createPackedInputSandbox(jobconfig.getSandboxFiles() + Sandbox.getGangaModulesAsSandboxFiles(Sandbox.getDefaultModules()))
appscriptpath = [jobconfig.getExeString()] + jobconfig.getArgStrings()
if self.nice:
appscriptpath = ['nice', '-n %d' % self.nice] + appscriptpath
if self.nice < 0:
logger.warning('increasing process priority is often not allowed, your job may fail due to this')
sharedoutputpath = job.getOutputWorkspace().getPath()
## FIXME DON'T just use the blind list here, request the list of files to be in the output from a method.
outputpatterns = jobconfig.outputbox
environment = dict() if jobconfig.env is None else jobconfig.env
import tempfile
workdir = tempfile.mkdtemp(dir=config['location'])
import inspect
script_location = os.path.join(os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))),
'LocalHostExec.py')
from Ganga.GPIDev.Lib.File import FileUtils
script = FileUtils.loadScript(script_location, '')
script = script.replace('###INLINEMODULES###', inspect.getsource(Sandbox.WNSandbox))
from Ganga.GPIDev.Lib.File.OutputFileManager import getWNCodeForOutputSandbox, getWNCodeForOutputPostprocessing, getWNCodeForDownloadingInputFiles, getWNCodeForInputdataListCreation
from Ganga.Utility.Config import getConfig
jobidRepr = repr(job.getFQID('.'))
script = script.replace('###OUTPUTSANDBOXPOSTPROCESSING###', getWNCodeForOutputSandbox(job, ['stdout', 'stderr', '__syslog__'], jobidRepr))
script = script.replace('###OUTPUTUPLOADSPOSTPROCESSING###', getWNCodeForOutputPostprocessing(job, ''))
script = script.replace('###DOWNLOADINPUTFILES###', getWNCodeForDownloadingInputFiles(job, ''))
script = script.replace('###CREATEINPUTDATALIST###', getWNCodeForInputdataListCreation(job, ''))
script = script.replace('###APPLICATION_NAME###', repr(job.application._name))
script = script.replace('###INPUT_SANDBOX###', repr(subjob_input_sandbox + master_input_sandbox))
script = script.replace('###SHAREDOUTPUTPATH###', repr(sharedoutputpath))
script = script.replace('###APPSCRIPTPATH###', repr(appscriptpath))
script = script.replace('###OUTPUTPATTERNS###', str(outputpatterns))
script = script.replace('###JOBID###', jobidRepr)
script = script.replace('###ENVIRONMENT###', repr(environment))
script = script.replace('###WORKDIR###', repr(workdir))
script = script.replace('###INPUT_DIR###', repr(job.getStringInputDir()))
self.workdir = workdir
script = script.replace('###GANGADIR###', repr(getConfig('System')['GANGA_PYTHONPATH']))
wrkspace = job.getInputWorkspace()
scriptPath = wrkspace.writefile(FileBuffer('__jobscript__', script), executable=1)
return scriptPath
def preparejob(self, jobconfig, master_input_sandbox):
job = self.getJobObject()
mon = job.getMonitoringService()
import Ganga.Core.Sandbox as Sandbox
from Ganga.GPIDev.Lib.File import File
from Ganga.Core.Sandbox.WNSandbox import PYTHON_DIR
import inspect
fileutils = File( inspect.getsourcefile(Ganga.Utility.files), subdir=PYTHON_DIR )
subjob_input_sandbox = job.createPackedInputSandbox(jobconfig.getSandboxFiles() + [ fileutils ] )
appscriptpath = [jobconfig.getExeString()] + jobconfig.getArgStrings()
sharedoutputpath = job.getOutputWorkspace().getPath()
## FIXME Check this isn't a GangaList
outputpatterns = jobconfig.outputbox
environment = jobconfig.env if not jobconfig.env is None else {}
import inspect
script_location = os.path.join(os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))),
'BatchScriptTemplate.py.template')
from Ganga.GPIDev.Lib.File import FileUtils
text = FileUtils.loadScript(script_location, '')
import Ganga.Core.Sandbox as Sandbox
import Ganga.Utility as Utility
from Ganga.Utility.Config import getConfig
from Ganga.GPIDev.Lib.File.OutputFileManager import getWNCodeForOutputSandbox, getWNCodeForOutputPostprocessing, getWNCodeForDownloadingInputFiles, getWNCodeForInputdataListCreation
jobidRepr = repr(self.getJobObject().getFQID('.'))
replace_dict = {
'###OUTPUTSANDBOXPOSTPROCESSING###' : getWNCodeForOutputSandbox(job, ['__syslog__'], jobidRepr),
'###OUTPUTUPLOADSPOSTPROCESSING###' : getWNCodeForOutputPostprocessing(job, ''),
'###DOWNLOADINPUTFILES###' : getWNCodeForDownloadingInputFiles(job, ''),
'###INLINEMODULES###' : inspect.getsource(Sandbox.WNSandbox),
'###INLINEHOSTNAMEFUNCTION###' : inspect.getsource(Utility.util.hostname),
'###APPSCRIPTPATH###' : repr(appscriptpath),
#'###SHAREDINPUTPATH###' : repr(sharedinputpath)),
'###INPUT_SANDBOX###' : repr(subjob_input_sandbox + master_input_sandbox),
'###CREATEINPUTDATALIST###' : getWNCodeForInputdataListCreation(job, ''),
'###SHAREDOUTPUTPATH###' : repr(sharedoutputpath),
'###OUTPUTPATTERNS###' : repr(outputpatterns),
'###JOBID###' : jobidRepr,
'###ENVIRONMENT###' : repr(environment),
'###PREEXECUTE###' : self.config['preexecute'],
'###POSTEXECUTE###' : self.config['postexecute'],
'###JOBIDNAME###' : self.config['jobid_name'],
'###QUEUENAME###' : self.config['queue_name'],
'###HEARTBEATFREQUENCE###' : self.config['heartbeat_frequency'],
'###INPUT_DIR###' : repr(job.getStringInputDir()),
'###GANGADIR###' : repr(getConfig('System')['GANGA_PYTHONPATH'])
}
for k, v in replace_dict.iteritems():
text = text.replace(str(k), str(v))
logger.debug('subjob input sandbox %s ', subjob_input_sandbox)
logger.debug('master input sandbox %s ', master_input_sandbox)
from Ganga.GPIDev.Lib.File import FileBuffer
return job.getInputWorkspace().writefile(FileBuffer('__jobscript__', text), executable=1)
def preparejob(self, jobconfig, master_input_sandbox):
job = self.getJobObject()
# print str(job.backend_output_postprocess)
mon = job.getMonitoringService()
import Ganga.Core.Sandbox as Sandbox
subjob_input_sandbox = job.createPackedInputSandbox(
jobconfig.getSandboxFiles()
+ Sandbox.getGangaModulesAsSandboxFiles(Sandbox.getDefaultModules())
+ Sandbox.getGangaModulesAsSandboxFiles(mon.getSandboxModules())
)
appscriptpath = [jobconfig.getExeString()] + jobconfig.getArgStrings()
if self.nice:
appscriptpath = ["nice", "-n %d" % self.nice] + appscriptpath
if self.nice < 0:
logger.warning("increasing process priority is often not allowed, your job may fail due to this")
sharedoutputpath = job.getOutputWorkspace().getPath()
outputpatterns = jobconfig.outputbox
environment = dict() if jobconfig.env is None else jobconfig.env
import tempfile
workdir = tempfile.mkdtemp(dir=config["location"])
import inspect
script_location = os.path.join(
os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))), "LocalHostExec.py"
)
from Ganga.GPIDev.Lib.File import FileUtils
script = FileUtils.loadScript(script_location, "")
script = script.replace("###INLINEMODULES###", inspect.getsource(Sandbox.WNSandbox))
from Ganga.GPIDev.Lib.File.OutputFileManager import (
getWNCodeForOutputSandbox,
getWNCodeForOutputPostprocessing,
getWNCodeForDownloadingInputFiles,
getWNCodeForInputdataListCreation,
)
from Ganga.Utility.Config import getConfig
jobidRepr = repr(job.getFQID("."))
script = script.replace(
"###OUTPUTSANDBOXPOSTPROCESSING###",
getWNCodeForOutputSandbox(job, ["stdout", "stderr", "__syslog__"], jobidRepr),
)
script = script.replace("###OUTPUTUPLOADSPOSTPROCESSING###", getWNCodeForOutputPostprocessing(job, ""))
script = script.replace("###DOWNLOADINPUTFILES###", getWNCodeForDownloadingInputFiles(job, ""))
script = script.replace("###CREATEINPUTDATALIST###", getWNCodeForInputdataListCreation(job, ""))
script = script.replace("###APPLICATION_NAME###", repr(job.application._name))
script = script.replace("###INPUT_SANDBOX###", repr(subjob_input_sandbox + master_input_sandbox))
script = script.replace("###SHAREDOUTPUTPATH###", repr(sharedoutputpath))
script = script.replace("###APPSCRIPTPATH###", repr(appscriptpath))
script = script.replace("###OUTPUTPATTERNS###", repr(outputpatterns))
script = script.replace("###JOBID###", jobidRepr)
script = script.replace("###ENVIRONMENT###", repr(environment))
script = script.replace("###WORKDIR###", repr(workdir))
script = script.replace("###INPUT_DIR###", repr(job.getStringInputDir()))
script = script.replace(
"###MONITORING_SERVICE###", job.getMonitoringService().getWrapperScriptConstructorText()
)
self.workdir = workdir
script = script.replace("###GANGADIR###", repr(getConfig("System")["GANGA_PYTHONPATH"]))
wrkspace = job.getInputWorkspace()
scriptPath = wrkspace.writefile(FileBuffer("__jobscript__", script), executable=1)
return scriptPath
def preparejob(self, jobconfig, master_input_sandbox):
job = self.getJobObject()
mon = job.getMonitoringService()
import Ganga.Core.Sandbox as Sandbox
from Ganga.GPIDev.Lib.File import File
from Ganga.Core.Sandbox.WNSandbox import PYTHON_DIR
import inspect
fileutils = File(inspect.getsourcefile(Ganga.Utility.files),
subdir=PYTHON_DIR)
sharedfiles = jobconfig.getSharedFiles()
subjob_input_sandbox = job.createPackedInputSandbox(
jobconfig.getSandboxFiles() + [fileutils])
appscriptpath = [jobconfig.getExeString()] + jobconfig.getArgStrings()
sharedoutputpath = job.getOutputWorkspace().getPath()
## FIXME Check this isn't a GangaList
outputpatterns = jobconfig.outputbox
environment = jobconfig.env if not jobconfig.env is None else {}
import inspect
script_location = os.path.join(
os.path.dirname(
os.path.abspath(inspect.getfile(inspect.currentframe()))),
'BatchScriptTemplate.py.template')
from Ganga.GPIDev.Lib.File import FileUtils
text = FileUtils.loadScript(script_location, '')
import Ganga.Core.Sandbox as Sandbox
import Ganga.Utility as Utility
from Ganga.Utility.Config import getConfig
from Ganga.GPIDev.Lib.File.OutputFileManager import getWNCodeForOutputSandbox, getWNCodeForOutputPostprocessing, getWNCodeForDownloadingInputFiles, getWNCodeForInputdataListCreation
jobidRepr = repr(self.getJobObject().getFQID('.'))
replace_dict = {
'###OUTPUTSANDBOXPOSTPROCESSING###':
getWNCodeForOutputSandbox(job, ['__syslog__'], jobidRepr),
'###OUTPUTUPLOADSPOSTPROCESSING###':
getWNCodeForOutputPostprocessing(job, ''),
'###DOWNLOADINPUTFILES###':
getWNCodeForDownloadingInputFiles(job, ''),
'###INLINEMODULES###':
inspect.getsource(Sandbox.WNSandbox),
'###INLINEHOSTNAMEFUNCTION###':
inspect.getsource(Utility.util.hostname),
'###APPSCRIPTPATH###':
repr(appscriptpath),
#'###SHAREDINPUTPATH###' : repr(sharedinputpath)),
'###INPUT_SANDBOX###':
repr(subjob_input_sandbox + master_input_sandbox + sharedfiles),
'###CREATEINPUTDATALIST###':
getWNCodeForInputdataListCreation(job, ''),
'###SHAREDOUTPUTPATH###':
repr(sharedoutputpath),
'###OUTPUTPATTERNS###':
repr(outputpatterns),
'###JOBID###':
jobidRepr,
'###ENVIRONMENT###':
repr(environment),
'###PREEXECUTE###':
self.config['preexecute'],
'###POSTEXECUTE###':
self.config['postexecute'],
'###JOBIDNAME###':
self.config['jobid_name'],
'###QUEUENAME###':
self.config['queue_name'],
'###HEARTBEATFREQUENCE###':
self.config['heartbeat_frequency'],
'###INPUT_DIR###':
repr(job.getStringInputDir()),
'###GANGADIR###':
repr(getConfig('System')['GANGA_PYTHONPATH'])
}
for k, v in replace_dict.iteritems():
text = text.replace(str(k), str(v))
logger.debug('subjob input sandbox %s ', subjob_input_sandbox)
logger.debug('master input sandbox %s ', master_input_sandbox)
from Ganga.GPIDev.Lib.File import FileBuffer
return job.getInputWorkspace().writefile(FileBuffer(
'__jobscript__', text),
executable=1)
