def preparejob(self, jobconfig, master_input_sandbox): job = self.getJobObject() # print str(job.backend_output_postprocess) mon = job.getMonitoringService() import Ganga.Core.Sandbox as Sandbox subjob_input_sandbox = job.createPackedInputSandbox(jobconfig.getSandboxFiles() + Sandbox.getGangaModulesAsSandboxFiles(Sandbox.getDefaultModules())) appscriptpath = [jobconfig.getExeString()] + jobconfig.getArgStrings() if self.nice: appscriptpath = ['nice', '-n %d' % self.nice] + appscriptpath if self.nice < 0: logger.warning('increasing process priority is often not allowed, your job may fail due to this') sharedoutputpath = job.getOutputWorkspace().getPath() ## FIXME DON'T just use the blind list here, request the list of files to be in the output from a method. outputpatterns = jobconfig.outputbox environment = dict() if jobconfig.env is None else jobconfig.env import tempfile workdir = tempfile.mkdtemp(dir=config['location']) import inspect script_location = os.path.join(os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))), 'LocalHostExec.py') from Ganga.GPIDev.Lib.File import FileUtils script = FileUtils.loadScript(script_location, '') script = script.replace('###INLINEMODULES###', inspect.getsource(Sandbox.WNSandbox)) from Ganga.GPIDev.Lib.File.OutputFileManager import getWNCodeForOutputSandbox, getWNCodeForOutputPostprocessing, getWNCodeForDownloadingInputFiles, getWNCodeForInputdataListCreation from Ganga.Utility.Config import getConfig jobidRepr = repr(job.getFQID('.')) script = script.replace('###OUTPUTSANDBOXPOSTPROCESSING###', getWNCodeForOutputSandbox(job, ['stdout', 'stderr', '__syslog__'], jobidRepr)) script = script.replace('###OUTPUTUPLOADSPOSTPROCESSING###', getWNCodeForOutputPostprocessing(job, '')) script = script.replace('###DOWNLOADINPUTFILES###', getWNCodeForDownloadingInputFiles(job, '')) script = script.replace('###CREATEINPUTDATALIST###', getWNCodeForInputdataListCreation(job, '')) script = script.replace('###APPLICATION_NAME###', repr(getName(job.application))) script = script.replace('###INPUT_SANDBOX###', repr(subjob_input_sandbox + master_input_sandbox)) script = script.replace('###SHAREDOUTPUTPATH###', repr(sharedoutputpath)) script = script.replace('###APPSCRIPTPATH###', repr(appscriptpath)) script = script.replace('###OUTPUTPATTERNS###', str(outputpatterns)) script = script.replace('###JOBID###', jobidRepr) script = script.replace('###ENVIRONMENT###', repr(environment)) script = script.replace('###WORKDIR###', repr(workdir)) script = script.replace('###INPUT_DIR###', repr(job.getStringInputDir())) self.workdir = workdir script = script.replace('###GANGADIR###', repr(getConfig('System')['GANGA_PYTHONPATH'])) wrkspace = job.getInputWorkspace() scriptPath = wrkspace.writefile(FileBuffer('__jobscript__', script), executable=1) return scriptPath
def preparejob(self, jobconfig, master_input_sandbox): job = self.getJobObject() # print str(job.backend_output_postprocess) mon = job.getMonitoringService() import Ganga.Core.Sandbox as Sandbox subjob_input_sandbox = job.createPackedInputSandbox(jobconfig.getSandboxFiles() + Sandbox.getGangaModulesAsSandboxFiles(Sandbox.getDefaultModules())) appscriptpath = [jobconfig.getExeString()] + jobconfig.getArgStrings() if self.nice: appscriptpath = ['nice', '-n %d' % self.nice] + appscriptpath if self.nice < 0: logger.warning('increasing process priority is often not allowed, your job may fail due to this') sharedoutputpath = job.getOutputWorkspace().getPath() ## FIXME DON'T just use the blind list here, request the list of files to be in the output from a method. outputpatterns = jobconfig.outputbox environment = dict() if jobconfig.env is None else jobconfig.env import tempfile workdir = tempfile.mkdtemp(dir=config['location']) import inspect script_location = os.path.join(os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))), 'LocalHostExec.py') from Ganga.GPIDev.Lib.File import FileUtils script = FileUtils.loadScript(script_location, '') script = script.replace('###INLINEMODULES###', inspect.getsource(Sandbox.WNSandbox)) from Ganga.GPIDev.Lib.File.OutputFileManager import getWNCodeForOutputSandbox, getWNCodeForOutputPostprocessing, getWNCodeForDownloadingInputFiles, getWNCodeForInputdataListCreation from Ganga.Utility.Config import getConfig jobidRepr = repr(job.getFQID('.')) script = script.replace('###OUTPUTSANDBOXPOSTPROCESSING###', getWNCodeForOutputSandbox(job, ['stdout', 'stderr', '__syslog__'], jobidRepr)) script = script.replace('###OUTPUTUPLOADSPOSTPROCESSING###', getWNCodeForOutputPostprocessing(job, '')) script = script.replace('###DOWNLOADINPUTFILES###', getWNCodeForDownloadingInputFiles(job, '')) script = script.replace('###CREATEINPUTDATALIST###', getWNCodeForInputdataListCreation(job, '')) script = script.replace('###APPLICATION_NAME###', repr(job.application._name)) script = script.replace('###INPUT_SANDBOX###', repr(subjob_input_sandbox + master_input_sandbox)) script = script.replace('###SHAREDOUTPUTPATH###', repr(sharedoutputpath)) script = script.replace('###APPSCRIPTPATH###', repr(appscriptpath)) script = script.replace('###OUTPUTPATTERNS###', str(outputpatterns)) script = script.replace('###JOBID###', jobidRepr) script = script.replace('###ENVIRONMENT###', repr(environment)) script = script.replace('###WORKDIR###', repr(workdir)) script = script.replace('###INPUT_DIR###', repr(job.getStringInputDir())) self.workdir = workdir script = script.replace('###GANGADIR###', repr(getConfig('System')['GANGA_PYTHONPATH'])) wrkspace = job.getInputWorkspace() scriptPath = wrkspace.writefile(FileBuffer('__jobscript__', script), executable=1) return scriptPath
def preparejob(self, jobconfig, master_input_sandbox): job = self.getJobObject() mon = job.getMonitoringService() import Ganga.Core.Sandbox as Sandbox from Ganga.GPIDev.Lib.File import File from Ganga.Core.Sandbox.WNSandbox import PYTHON_DIR import inspect fileutils = File( inspect.getsourcefile(Ganga.Utility.files), subdir=PYTHON_DIR ) subjob_input_sandbox = job.createPackedInputSandbox(jobconfig.getSandboxFiles() + [ fileutils ] ) appscriptpath = [jobconfig.getExeString()] + jobconfig.getArgStrings() sharedoutputpath = job.getOutputWorkspace().getPath() ## FIXME Check this isn't a GangaList outputpatterns = jobconfig.outputbox environment = jobconfig.env if not jobconfig.env is None else {} import inspect script_location = os.path.join(os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))), 'BatchScriptTemplate.py.template') from Ganga.GPIDev.Lib.File import FileUtils text = FileUtils.loadScript(script_location, '') import Ganga.Core.Sandbox as Sandbox import Ganga.Utility as Utility from Ganga.Utility.Config import getConfig from Ganga.GPIDev.Lib.File.OutputFileManager import getWNCodeForOutputSandbox, getWNCodeForOutputPostprocessing, getWNCodeForDownloadingInputFiles, getWNCodeForInputdataListCreation jobidRepr = repr(self.getJobObject().getFQID('.')) replace_dict = { '###OUTPUTSANDBOXPOSTPROCESSING###' : getWNCodeForOutputSandbox(job, ['__syslog__'], jobidRepr), '###OUTPUTUPLOADSPOSTPROCESSING###' : getWNCodeForOutputPostprocessing(job, ''), '###DOWNLOADINPUTFILES###' : getWNCodeForDownloadingInputFiles(job, ''), '###INLINEMODULES###' : inspect.getsource(Sandbox.WNSandbox), '###INLINEHOSTNAMEFUNCTION###' : inspect.getsource(Utility.util.hostname), '###APPSCRIPTPATH###' : repr(appscriptpath), #'###SHAREDINPUTPATH###' : repr(sharedinputpath)), '###INPUT_SANDBOX###' : repr(subjob_input_sandbox + master_input_sandbox), '###CREATEINPUTDATALIST###' : getWNCodeForInputdataListCreation(job, ''), '###SHAREDOUTPUTPATH###' : repr(sharedoutputpath), '###OUTPUTPATTERNS###' : repr(outputpatterns), '###JOBID###' : jobidRepr, '###ENVIRONMENT###' : repr(environment), '###PREEXECUTE###' : self.config['preexecute'], '###POSTEXECUTE###' : self.config['postexecute'], '###JOBIDNAME###' : self.config['jobid_name'], '###QUEUENAME###' : self.config['queue_name'], '###HEARTBEATFREQUENCE###' : self.config['heartbeat_frequency'], '###INPUT_DIR###' : repr(job.getStringInputDir()), '###GANGADIR###' : repr(getConfig('System')['GANGA_PYTHONPATH']) } for k, v in replace_dict.iteritems(): text = text.replace(str(k), str(v)) logger.debug('subjob input sandbox %s ', subjob_input_sandbox) logger.debug('master input sandbox %s ', master_input_sandbox) from Ganga.GPIDev.Lib.File import FileBuffer return job.getInputWorkspace().writefile(FileBuffer('__jobscript__', text), executable=1)
def preparejob(self, jobconfig, master_input_sandbox): job = self.getJobObject() # print str(job.backend_output_postprocess) mon = job.getMonitoringService() import Ganga.Core.Sandbox as Sandbox subjob_input_sandbox = job.createPackedInputSandbox( jobconfig.getSandboxFiles() + Sandbox.getGangaModulesAsSandboxFiles(Sandbox.getDefaultModules()) + Sandbox.getGangaModulesAsSandboxFiles(mon.getSandboxModules()) ) appscriptpath = [jobconfig.getExeString()] + jobconfig.getArgStrings() if self.nice: appscriptpath = ["nice", "-n %d" % self.nice] + appscriptpath if self.nice < 0: logger.warning("increasing process priority is often not allowed, your job may fail due to this") sharedoutputpath = job.getOutputWorkspace().getPath() outputpatterns = jobconfig.outputbox environment = dict() if jobconfig.env is None else jobconfig.env import tempfile workdir = tempfile.mkdtemp(dir=config["location"]) import inspect script_location = os.path.join( os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))), "LocalHostExec.py" ) from Ganga.GPIDev.Lib.File import FileUtils script = FileUtils.loadScript(script_location, "") script = script.replace("###INLINEMODULES###", inspect.getsource(Sandbox.WNSandbox)) from Ganga.GPIDev.Lib.File.OutputFileManager import ( getWNCodeForOutputSandbox, getWNCodeForOutputPostprocessing, getWNCodeForDownloadingInputFiles, getWNCodeForInputdataListCreation, ) from Ganga.Utility.Config import getConfig jobidRepr = repr(job.getFQID(".")) script = script.replace( "###OUTPUTSANDBOXPOSTPROCESSING###", getWNCodeForOutputSandbox(job, ["stdout", "stderr", "__syslog__"], jobidRepr), ) script = script.replace("###OUTPUTUPLOADSPOSTPROCESSING###", getWNCodeForOutputPostprocessing(job, "")) script = script.replace("###DOWNLOADINPUTFILES###", getWNCodeForDownloadingInputFiles(job, "")) script = script.replace("###CREATEINPUTDATALIST###", getWNCodeForInputdataListCreation(job, "")) script = script.replace("###APPLICATION_NAME###", repr(job.application._name)) script = script.replace("###INPUT_SANDBOX###", repr(subjob_input_sandbox + master_input_sandbox)) script = script.replace("###SHAREDOUTPUTPATH###", repr(sharedoutputpath)) script = script.replace("###APPSCRIPTPATH###", repr(appscriptpath)) script = script.replace("###OUTPUTPATTERNS###", repr(outputpatterns)) script = script.replace("###JOBID###", jobidRepr) script = script.replace("###ENVIRONMENT###", repr(environment)) script = script.replace("###WORKDIR###", repr(workdir)) script = script.replace("###INPUT_DIR###", repr(job.getStringInputDir())) script = script.replace( "###MONITORING_SERVICE###", job.getMonitoringService().getWrapperScriptConstructorText() ) self.workdir = workdir script = script.replace("###GANGADIR###", repr(getConfig("System")["GANGA_PYTHONPATH"])) wrkspace = job.getInputWorkspace() scriptPath = wrkspace.writefile(FileBuffer("__jobscript__", script), executable=1) return scriptPath
def preparejob(self, jobconfig, master_input_sandbox): job = self.getJobObject() mon = job.getMonitoringService() import Ganga.Core.Sandbox as Sandbox from Ganga.GPIDev.Lib.File import File from Ganga.Core.Sandbox.WNSandbox import PYTHON_DIR import inspect fileutils = File(inspect.getsourcefile(Ganga.Utility.files), subdir=PYTHON_DIR) sharedfiles = jobconfig.getSharedFiles() subjob_input_sandbox = job.createPackedInputSandbox( jobconfig.getSandboxFiles() + [fileutils]) appscriptpath = [jobconfig.getExeString()] + jobconfig.getArgStrings() sharedoutputpath = job.getOutputWorkspace().getPath() ## FIXME Check this isn't a GangaList outputpatterns = jobconfig.outputbox environment = jobconfig.env if not jobconfig.env is None else {} import inspect script_location = os.path.join( os.path.dirname( os.path.abspath(inspect.getfile(inspect.currentframe()))), 'BatchScriptTemplate.py.template') from Ganga.GPIDev.Lib.File import FileUtils text = FileUtils.loadScript(script_location, '') import Ganga.Core.Sandbox as Sandbox import Ganga.Utility as Utility from Ganga.Utility.Config import getConfig from Ganga.GPIDev.Lib.File.OutputFileManager import getWNCodeForOutputSandbox, getWNCodeForOutputPostprocessing, getWNCodeForDownloadingInputFiles, getWNCodeForInputdataListCreation jobidRepr = repr(self.getJobObject().getFQID('.')) replace_dict = { '###OUTPUTSANDBOXPOSTPROCESSING###': getWNCodeForOutputSandbox(job, ['__syslog__'], jobidRepr), '###OUTPUTUPLOADSPOSTPROCESSING###': getWNCodeForOutputPostprocessing(job, ''), '###DOWNLOADINPUTFILES###': getWNCodeForDownloadingInputFiles(job, ''), '###INLINEMODULES###': inspect.getsource(Sandbox.WNSandbox), '###INLINEHOSTNAMEFUNCTION###': inspect.getsource(Utility.util.hostname), '###APPSCRIPTPATH###': repr(appscriptpath), #'###SHAREDINPUTPATH###' : repr(sharedinputpath)), '###INPUT_SANDBOX###': repr(subjob_input_sandbox + master_input_sandbox + sharedfiles), '###CREATEINPUTDATALIST###': getWNCodeForInputdataListCreation(job, ''), '###SHAREDOUTPUTPATH###': repr(sharedoutputpath), '###OUTPUTPATTERNS###': repr(outputpatterns), '###JOBID###': jobidRepr, '###ENVIRONMENT###': repr(environment), '###PREEXECUTE###': self.config['preexecute'], '###POSTEXECUTE###': self.config['postexecute'], '###JOBIDNAME###': self.config['jobid_name'], '###QUEUENAME###': self.config['queue_name'], '###HEARTBEATFREQUENCE###': self.config['heartbeat_frequency'], '###INPUT_DIR###': repr(job.getStringInputDir()), '###GANGADIR###': repr(getConfig('System')['GANGA_PYTHONPATH']) } for k, v in replace_dict.iteritems(): text = text.replace(str(k), str(v)) logger.debug('subjob input sandbox %s ', subjob_input_sandbox) logger.debug('master input sandbox %s ', master_input_sandbox) from Ganga.GPIDev.Lib.File import FileBuffer return job.getInputWorkspace().writefile(FileBuffer( '__jobscript__', text), executable=1)