def prepare(self,app,appconfig,appmasterconfig,jobmasterconfig): from Ganga.GPIDev.Adapters.StandardJobConfig import StandardJobConfig c = StandardJobConfig(app.exe,app.inputs,app.args,app.outputs,app.envs) c.monitoring_svc = mc['GangaTutorial'] return c
def prepare(self, app, appconfig, appmasterconfig, jobmasterconfig): from Ganga.GPIDev.Adapters.StandardJobConfig import StandardJobConfig c = StandardJobConfig(app.exe, app.inputs, app.args, app.outputs, app.envs) c.monitoring_svc = mc['GangaTutorial'] return c
def test_LHCbGaudiDiracRunTimeHandler_prepare(self): # app = self.app # app.extra = self.extra sjc = StandardJobConfig(inputbox=self.input_files, outputbox=["dummy1.out", "dummy2.out", "dummy3.out"]) jobmasterconfig = StandardJobConfig() jobmasterconfig.outputdata = [] stdjobconfig = self.rth.prepare(self.app, sjc, self.appmasterconfig, jobmasterconfig) # should have subjob.in(buffer), data.opts and gaudiscript.py print("sandbox =", stdjobconfig.getSandboxFiles()) print("sandbox =", [file.name for file in stdjobconfig.getSandboxFiles()]) assert len(stdjobconfig.getSandboxFiles()) == 3, "inputsandbox error" l = len(stdjobconfig.getOutputSandboxFiles()) print("outputsandbox =", stdjobconfig.getOutputSandboxFiles()) assert l == 3, "outputsandbox error"
def prepare(self, app, appconfig, appmasterconfig, jobmasterconfig): # some extra lines for simulation job options on DIRAC site opts = 'DatabaseSvc.DbType = "sqlite";\n' app.extra.input_buffers['data.opts'] += opts # extra lines for reconstruction and remove empty files if app.recoptsfile: app.extra.input_buffers['recdata.opts'] += opts extraopts = 'MixerAlg.UseNewDataDir = ".";\n' app.extra.input_buffers['recdata.opts'] += extraopts else: app.extra.input_buffers.pop('recdata.opts', None) # extra lines for analysis and remove empty files if app.anaoptsfile: app.extra.input_buffers['anadata.opts'] += opts else: app.extra.input_buffers.pop('anadata.opts', None) if app.extra.inputdata and app.extra.inputdata.hasLFNs(): cat_opts = '\nFileCatalog().Catalogs = ' \ '["xmlcatalog_file:pool_xml_catalog.xml"]\n' # app.extra.input_buffers['data.py'] += cat_opts script = self._create_boss_script(app) sandbox = get_input_sandbox(app.extra) app.extra.outputsandbox += [ 'script.log', 'rantrg.log', 'bosslog', 'bosserr', 'recbosslog', 'recbosserr', 'anabosslog', 'anabosserr' ] outputsandbox = app.extra.outputsandbox c = StandardJobConfig(script, sandbox, [], outputsandbox, None) dirac_script = DiracScript() dirac_script.job_type = 'Job()' dirac_script.exe = DiracApplication(app, script) dirac_script.platform = app.platform dirac_script.output_sandbox = outputsandbox dirac_script.default_sites = gConfig.getValue( '/Resources/Applications/DefaultSites/%s' % DfcOperation().getGroupName(), []) if app.extra.inputdata: dirac_script.inputdata = DiracInputData(app.extra.inputdata) if app.extra.outputdata: dirac_script.outputdata = app.extra.outputdata c.script = dirac_script return c
def test_GaudiRunTimeHandler_prepare(self): sjc = StandardJobConfig( inputbox=self.input_files, outputbox=['dummy1.out', 'dummy2.out', 'dummy3.out']) jobmasterconfig = StandardJobConfig() jobmasterconfig.outputdata = [] stdjobconfig = self.rth.prepare( self.app, sjc, self.appmasterconfig, jobmasterconfig) # should have subjob.in(buffer), data.opts and gaudiscript.py print("sandbox =", stdjobconfig.getSandboxFiles()) print( "sandbox =", [file.name for file in stdjobconfig.getSandboxFiles()]) assert len(stdjobconfig.getSandboxFiles()) == 4, 'inputsandbox error' l = len(stdjobconfig.getOutputSandboxFiles()) print("outputsandbox =", stdjobconfig.getOutputSandboxFiles()) assert l == 4, 'outputsandbox error'
def master_prepare(self, app, appmasterconfig): """ Prepare the RTHandler for the master job so that applications to be submitted Args: app (GaudiExec): This application is only expected to handle GaudiExec Applications here appmasterconfig (unknown): Output passed from the application master configuration call """ inputsandbox, outputsandbox = master_sandbox_prepare( app, appmasterconfig) if isinstance(app.jobScriptArchive, LocalFile): app.jobScriptArchive = None generateJobScripts(app, appendJobScripts=True) scriptArchive = os.path.join(app.jobScriptArchive.localDir, app.jobScriptArchive.namePattern) inputsandbox.append(File(name=scriptArchive)) if app.getMetadata: logger.info("Adding options to make the summary.xml") inputsandbox.append( FileBuffer( 'summary.py', "\nfrom Gaudi.Configuration import *\nfrom Configurables import LHCbApp\nLHCbApp().XMLSummary='summary.xml'" )) return StandardJobConfig(inputbox=unique(inputsandbox), outputbox=unique(outputsandbox))
def master_prepare(self, app, appmasterconfig): """ Prepare the RTHandler for the master job so that applications to be submitted Args: app (GaudiExec): This application is only expected to handle GaudiExec Applications here appmasterconfig (unknown): Output passed from the application master configuration call """ inputsandbox, outputsandbox = master_sandbox_prepare(app, appmasterconfig) # If we are getting the metadata we need to make sure the summary.xml is added to the output sandbox if not there already. if app.getMetadata and not 'summary.xml' in outputsandbox: outputsandbox += ['summary.xml'] if not isinstance(app.uploadedInput, DiracFile): generateDiracInput(app) assert isinstance(app.uploadedInput, DiracFile), "Failed to upload needed file, aborting submit. Tried to upload to: %s\nIf your Ganga installation is not at CERN your username may be trying to create a non-existent LFN. Try setting the 'DIRAC' configuration 'DiracLFNBase' to your grid user path.\n" % DiracFile.diracLFNBase() rep_data = app.uploadedInput.getReplicas() assert rep_data != {}, "Failed to find a replica, aborting submit" if isinstance(app.jobScriptArchive, (DiracFile, LocalFile)): app.jobScriptArchive = None generateDiracScripts(app) assert isinstance(app.jobScriptArchive, DiracFile), "Failed to upload needed file, aborting submit" rep_data = app.jobScriptArchive.getReplicas() assert rep_data != {}, "Failed to find a replica, aborting submit" return StandardJobConfig(inputbox=unique(inputsandbox), outputbox=unique(outputsandbox))
def configure(self, master_appconfig): ## strip leading and trailing blanks from arguments self.arguments = [ a.strip() for a in self.arguments ] ## strip leading and trailing blanks from the command self.commands = [ a.strip() for a in self.commands ] ## the script layout the_script = layout.format ( scripts = [ os.path.join ( f.subdir , os.path.basename ( f.name ) ) for f in self.scripts ] , imports = [ os.path.join ( f.subdir , os.path.basename ( f.name ) ) for f in self.imports ] , arguments = self.arguments , command = self.commands ) print 'SCRIPT:\n', the_script # add summary.xml outputsandbox_temp = XMLPostProcessor._XMLJobFiles() outputsandbox_temp += unique(self.getJobObject().outputsandbox) outputsandbox = unique(outputsandbox_temp) input_files = [] input_files += [ FileBuffer('gaudipython-wrapper.py', the_script ) ] logger.debug("Returning StandardJobConfig") return (None, StandardJobConfig(inputbox=input_files, outputbox=outputsandbox))
def prepare(self,app,appconfig,appmasterconfig,jobmasterconfig): from Ganga.GPIDev.Adapters.StandardJobConfig import StandardJobConfig #prepared_exe = File(os.path.join(shared_path,self.is_prepared.name, self.script_name)) prepared_exe = File(os.path.join( os.path.dirname(__file__), "runNA62MC.sh")) #if app.is_prepared is not None: # logger.info("Submitting a prepared application; taking any input files from %s" %(app.is_prepared.name)) # prepared_exe = File(os.path.join(os.path.join(shared_path,app.is_prepared.name),os.path.basename(prepared_exe.name))) inputbox = [ File(os.path.join(os.path.join(shared_path,app.is_prepared.name), app.getMACFileName())) ] inputbox.append( File(os.path.join(shared_path,app.is_prepared.name, app.script_name)) ) inputbox.append( File(os.path.join(shared_path,app.is_prepared.name, "input_files.tgz")) ) inputbox += app._getParent().inputsandbox #outputbox = [ 'na62run%d.stderr.err' % app.run_number, 'na62run%d.stdout.out' % app.run_number ] outputbox = [] outputbox += app._getParent().outputsandbox env = {'NA62SCRIPT':app.script_name, 'NA62STDOUT':'na62run%d.out' % app.run_number, 'NA62STDERR':'na62run%d.err' % app.run_number} args = [app.getMACFileName(), "%s/r%d/%s" % (app.job_type, app.getRevision(), app.getDecayString())] c = StandardJobConfig(prepared_exe,inputbox,args,outputbox,env) return c
def master_prepare(self, app, appconfig): inputsandbox = app._getParent().inputsandbox[:] if type(app.exe) == File: os.system('chmod +x %s' % app.exe.name) inputsandbox.append(app.exe) c = StandardJobConfig('', inputsandbox, [], [], None) return c
def setUp(self): j = Job(application=DaVinci()) j.prepare() from Ganga.Utility.Config import getConfig if getConfig('Output')['ForbidLegacyInput']: j.inputfiles = [LocalFile(name='dummy.in')] else: j.inputsandbox = [File(name='dummy.in')] self.app = j.application._impl #self.extra = GaudiExtras() # self.extra.master_input_buffers['master.buffer'] = '###MASTERBUFFER###' #self.extra.master_input_files = [File(name='master.in')] # self.extra.input_buffers['subjob.buffer'] = '###SUBJOBBUFFER###' self.input_files = [ File(name='subjob.in'), File( FileBuffer('subjob.buffer', '###SUBJOBBUFFER###').create().name) ] self.appmasterconfig = StandardJobConfig(inputbox=[ File(name='master.in'), File( FileBuffer('master.buffer', '###MASTERBUFFER###').create().name) ]) j.outputfiles = ['dummy1.out', 'dummy2.out', 'dummy3.out'] self.rth = LHCbGaudiRunTimeHandler()
def test_GaudiRunTimeHandler_prepare(self): sjc = StandardJobConfig( inputbox=self.input_files, outputbox=['dummy1.out', 'dummy2.out', 'dummy3.out']) jobmasterconfig = StandardJobConfig() jobmasterconfig.outputdata = [] stdjobconfig = self.rth.prepare(self.app, sjc, self.appmasterconfig, jobmasterconfig) # should have subjob.in(buffer), data.opts and gaudiscript.py print("sandbox =", stdjobconfig.getSandboxFiles()) print("sandbox =", [file.name for file in stdjobconfig.getSandboxFiles()]) assert len(stdjobconfig.getSandboxFiles()) == 4, 'inputsandbox error' l = len(stdjobconfig.getOutputSandboxFiles()) print("outputsandbox =", stdjobconfig.getOutputSandboxFiles()) assert l == 4, 'outputsandbox error'
def master_prepare(self, app, appconfig): # check file is set OK if not app.script.name: msg = 'Root.script.name must be set.' raise ApplicationConfigurationError(None, msg) if not os.path.exists(app.script.name): msg = 'Script must exist!' raise ApplicationConfigurationError(None, msg) # check root version global rootVersions if not rootVersions: from Dirac import Dirac result = Dirac.execAPI('result = DiracCommands.getRootVersions()') if not result_ok(result): logger.error('Could not obtain available ROOT versions: %s' \ % str(result)) logger.error('ROOT version will not be validated.') else: rootVersions = result['Value'] if rootVersions: found = False versions = [] for v in rootVersions: versions.append(v) if app.version.find(v) >= 0: found = True break if not found: msg = 'Invalid ROOT version: %s. Valid versions: %s' \ % (app.version, str(versions)) raise ApplicationConfigurationError(None, msg) inputsandbox = app._getParent().inputsandbox[:] c = StandardJobConfig('', inputsandbox, [], [], None) return c
def prepare(self, app, appconfig, appmasterconfig, jobmasterconfig): from Ganga.GPIDev.Adapters.StandardJobConfig import StandardJobConfig c = StandardJobConfig(app._scriptname, app._getParent().inputsandbox, [], app._getParent().outputsandbox, app.env) return c
def prepare(self, app, appconfig, appmasterconfig, jobmasterconfig): """ Prepare the job in order to submit to the Local backend Args: app (GaudiExec): This application is only expected to handle GaudiExec Applications here appconfig (unknown): Output passed from the application configuration call appmasterconfig (unknown): Output passed from the application master_configure call jobmasterconfig (tuple): Output from the master job prepare step """ job = app.getJobObject() # Setup the command which to be run and the input and output input_sand = genDataFiles(job) output_sand = [] # NB with inputfiles the mechanics of getting the inputfiled to the input of the Localhost backend is taken care of for us # We don't have to do anything to get our files when we start running # Also we don't manage the outputfiles here! job_command = prepareCommand(app) # Generate the script which is to be executed for us on the WN scriptToRun = generateWNScript(job_command, app) input_sand.append(scriptToRun) logger.debug("input_sand: %s" % input_sand) # It's this authors opinion that the script should be in the PATH on the WN # As it stands policy is that is isn't so we have to call it in a relative way, hence "./" c = StandardJobConfig('./'+os.path.join(scriptToRun.subdir, scriptToRun.name), input_sand, [], output_sand) return c
def _preparePyRootJobConfig(self, app, appconfig, appmasterconfig, jobmasterconfig): """JobConfig for executing a Root script using CINT.""" from Ganga.GPIDev.Adapters.StandardJobConfig import StandardJobConfig from os.path import join, split from Ganga.GPIDev.Lib.File import getSharedPath script = app.script if script == File(): script = File(defaultPyRootScript()) else: script = File( os.path.join( os.path.join(Ganga.GPIDev.Lib.File.getSharedPath(), app.is_prepared.name), os.path.basename(app.script.name))) arguments = [join('.', script.subdir, split(script.name)[1])] arguments.extend([str(s) for s in app.args]) arguments.append('-b') inputsandbox = app._getParent().inputsandbox + [script] (rootenv, _) = self._getRootEnvSys(app.version, usepython=True) logger.debug("PyRoot environment:\n %s: ", str(rootenv)) return StandardJobConfig('python', inputsandbox, arguments, app._getParent().outputsandbox)
def prepare(self, app, appconfig, appmasterconfig, jobmasterconfig): logger.debug('RTHandler prepare called.') from Ganga.GPIDev.Adapters.StandardJobConfig import StandardJobConfig return StandardJobConfig(app.exe, app._getParent().inputsandbox, convertIntToStringArgs(app.args), app._getParent().outputsandbox, app.env)
def prepare(self, app, appconfig, appmasterconfig, jobmasterconfig): j = app.getJobObject() script = self._create_exe_script(app) c = StandardJobConfig(script, [File(script)], app.args, j.outputsandbox, app.env) dirac_script = DiracScript() dirac_script.job_type = 'Job()' #dirac_script.exe = DiracExe(app.exe,app.args) dirac_script.exe = DiracExe('exe-script.py', []) dirac_script.output_sandbox = j.outputsandbox[:] if j.inputdata: dirac_script.inputdata = DiracInputData(j.inputdata) if j.outputdata: dirac_script.outputdata = j.outputdata c.script = dirac_script return c
def master_prepare(self, app, extra): if extra.inputdata and extra.inputdata.hasLFNs(): xml_catalog_str = extra.inputdata.getCatalog() extra.master_input_buffers['catalog.xml'] = xml_catalog_str sandbox = get_master_input_sandbox(app.getJobObject(), extra) return StandardJobConfig('', inputbox=sandbox, args=[])
def prepare(self,app,appconfig,appmasterconfig,jobmasterconfig): j = app.getJobObject() script = self._create_exe_script(app) c = StandardJobConfig(script,[File(script)],app.args, j.outputsandbox,app.env) dirac_script = DiracScript() dirac_script.job_type = 'Job()' #dirac_script.exe = DiracExe(app.exe,app.args) dirac_script.exe = DiracExe('exe-script.py',[]) dirac_script.output_sandbox = j.outputsandbox[:] if j.inputdata: dirac_script.inputdata = DiracInputData(j.inputdata) if j.outputdata: dirac_script.outputdata = j.outputdata c.script = dirac_script return c
def master_prepare(self, app, appmasterconfig): inputsandbox, outputsandbox = master_sandbox_prepare(app, appmasterconfig, ['inputsandbox']) # add summary.xml outputsandbox += ['summary.xml', '__parsedxmlsummary__'] return StandardJobConfig(inputbox=unique(inputsandbox), outputbox=unique(outputsandbox))
def prepare(self, app, appconfig, appmasterconfig, jobmasterconfig): from Ganga.GPIDev.Adapters.StandardJobConfig import StandardJobConfig runScript, inputsandbox, rootenv = downloadWrapper(app) # propage command line args argList = [str(s) for s in app.args] return StandardJobConfig(runScript, inputsandbox, argList, app._getParent().outputsandbox)
def master_prepare(self, app, appmasterconfig): """ This function prepares the application of a master job during submit. A priori we aren't doing anything with this in Im3ShapeApp but until this is understood I'd rather not remove it Args: app (IApplication): This is the application given in the master job appasterconfig (tuple): This is the configuration which is to prepare the app in the master job # TODO check type and this interface """ inputsandbox, outputsandbox = master_sandbox_prepare(app, appmasterconfig) return StandardJobConfig(inputbox=unique(inputsandbox), outputbox=unique(outputsandbox))
def prepare(self, app, appconfig, appmasterconfig, jobmasterconfig): j = app.getJobObject() input_dir = j.getInputWorkspace().getPath() script_name = '%s/%s' % (input_dir, os.path.basename(app.script.name)) c = StandardJobConfig(script_name, [], [], j.outputsandbox, {}) # copy script to input dir os.system('cp %s %s/.' % (app.script.name, input_dir)) dirac_script = DiracScript() dirac_script.job_type = 'Job()' dirac_script.exe = DiracRoot(app, script_name) dirac_script.platform = getConfig('ROOT')['arch'] dirac_script.output_sandbox = j.outputsandbox[:] if j.inputdata: dirac_script.inputdata = DiracInputData(j.inputdata) if j.outputdata: dirac_script.outputdata = j.outputdata c.script = dirac_script return c
def prepare(self,app,appconfig,appmasterconfig,jobmasterconfig): j = app.getJobObject() input_dir = j.getInputWorkspace().getPath() script_name = '%s/%s' % (input_dir,os.path.basename(app.script.name)) c = StandardJobConfig(script_name,[],[],j.outputsandbox,{}) # copy script to input dir os.system('cp %s %s/.' % (app.script.name,input_dir)) dirac_script = DiracScript() dirac_script.job_type = 'Job()' dirac_script.exe = DiracRoot(app,script_name) dirac_script.platform = getConfig('ROOT')['arch'] dirac_script.output_sandbox = j.outputsandbox[:] if j.inputdata: dirac_script.inputdata = DiracInputData(j.inputdata) if j.outputdata: dirac_script.outputdata = j.outputdata c.script = dirac_script return c
def master_prepare(self, app, appmasterconfig): inputsandbox, outputsandbox = master_sandbox_prepare( app, appmasterconfig) if type(app.exe) == File: input_dir = app.getJobObject().getInputWorkspace().getPath() exefile = os.path.join(input_dir, os.path.basename(app.exe.name)) if not os.path.exists(exefile): msg = 'Executable: "%s" must exist!' % str(exefile) raise ApplicationConfigurationError(None, msg) os.system('chmod +x %s' % exefile) return StandardJobConfig(inputbox=unique(inputsandbox), outputbox=unique(outputsandbox))
def master_prepare(self, app, appmasterconfig): logger.debug("Master Prepare") inputsandbox, outputsandbox = master_sandbox_prepare( app, appmasterconfig, ['inputsandbox']) # add summary.xml outputsandbox += ['summary.xml', '__parsedxmlsummary__'] logger.debug("Master Prepare LHCbGaudiDiracRunTimeHandler") return StandardJobConfig(inputbox=unique(inputsandbox), outputbox=unique(outputsandbox))
def prepare(self, app, appsubconfig, appmasterconfig, jobmasterconfig): inputsandbox, outputsandbox = sandbox_prepare(app, appsubconfig, appmasterconfig, jobmasterconfig) run_script = self.__create_run_script(app, appsubconfig, appmasterconfig, jobmasterconfig, inputsandbox, outputsandbox) return StandardJobConfig(FileBuffer('gaudi-script.py', run_script, executable=1), inputbox=unique(inputsandbox), outputbox=unique(outputsandbox))
def prepare(self, app, appsubconfig, appmasterconfig, jobmasterconfig): inputsandbox, outputsandbox = sandbox_prepare( app, appsubconfig, appmasterconfig, jobmasterconfig) input_data, parametricinput_data = dirac_inputdata(app) job = app.getJobObject() outputfiles = [this_file.namePattern for this_file in job.outputfiles if isinstance(this_file, DiracFile)] gaudi_script_path = os.path.join( job.getInputWorkspace().getPath(), "gaudi-script.py") script_generator(gaudi_script_template(), #remove_unreplaced = False, outputfile_path=gaudi_script_path, PLATFORM=app.platform, COMMAND='gaudirun.py' # , #OUTPUTFILESINJECTEDCODE = getWNCodeForOutputPostprocessing(job, ' ') ) dirac_script = script_generator(diracAPI_script_template(), DIRAC_IMPORT='from DIRAC.Interfaces.API.Dirac import Dirac', DIRAC_JOB_IMPORT='from DIRAC.Interfaces.API.Job import Job', DIRAC_OBJECT='Dirac()', JOB_OBJECT='Job()', NAME=mangle_job_name(app), EXE=gaudi_script_path, EXE_ARG_STR=' '.join( [str(arg) for arg in app.args]), EXE_LOG_FILE='Ganga_%s_%s.log' % ( app.appname, app.version), INPUTDATA=input_data, PARAMETRIC_INPUTDATA=parametricinput_data, OUTPUT_SANDBOX=outputsandbox, OUTPUTDATA=list(outputfiles), OUTPUT_PATH="", # job.fqid, OUTPUT_SE=getConfig( 'DIRAC')['DiracOutputDataSE'], SETTINGS=diracAPI_script_settings(app), DIRAC_OPTS=job.backend.diracOpts, PLATFORM=app.platform, # leave the sandbox for altering later as needs # to be done in backend.submit to combine master. # Note only using 2 #s as auto-remove 3 INPUT_SANDBOX='##INPUT_SANDBOX##' ) return StandardJobConfig(dirac_script, inputbox=unique(inputsandbox), outputbox=unique(outputsandbox))
def _prepareCintJobConfig(self, app, appconfig, appmasterconfig, jobmasterconfig): """JobConfig for executing a Root script using CINT.""" from Ganga.GPIDev.Adapters.StandardJobConfig import StandardJobConfig from os.path import join, split import string from Ganga.GPIDev.Lib.File import getSharedPath # Arguments to the ROOT script needs to be a comma separated list # enclosed in (). Strings should be enclosed in escaped double quotes. arglist = [] for arg in app.args: if isinstance(arg, str): arglist.append(self.quoteCintArgString(arg)) else: arglist.append(arg) rootarg = '\(' + string.join([repr(s) for s in arglist], ',') + '\)' script = app.script if script == File(): script = File(defaultScript()) else: script = File( os.path.join( os.path.join(Ganga.GPIDev.Lib.File.getSharedPath(), app.is_prepared.name), os.path.basename(app.script.name))) # Start ROOT with the -b and -q options to run without a # terminal attached. arguments = [ '-b', '-q', os.path.relpath(join('.', script.subdir, split(script.name)[1])) + rootarg ] inputsandbox = app._getParent().inputsandbox + [script] (rootenv, _) = self._getRootEnvSys(app.version) logger.debug("ROOT environment:\n %s: ", str(rootenv)) returnable = StandardJobConfig('root.exe', inputsandbox, arguments, app._getParent().outputsandbox) logger.debug("root jobconfig: %s" % str(returnable)) return returnable
def master_prepare(self, app, appmasterconfig): inputsandbox, outputsandbox = master_sandbox_prepare( app, appmasterconfig) # check file is set OK if not app.script.name: msg = 'Root.script.name must be set.' raise ApplicationConfigurationError(msg) sharedir_scriptpath = os.path.join(get_share_path(app), os.path.basename(app.script.name)) if not os.path.exists(sharedir_scriptpath): msg = 'Script must exist!' raise ApplicationConfigurationError(msg) return StandardJobConfig(inputbox=unique(inputsandbox), outputbox=unique(outputsandbox))
def prepare(self, app, extra, appmasterconfig, jobmasterconfig): if extra.inputdata and extra.inputdata.hasLFNs(): s = '\nFileCatalog().Catalogs = ["xmlcatalog_file:catalog.xml"]\n' extra.input_buffers['data.py'] += s sandbox = get_input_sandbox(extra) outdata = extra.outputdata if not outdata: outdata = app.getJobObject().outputdata script = create_runscript(app, outdata, app.getJobObject()) #logger.error("zhangxm log: gaudiscript.py:\n %s " % script) return StandardJobConfig(FileBuffer('gaudiscript.py', script, executable=1), inputbox=sandbox, args=[], outputbox=extra.outputsandbox)
def test_submit(self): j = Job(backend=self.db) self.db._parent = j._impl file1 = tempfile.NamedTemporaryFile('w') file2 = tempfile.NamedTemporaryFile('w') file3 = tempfile.NamedTemporaryFile('w') sjc = StandardJobConfig(exe=self.script, inputbox=[File(file1.name)._impl, File(file2.name)._impl, File(file3.name)._impl], outputbox=['d', 'e', 'f']) def _addition_sandbox_content(subjobconfig): self.assertEqual(subjobconfig, sjc, 'config objects not equal') return ['g'] def _common_submit(dirac_script): # this needs to change to workerpool #from GangaDirac.Lib.Server.DiracClient import DiracClient #self.assertTrue(isinstance(server, DiracClient),'not a dirac client') f = open(dirac_script, 'r') script = f.read() f.close() self.assertNotEqual(script, self.script, 'script not changed') self.assertEqual(self.script.replace('##INPUT_SANDBOX##', str(['a', 'b', 'c'] + [os.path.join(j._impl.getInputWorkspace().getPath(), '_input_sandbox_0.tgz')] + ['g'])), script, 'script not what it should be') return True setattr( self.db, '_addition_sandbox_content', _addition_sandbox_content) setattr(self.db, '_common_submit', _common_submit) self.assertTrue(self.db.submit(sjc, ['a', 'b', 'c']), 'didnt run') file1.close() file2.close() file3.close()
def master_prepare(self, app, appmasterconfig): """ Prepare the RTHandler for the master job so that applications to be submitted Args: app (GaudiExec): This application is only expected to handle GaudiExec Applications here appmasterconfig (unknown): Output passed from the application master configuration call """ inputsandbox, outputsandbox = master_sandbox_prepare(app, appmasterconfig) if isinstance(app.jobScriptArchive, LocalFile): app.jobScriptArchive = None generateJobScripts(app, appendJobScripts=True) scriptArchive = os.path.join(app.jobScriptArchive.localDir, app.jobScriptArchive.namePattern) inputsandbox.append(File(name=scriptArchive)) return StandardJobConfig(inputbox=unique(inputsandbox), outputbox=unique(outputsandbox))
def configure(self, master_appconfig): # self._configure() name = join('.', self.script[0].subdir, split(self.script[0].name)[-1]) script = "from Gaudi.Configuration import *\n" if self.args: script += 'import sys\nsys.argv += %s\n' % str(self.args) script += "importOptions('data.py')\n" script += "execfile(\'%s\')\n" % name # add summary.xml outputsandbox_temp = XMLPostProcessor._XMLJobFiles() outputsandbox_temp += unique(self.getJobObject().outputsandbox) outputsandbox = unique(outputsandbox_temp) input_files = [] input_files += [FileBuffer('gaudipython-wrapper.py', script)] logger.debug("Returning Job Configuration") return (None, StandardJobConfig(inputbox=input_files, outputbox=outputsandbox))
def __init__(self, exe=None, inputbox=[], outputbox=[], args=[], env=None): if len(env) != 0: logger.error("Clobi backend currently does not support exe env") env = {} StandardJobConfig.__init__(self,exe,inputbox,args,outputbox,env)