def _check_inputs(self): """Checks the validity of some of user's entries for Gaudi schema""" # Warn user that no optsfiles given if len(self.optsfile) == 0: logger.warning("The 'optsfile' is not set. I hope this is OK!") else: # Check for non-exting optsfiles defined nonexistentOptFiles = [] for f in self.optsfile: from Ganga.GPIDev.Lib.File.File import File if type(f) is str: myFile = File(f) else: myFile = f myFile.name = fullpath(myFile.name) if not os.path.isfile(myFile.name): nonexistentOptFiles.append(myFile) if len(nonexistentOptFiles): tmpmsg = "The 'optsfile' attribute contains non-existent file/s: [" for _f in nonexistentOptFiles: tmpmsg += "'%s', " % _f.name msg = tmpmsg[:-2] + ']' raise ApplicationConfigurationError(msg)
def _check_inputs(self): """Checks the validity of some of user's entries for Gaudi schema""" # Warn user that no optsfiles given if len(self.optsfile) == 0: logger.warning("The 'optsfile' is not set. I hope this is OK!") else: # Check for non-exting optsfiles defined nonexistentOptFiles = [] for f in self.optsfile: from Ganga.GPIDev.Lib.File.File import File if type(f) is str: myFile = File(f) else: myFile = f myFile.name = fullpath(myFile.name) if not os.path.isfile(myFile.name): nonexistentOptFiles.append(myFile) if len(nonexistentOptFiles): tmpmsg = "The 'optsfile' attribute contains non-existent file/s: [" for _f in nonexistentOptFiles: tmpmsg += "'%s', " % _f.name msg = tmpmsg[:-2] + ']' raise ApplicationConfigurationError(msg)
def prepare(self, app, appconfig, appmasterconfig, jobmasterconfig): from Ganga.Lib.gLite import gLiteJobConfig prepared_exe = app.exe if app.is_prepared is not None: shared_path = os.path.join(expandfilename(getConfig('Configuration')['gangadir']), 'shared', getConfig('Configuration')['user']) if isinstance(app.exe, str): # we have a file. is it an absolute path? if os.path.abspath(app.exe) == app.exe: logger.info("Submitting a prepared application; taking any input files from %s" % ( app.is_prepared.name)) prepared_exe = File(os.path.join(os.path.join( shared_path, app.is_prepared.name), os.path.basename(File(app.exe).name))) # else assume it's a system binary, so we don't need to # transport anything to the sharedir else: prepared_exe = app.exe elif isinstance(app.exe, File): logger.info("Submitting a prepared application; taking any input files from %s" % ( app.is_prepared.name)) prepared_exe = File(os.path.join(os.path.join( shared_path, app.is_prepared.name), os.path.basename(File(app.exe).name))) return gLiteJobConfig(prepared_exe, app._getParent().inputsandbox, convertIntToStringArgs(app.args), app._getParent().outputsandbox, app.env)
def setUp(self): j = Job(application=DaVinci()) j.prepare() from Ganga.Utility.Config import getConfig if getConfig('Output')['ForbidLegacyInput']: j.inputfiles = [LocalFile(name='dummy.in')] else: j.inputsandbox = [File(name='dummy.in')] self.app = j.application._impl #self.extra = GaudiExtras() # self.extra.master_input_buffers['master.buffer'] = '###MASTERBUFFER###' #self.extra.master_input_files = [File(name='master.in')] # self.extra.input_buffers['subjob.buffer'] = '###SUBJOBBUFFER###' self.input_files = [ File(name='subjob.in'), File( FileBuffer('subjob.buffer', '###SUBJOBBUFFER###').create().name) ] self.appmasterconfig = StandardJobConfig(inputbox=[ File(name='master.in'), File( FileBuffer('master.buffer', '###MASTERBUFFER###').create().name) ]) j.outputfiles = ['dummy1.out', 'dummy2.out', 'dummy3.out'] self.rth = LHCbGaudiRunTimeHandler()
def master_prepare(self, app, appmasterconfig): """ Prepare the RTHandler for the master job so that applications to be submitted Args: app (GaudiExec): This application is only expected to handle GaudiExec Applications here appmasterconfig (unknown): Output passed from the application master configuration call """ inputsandbox, outputsandbox = master_sandbox_prepare( app, appmasterconfig) if isinstance(app.jobScriptArchive, LocalFile): app.jobScriptArchive = None generateJobScripts(app, appendJobScripts=True) scriptArchive = os.path.join(app.jobScriptArchive.localDir, app.jobScriptArchive.namePattern) inputsandbox.append(File(name=scriptArchive)) if app.getMetadata: logger.info("Adding options to make the summary.xml") inputsandbox.append( FileBuffer( 'summary.py', "\nfrom Gaudi.Configuration import *\nfrom Configurables import LHCbApp\nLHCbApp().XMLSummary='summary.xml'" )) return StandardJobConfig(inputbox=unique(inputsandbox), outputbox=unique(outputsandbox))
def prepare(self, force=False): from Ganga.GPIDev.Lib.GangaList.GangaList import GangaList from Ganga.GPIDev.Lib.File.File import File if isType(self.optsfile, (list, tuple, GangaList)): for this_file in self.optsfile: if type(this_file) is str: this_file = File(this_file) else: continue elif type(self.optsfile) is str: self.optsfile = [File(self.optsfile)] try: super(Gaudi, self).prepare(force) except Exception, err: logger.debug("Super Prepare Error:\n%s" % str(err)) raise err
def master_prepare(self, masterjobconfig): """ Prepare the master job (shared sandbox files). This method is/should be called by master_submit() exactly once. The input sandbox is created according to self._packed_input_sandbox flag (a class attribute) """ from Ganga.GPIDev.Lib.File.OutputFileManager import getInputFilesPatterns from Ganga.GPIDev.Lib.File.File import File, ShareDir job = self.getJobObject() create_sandbox = job.createInputSandbox if self._packed_input_sandbox: create_sandbox = job.createPackedInputSandbox if masterjobconfig: if hasattr(job.application, 'is_prepared') and isType( job.application.is_prepared, ShareDir): sharedir_pred = lambda f: f.name.find(job.application. is_prepared.name) > -1 sharedir_files = itertools.ifilter( sharedir_pred, masterjobconfig.getSandboxFiles()) nonsharedir_files = itertools.ifilterfalse( sharedir_pred, masterjobconfig.getSandboxFiles()) # ATLAS use bool to bypass the prepare mechanism and some ATLAS # apps have no is_prepared else: sharedir_files = [] nonsharedir_files = masterjobconfig.getSandboxFiles() inputsandbox = create_sandbox(nonsharedir_files, master=True) inputsandbox.extend( itertools.imap(lambda f: f.name, sharedir_files)) return inputsandbox tmpDir = None files = [] if len(job.inputfiles) > 0 or (len(job.subjobs) == 0 and job.inputdata and\ isType(job.inputdata, GangaDataset) and job.inputdata.treat_as_inputfiles): (fileNames, tmpDir) = getInputFilesPatterns(job) files = itertools.imap(lambda f: File(f), fileNames) else: # RTHandler is not required to produce masterjobconfig, in that # case just use the inputsandbox files = job.inputsandbox result = create_sandbox(files, master=True) if tmpDir is not None: import shutil # remove temp dir if os.path.exists(tmpDir): shutil.rmtree(tmpDir) return result
def master_prepare(self, app, appmasterconfig): """ Prepare the RTHandler for the master job so that applications to be submitted Args: app (GaudiExec): This application is only expected to handle GaudiExec Applications here appmasterconfig (unknown): Output passed from the application master configuration call """ inputsandbox, outputsandbox = master_sandbox_prepare(app, appmasterconfig) if isinstance(app.jobScriptArchive, LocalFile): app.jobScriptArchive = None generateJobScripts(app, appendJobScripts=True) scriptArchive = os.path.join(app.jobScriptArchive.localDir, app.jobScriptArchive.namePattern) inputsandbox.append(File(name=scriptArchive)) return StandardJobConfig(inputbox=unique(inputsandbox), outputbox=unique(outputsandbox))
def __populate(): if len(transformDictionary) == 0: file_obj = File() transformDictionary[type(file_obj)] = LocalFile