def master_sandbox_prepare(app, appmasterconfig, sharedir_roots=None): if sharedir_roots is None: sharedir_roots = [''] logger.debug("RTUTils master_sandbox_prepare") # catch errors from not preparing properly if not hasattr(stripProxy(app), 'is_prepared') or app.is_prepared is None: logger.warning('Application is not prepared properly') if hasattr(stripProxy(app), 'is_prepared'): logger.warning("app.is_prepared: %s" % str(app.is_prepared)) import traceback traceback.print_stack() raise GangaException(None, 'Application not prepared properly') # Note EITHER the master inputsandbox OR the job.inputsandbox is added to # the subjob inputsandbox depending if the jobmasterconfig object is present # or not... Therefore combine the job.inputsandbox with appmasterconfig. job = stripProxy(app).getJobObject() # user added items from the interactive GPI from Ganga.Utility.Config import getConfig if not getConfig('Output')['ForbidLegacyInput']: inputsandbox = job.inputsandbox[:] else: if len(job.inputsandbox) > 0: from Ganga.GPIDev.Lib.Job import JobError raise JobError( "InputFiles have been requested but there are objects in the inputSandBox... Aborting Job Prepare!" ) inputsandbox = [] for filepattern in getInputFilesPatterns(job)[0]: inputsandbox.append(File(filepattern)) if len(inputsandbox) > 100: logger.warning( 'InputSandbox exceeds maximum size (100) supported by the Dirac backend' ) raise GangaException(None, 'InputSandbox exceed maximum size') outputsandbox = getOutputSandboxPatterns(job) # job.outputsandbox[:] # inputsandbox files stored in share_dir from prepare method sharedir_handler(app, sharedir_roots, inputsandbox) # Here add any sandbox files/data coming from the appmasterconfig # from master_configure. Catch the case where None is passed (as in tests) if appmasterconfig: inputsandbox += appmasterconfig.getSandboxFiles() outputsandbox += appmasterconfig.getOutputSandboxFiles() return unique(inputsandbox), unique(outputsandbox)
def master_prepare(self, masterjobconfig): """ Prepare the master job (shared sandbox files). This method is/should be called by master_submit() exactly once. The input sandbox is created according to self._packed_input_sandbox flag (a class attribute) """ from Ganga.GPIDev.Lib.File.OutputFileManager import getInputFilesPatterns from Ganga.GPIDev.Lib.File.File import File, ShareDir job = self.getJobObject() create_sandbox = job.createInputSandbox if self._packed_input_sandbox: create_sandbox = job.createPackedInputSandbox if masterjobconfig: if hasattr(job.application, "is_prepared") and isType(job.application.is_prepared, ShareDir): sharedir_pred = lambda f: f.name.find(job.application.is_prepared.name) > -1 sharedir_files = itertools.ifilter(sharedir_pred, masterjobconfig.getSandboxFiles()) nonsharedir_files = itertools.ifilterfalse(sharedir_pred, masterjobconfig.getSandboxFiles()) # ATLAS use bool to bypass the prepare mechanism and some ATLAS # apps have no is_prepared else: sharedir_files = [] nonsharedir_files = masterjobconfig.getSandboxFiles() inputsandbox = create_sandbox(nonsharedir_files, master=True) inputsandbox.extend(itertools.imap(lambda f: f.name, sharedir_files)) return inputsandbox tmpDir = None files = [] if len(job.inputfiles) > 0 or ( len(job.subjobs) == 0 and job.inputdata and isType(job.inputdata, GangaDataset) and job.inputdata.treat_as_inputfiles ): (fileNames, tmpDir) = getInputFilesPatterns(job) files = itertools.imap(lambda f: File(f), fileNames) else: # RTHandler is not required to produce masterjobconfig, in that # case just use the inputsandbox files = job.inputsandbox result = create_sandbox(files, master=True) if tmpDir is not None: import shutil # remove temp dir if os.path.exists(tmpDir): shutil.rmtree(tmpDir) return result
def master_prepare(self, masterjobconfig): """ Prepare the master job (shared sandbox files). This method is/should be called by master_submit() exactly once. The input sandbox is created according to self._packed_input_sandbox flag (a class attribute) """ from Ganga.GPIDev.Lib.File.OutputFileManager import getInputFilesPatterns from Ganga.GPIDev.Lib.File.File import File, ShareDir job = self.getJobObject() create_sandbox = job.createInputSandbox if self._packed_input_sandbox: create_sandbox = job.createPackedInputSandbox if masterjobconfig: if hasattr(job.application, 'is_prepared') and isType( job.application.is_prepared, ShareDir): sharedir_pred = lambda f: f.name.find(job.application. is_prepared.name) > -1 sharedir_files = itertools.ifilter( sharedir_pred, masterjobconfig.getSandboxFiles()) nonsharedir_files = itertools.ifilterfalse( sharedir_pred, masterjobconfig.getSandboxFiles()) # ATLAS use bool to bypass the prepare mechanism and some ATLAS # apps have no is_prepared else: sharedir_files = [] nonsharedir_files = masterjobconfig.getSandboxFiles() inputsandbox = create_sandbox(nonsharedir_files, master=True) inputsandbox.extend( itertools.imap(lambda f: f.name, sharedir_files)) return inputsandbox tmpDir = None files = [] if len(job.inputfiles) > 0 or (len(job.subjobs) == 0 and job.inputdata and\ isType(job.inputdata, GangaDataset) and job.inputdata.treat_as_inputfiles): (fileNames, tmpDir) = getInputFilesPatterns(job) files = itertools.imap(lambda f: File(f), fileNames) else: # RTHandler is not required to produce masterjobconfig, in that # case just use the inputsandbox files = job.inputsandbox result = create_sandbox(files, master=True) if tmpDir is not None: import shutil # remove temp dir if os.path.exists(tmpDir): shutil.rmtree(tmpDir) return result
def master_sandbox_prepare(app, appmasterconfig, sharedir_roots=None): if sharedir_roots is None: sharedir_roots = [''] logger.debug("RTUTils master_sandbox_prepare") # catch errors from not preparing properly if not hasattr(stripProxy(app), 'is_prepared') or app.is_prepared is None: logger.warning('Application is not prepared properly') if hasattr(stripProxy(app), 'is_prepared'): logger.warning("app.is_prepared: %s" % str(app.is_prepared)) import traceback traceback.print_stack() raise GangaException(None, 'Application not prepared properly') # Note EITHER the master inputsandbox OR the job.inputsandbox is added to # the subjob inputsandbox depending if the jobmasterconfig object is present # or not... Therefore combine the job.inputsandbox with appmasterconfig. job = stripProxy(app).getJobObject() # user added items from the interactive GPI from Ganga.Utility.Config import getConfig if not getConfig('Output')['ForbidLegacyInput']: inputsandbox = job.inputsandbox[:] else: if len(job.inputsandbox) > 0: from Ganga.GPIDev.Lib.Job import JobError raise JobError("InputFiles have been requested but there are objects in the inputSandBox... Aborting Job Prepare!") inputsandbox = [] for filepattern in getInputFilesPatterns(job)[0]: inputsandbox.append(File(filepattern)) if len(inputsandbox) > 100: logger.warning('InputSandbox exceeds maximum size (100) supported by the Dirac backend') raise GangaException(None, 'InputSandbox exceed maximum size') outputsandbox = getOutputSandboxPatterns(job) # job.outputsandbox[:] # inputsandbox files stored in share_dir from prepare method sharedir_handler(app, sharedir_roots, inputsandbox) # Here add any sandbox files/data coming from the appmasterconfig # from master_configure. Catch the case where None is passed (as in tests) if appmasterconfig: inputsandbox += appmasterconfig.getSandboxFiles() outputsandbox += appmasterconfig.getOutputSandboxFiles() return unique(inputsandbox), unique(outputsandbox)