Exemplo n.º 1
0
    def _check_inputs(self):
        """Checks the validity of some of user's entries for Gaudi schema"""

        # Warn user that no optsfiles given
        if len(self.optsfile) == 0:
            logger.warning("The 'optsfile' is not set. I hope this is OK!")
        else:
            # Check for non-exting optsfiles defined
            nonexistentOptFiles = []
            for f in self.optsfile:
                from Ganga.GPIDev.Lib.File.File import File
                if type(f) is str:
                    myFile = File(f)
                else:
                    myFile = f
                myFile.name = fullpath(myFile.name)
                if not os.path.isfile(myFile.name):
                    nonexistentOptFiles.append(myFile)

            if len(nonexistentOptFiles):
                tmpmsg = "The 'optsfile' attribute contains non-existent file/s: ["
                for _f in nonexistentOptFiles:
                    tmpmsg += "'%s', " % _f.name
                msg = tmpmsg[:-2] + ']'
                raise ApplicationConfigurationError(msg)
Exemplo n.º 2
0
    def _check_inputs(self):
        """Checks the validity of some of user's entries for Gaudi schema"""

        # Warn user that no optsfiles given
        if len(self.optsfile) == 0:
            logger.warning("The 'optsfile' is not set. I hope this is OK!")
        else:
            # Check for non-exting optsfiles defined
            nonexistentOptFiles = []
            for f in self.optsfile:
                from Ganga.GPIDev.Lib.File.File import File
                if type(f) is str:
                    myFile = File(f)
                else:
                    myFile = f
                myFile.name = fullpath(myFile.name)
                if not os.path.isfile(myFile.name):
                    nonexistentOptFiles.append(myFile)

            if len(nonexistentOptFiles):
                tmpmsg = "The 'optsfile' attribute contains non-existent file/s: ["
                for _f in nonexistentOptFiles:
                    tmpmsg += "'%s', " % _f.name
                msg = tmpmsg[:-2] + ']'
                raise ApplicationConfigurationError(msg)
Exemplo n.º 3
0
    def prepare(self, app, appconfig, appmasterconfig, jobmasterconfig):
        from Ganga.Lib.gLite import gLiteJobConfig

        prepared_exe = app.exe
        if app.is_prepared is not None:
            shared_path = os.path.join(expandfilename(getConfig('Configuration')['gangadir']),
                                       'shared', getConfig('Configuration')['user'])
            if isinstance(app.exe, str):
                # we have a file. is it an absolute path?
                if os.path.abspath(app.exe) == app.exe:
                    logger.info("Submitting a prepared application; taking any input files from %s" % (
                        app.is_prepared.name))
                    prepared_exe = File(os.path.join(os.path.join(
                        shared_path, app.is_prepared.name), os.path.basename(File(app.exe).name)))
                # else assume it's a system binary, so we don't need to
                # transport anything to the sharedir
                else:
                    prepared_exe = app.exe
            elif isinstance(app.exe, File):
                logger.info("Submitting a prepared application; taking any input files from %s" % (
                    app.is_prepared.name))
                prepared_exe = File(os.path.join(os.path.join(
                    shared_path, app.is_prepared.name), os.path.basename(File(app.exe).name)))

        return gLiteJobConfig(prepared_exe, app._getParent().inputsandbox, convertIntToStringArgs(app.args), app._getParent().outputsandbox, app.env)
Exemplo n.º 4
0
 def setUp(self):
     j = Job(application=DaVinci())
     j.prepare()
     from Ganga.Utility.Config import getConfig
     if getConfig('Output')['ForbidLegacyInput']:
         j.inputfiles = [LocalFile(name='dummy.in')]
     else:
         j.inputsandbox = [File(name='dummy.in')]
     self.app = j.application._impl
     #self.extra = GaudiExtras()
     # self.extra.master_input_buffers['master.buffer'] = '###MASTERBUFFER###'
     #self.extra.master_input_files = [File(name='master.in')]
     # self.extra.input_buffers['subjob.buffer'] = '###SUBJOBBUFFER###'
     self.input_files = [
         File(name='subjob.in'),
         File(
             FileBuffer('subjob.buffer',
                        '###SUBJOBBUFFER###').create().name)
     ]
     self.appmasterconfig = StandardJobConfig(inputbox=[
         File(name='master.in'),
         File(
             FileBuffer('master.buffer',
                        '###MASTERBUFFER###').create().name)
     ])
     j.outputfiles = ['dummy1.out', 'dummy2.out', 'dummy3.out']
     self.rth = LHCbGaudiRunTimeHandler()
Exemplo n.º 5
0
    def master_prepare(self, app, appmasterconfig):
        """
        Prepare the RTHandler for the master job so that applications to be submitted
        Args:
            app (GaudiExec): This application is only expected to handle GaudiExec Applications here
            appmasterconfig (unknown): Output passed from the application master configuration call
        """

        inputsandbox, outputsandbox = master_sandbox_prepare(
            app, appmasterconfig)

        if isinstance(app.jobScriptArchive, LocalFile):
            app.jobScriptArchive = None

        generateJobScripts(app, appendJobScripts=True)

        scriptArchive = os.path.join(app.jobScriptArchive.localDir,
                                     app.jobScriptArchive.namePattern)

        inputsandbox.append(File(name=scriptArchive))

        if app.getMetadata:
            logger.info("Adding options to make the summary.xml")
            inputsandbox.append(
                FileBuffer(
                    'summary.py',
                    "\nfrom Gaudi.Configuration import *\nfrom Configurables import LHCbApp\nLHCbApp().XMLSummary='summary.xml'"
                ))

        return StandardJobConfig(inputbox=unique(inputsandbox),
                                 outputbox=unique(outputsandbox))
Exemplo n.º 6
0
    def prepare(self, force=False):

        from Ganga.GPIDev.Lib.GangaList.GangaList import GangaList
        from Ganga.GPIDev.Lib.File.File import File
        if isType(self.optsfile, (list, tuple, GangaList)):
            for this_file in self.optsfile:
                if type(this_file) is str:
                    this_file = File(this_file)
                else:
                    continue

        elif type(self.optsfile) is str:
            self.optsfile = [File(self.optsfile)]

        try:
            super(Gaudi, self).prepare(force)
        except Exception, err:
            logger.debug("Super Prepare Error:\n%s" % str(err))
            raise err
Exemplo n.º 7
0
    def master_prepare(self, masterjobconfig):
        """ Prepare the master job (shared sandbox files). This method is/should be called by master_submit() exactly once.
        The input sandbox is created according to self._packed_input_sandbox flag (a class attribute)
        """
        from Ganga.GPIDev.Lib.File.OutputFileManager import getInputFilesPatterns
        from Ganga.GPIDev.Lib.File.File import File, ShareDir

        job = self.getJobObject()

        create_sandbox = job.createInputSandbox
        if self._packed_input_sandbox:
            create_sandbox = job.createPackedInputSandbox

        if masterjobconfig:
            if hasattr(job.application, 'is_prepared') and isType(
                    job.application.is_prepared, ShareDir):
                sharedir_pred = lambda f: f.name.find(job.application.
                                                      is_prepared.name) > -1
                sharedir_files = itertools.ifilter(
                    sharedir_pred, masterjobconfig.getSandboxFiles())
                nonsharedir_files = itertools.ifilterfalse(
                    sharedir_pred, masterjobconfig.getSandboxFiles())
            # ATLAS use bool to bypass the prepare mechanism and some ATLAS
            # apps have no is_prepared
            else:
                sharedir_files = []
                nonsharedir_files = masterjobconfig.getSandboxFiles()
            inputsandbox = create_sandbox(nonsharedir_files, master=True)
            inputsandbox.extend(
                itertools.imap(lambda f: f.name, sharedir_files))
            return inputsandbox

        tmpDir = None
        files = []
        if len(job.inputfiles) > 0 or (len(job.subjobs) == 0 and job.inputdata and\
                isType(job.inputdata, GangaDataset) and job.inputdata.treat_as_inputfiles):
            (fileNames, tmpDir) = getInputFilesPatterns(job)
            files = itertools.imap(lambda f: File(f), fileNames)
        else:
            # RTHandler is not required to produce masterjobconfig, in that
            # case just use the inputsandbox
            files = job.inputsandbox

        result = create_sandbox(files, master=True)
        if tmpDir is not None:
            import shutil
            # remove temp dir
            if os.path.exists(tmpDir):
                shutil.rmtree(tmpDir)
        return result
Exemplo n.º 8
0
    def master_prepare(self, app, appmasterconfig):
        """
        Prepare the RTHandler for the master job so that applications to be submitted
        Args:
            app (GaudiExec): This application is only expected to handle GaudiExec Applications here
            appmasterconfig (unknown): Output passed from the application master configuration call
        """

        inputsandbox, outputsandbox = master_sandbox_prepare(app, appmasterconfig)

        if isinstance(app.jobScriptArchive, LocalFile):
            app.jobScriptArchive = None

        generateJobScripts(app, appendJobScripts=True)

        scriptArchive = os.path.join(app.jobScriptArchive.localDir, app.jobScriptArchive.namePattern)

        inputsandbox.append(File(name=scriptArchive))
        return StandardJobConfig(inputbox=unique(inputsandbox), outputbox=unique(outputsandbox))
Exemplo n.º 9
0
def __populate():
    if len(transformDictionary) == 0:
        file_obj = File()
        transformDictionary[type(file_obj)] = LocalFile