コード例 #1
0
    def get_output(self, job):
        '''Builds lists of output files and output data.'''

        outputdata = [
            f.namePattern for f in job.outputfiles
            if outputFilePostProcessingOnWN(job, getName(f))
        ]
        outsandbox = [
            f.namePattern for f in job.outputfiles
            if not outputFilePostProcessingOnWN(job, getName(f))
        ]

        # if user put any files in both, remove them from the sandbox
        for f in outsandbox:
            if outputdata.count(f) != 0:
                outsandbox.remove(f)
                msg = 'User placed the file %s in both the outputsandbox and '
                msg += 'outputdata. It will be removed from the sandbox.'
                logger.warning(msg, f)

        gaudi_outsandbox, gaudi_outputdata = self.get_output_files()

        # handle (as best we can) any user supplied wildcards
        datalist = []  # files in sandbox that match pattern in data
        for f in outputdata:
            datalist += fnmatch.filter(gaudi_outsandbox, f)

        sandlist = []  # files in data that match sandbox pattern
        for f in outsandbox:
            sandlist += fnmatch.filter(gaudi_outputdata, f)

        datadatalist = []  # files in data that match patterns in data
        for f in outputdata:
            datadatalist += fnmatch.filter(gaudi_outputdata, f)

        # files in sandbox which match patterns in data -> data
        for f in datalist:
            gaudi_outputdata.append(f)
            if f in gaudi_outsandbox:
                gaudi_outsandbox.remove(f)
        # files in data which match patterns in sandbox but not data -> sandbox
        for f in sandlist:
            if datalist.count(f) == 0 and datadatalist.count(f) == 0:
                gaudi_outsandbox.append(f)
                if f in gaudi_outputdata:
                    gaudi_outputdata.remove(f)

        outsandbox += gaudi_outsandbox
        outputdata += gaudi_outputdata

        return unique(outsandbox), unique(outputdata)
コード例 #2
0
ファイル: PythonOptionsParser.py プロジェクト: MannyMoo/ganga
    def get_output(self, job):
        '''Builds lists of output files and output data.'''

        outputdata = [f.namePattern for f in job.outputfiles if outputFilePostProcessingOnWN(job, getName(f)) ]
        outsandbox = [f.namePattern for f in job.outputfiles if not outputFilePostProcessingOnWN(job, getName(f)) ]

        # if user put any files in both, remove them from the sandbox
        for f in outsandbox:
            if outputdata.count(f) != 0:
                outsandbox.remove(f)
                msg = 'User placed the file %s in both the outputsandbox and '
                msg += 'outputdata. It will be removed from the sandbox.'
                logger.warning(msg, f)

        gaudi_outsandbox, gaudi_outputdata = self.get_output_files()

        # handle (as best we can) any user supplied wildcards
        datalist = []  # files in sandbox that match pattern in data
        for f in outputdata:
            datalist += fnmatch.filter(gaudi_outsandbox, f)

        sandlist = []  # files in data that match sandbox pattern
        for f in outsandbox:
            sandlist += fnmatch.filter(gaudi_outputdata, f)

        datadatalist = []  # files in data that match patterns in data
        for f in outputdata:
            datadatalist += fnmatch.filter(gaudi_outputdata, f)

        # files in sandbox which match patterns in data -> data
        for f in datalist:
            gaudi_outputdata.append(f)
            if f in gaudi_outsandbox:
                gaudi_outsandbox.remove(f)
        # files in data which match patterns in sandbox but not data -> sandbox
        for f in sandlist:
            if datalist.count(f) == 0 and datadatalist.count(f) == 0:
                gaudi_outsandbox.append(f)
                if f in gaudi_outputdata:
                    gaudi_outputdata.remove(f)

        outsandbox += gaudi_outsandbox
        outputdata += gaudi_outputdata

        return unique(outsandbox), unique(outputdata)
コード例 #3
0
ファイル: Interactive.py プロジェクト: alexpearce/ganga
    def preparejob(self, jobconfig, master_input_sandbox):
        """Method for preparing job script"""

        job = self.getJobObject()

        inputfiles = jobconfig.getSandboxFiles()
        inbox = job.createPackedInputSandbox(inputfiles)

        inbox.extend(master_input_sandbox)
        inpDir = job.getInputWorkspace(create=True).getPath()
        outDir = job.getOutputWorkspace(create=True).getPath()
        workdir = tempfile.mkdtemp()
        self.workdir = workdir
        exeString = jobconfig.getExeString()
        argList = jobconfig.getArgStrings()
        argString = " ".join(map(lambda x: " %s " % x, argList))

        outputSandboxPatterns = jobconfig.outputbox
        patternsToZip = []
        wnCodeForPostprocessing = ''
        wnCodeToDownloadInputFiles = ''

        if (len(job.outputfiles) > 0):

            from Ganga.GPIDev.Lib.File.OutputFileManager import getOutputSandboxPatternsForInteractive, getWNCodeForOutputPostprocessing
            (outputSandboxPatterns,
                    patternsToZip) = getOutputSandboxPatternsForInteractive(job)

            wnCodeForPostprocessing = 'def printError(message):pass\ndef printInfo(message):pass' + \
                getWNCodeForOutputPostprocessing(job, '')
 
        all_inputfiles = [this_file for this_file in job.inputfiles]
        if job.master: all_inputfiles.extend([this_file for this_file in job.master.inputfiles])

        wnCodeToDownloadInputFiles = ''

        if(len(all_inputfiles) > 0):

            from Ganga.GPIDev.Lib.File.OutputFileManager import outputFilePostProcessingOnWN

            for inputFile in all_inputfiles:

                inputfileClassName = getName(inputFile)

                logger.debug("name: %s" % inputfileClassName)
                logger.debug("result: %s" % str(outputFilePostProcessingOnWN(job, inputfileClassName)))

                if outputFilePostProcessingOnWN(job, inputfileClassName):
                    inputFile.processWildcardMatches()
                    if inputFile.subfiles:
                        getfromFile = False
                        for subfile in inputFile.subfiles:
                            wnCodeToDownloadInputFiles += subfile.getWNScriptDownloadCommand('')
                        else:
                            getfromFile = True
                    else:
                        getFromFile = True

                    if getFromFile:
                        wnCodeToDownloadInputFiles += inputFile.getWNScriptDownloadCommand('')

        wnCodeToDownloadInputData = ''

        if job.inputdata and (len(job.inputdata) > 0):

            from Ganga.GPIDev.Lib.File.OutputFileManager import getWNCodeForDownloadingInputFiles

            wnCodeToDownloadInputData = getWNCodeForDownloadingInputFiles(job, '')

        

        import inspect

        replace_dict = {
        '###CONSTRUCT_TIME###' : (time.strftime("%c")),
        '###WNSANDBOX_SOURCE###' : inspect.getsource(Sandbox.WNSandbox),
        '###GANGA_PYTHONPATH###' : getConfig("System")["GANGA_PYTHONPATH"],
        '###OUTPUTDIR###' : outDir,
        '###WORKDIR###' : workdir,
        '###IN_BOX###' : inbox,
        '###WN_INPUTFILES###' : wnCodeToDownloadInputFiles,
        '###WN_INPUTDATA###' : wnCodeToDownloadInputData,
        '###JOBCONFIG_ENV###' : jobconfig.env if jobconfig.env is not None else dict(),
        '###EXE_STRING###' : exeString,
        '###ARG_STRING###' : argString,
        '###WN_POSTPROCESSING###' : wnCodeForPostprocessing,
        '###PATTERNS_TO_ZIP###' : patternsToZip,
        '###OUTPUT_SANDBOX_PATTERNS###' : outputSandboxPatterns
        }

        script_location = os.path.join(os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))),
                'InteractiveScriptTemplate.py')

        from Ganga.GPIDev.Lib.File import FileUtils
        commandString = FileUtils.loadScript(script_location, '')

        for k, v in replace_dict.iteritems():
            commandString = commandString.replace(str(k), str(v))

        return job.getInputWorkspace().writefile(FileBuffer("__jobscript__", commandString), executable=1)
コード例 #4
0
ファイル: Interactive.py プロジェクト: tarunesh1234/ganga
    def preparejob(self, jobconfig, master_input_sandbox):
        """Method for preparing job script"""

        job = self.getJobObject()

        from Ganga.GPIDev.Lib.File import File
        from Ganga.Core.Sandbox.WNSandbox import PYTHON_DIR
        import Ganga.Utility.files
        import inspect

        fileutils = File(inspect.getsourcefile(Ganga.Utility.files),
                         subdir=PYTHON_DIR)
        inputfiles = jobconfig.getSandboxFiles() + [fileutils]
        inbox = job.createPackedInputSandbox(inputfiles)

        inbox.extend(master_input_sandbox)
        inpDir = job.getInputWorkspace(create=True).getPath()
        outDir = job.getOutputWorkspace(create=True).getPath()
        workdir = tempfile.mkdtemp()
        self.workdir = workdir
        exeString = jobconfig.getExeString()
        argList = jobconfig.getArgStrings()
        argString = " ".join(map(lambda x: " %s " % x, argList))

        outputSandboxPatterns = jobconfig.outputbox
        patternsToZip = []
        wnCodeForPostprocessing = ''
        wnCodeToDownloadInputFiles = ''

        if (len(job.outputfiles) > 0):

            from Ganga.GPIDev.Lib.File.OutputFileManager import getOutputSandboxPatternsForInteractive, getWNCodeForOutputPostprocessing
            (outputSandboxPatterns,
             patternsToZip) = getOutputSandboxPatternsForInteractive(job)

            wnCodeForPostprocessing = 'def printError(message):pass\ndef printInfo(message):pass' + \
                getWNCodeForOutputPostprocessing(job, '')

        all_inputfiles = [this_file for this_file in job.inputfiles]
        if job.master is not None:
            all_inputfiles.extend(
                [this_file for this_file in job.master.inputfiles])

        wnCodeToDownloadInputFiles = ''

        if (len(all_inputfiles) > 0):

            from Ganga.GPIDev.Lib.File.OutputFileManager import outputFilePostProcessingOnWN

            for inputFile in all_inputfiles:

                inputfileClassName = getName(inputFile)

                logger.debug("name: %s" % inputfileClassName)
                logger.debug(
                    "result: %s" %
                    str(outputFilePostProcessingOnWN(job, inputfileClassName)))

                if outputFilePostProcessingOnWN(job, inputfileClassName):
                    inputFile.processWildcardMatches()
                    if inputFile.subfiles:
                        getfromFile = False
                        for subfile in inputFile.subfiles:
                            wnCodeToDownloadInputFiles += subfile.getWNScriptDownloadCommand(
                                '')
                        else:
                            getfromFile = True
                    else:
                        getFromFile = True

                    if getFromFile:
                        wnCodeToDownloadInputFiles += inputFile.getWNScriptDownloadCommand(
                            '')

        wnCodeToDownloadInputData = ''

        if job.inputdata and (len(job.inputdata) > 0):

            from Ganga.GPIDev.Lib.File.OutputFileManager import getWNCodeForDownloadingInputFiles

            wnCodeToDownloadInputData = getWNCodeForDownloadingInputFiles(
                job, '')

        import inspect

        replace_dict = {
            '###CONSTRUCT_TIME###': (time.strftime("%c")),
            '###WNSANDBOX_SOURCE###':
            inspect.getsource(Sandbox.WNSandbox),
            '###GANGA_PYTHONPATH###':
            getConfig("System")["GANGA_PYTHONPATH"],
            '###OUTPUTDIR###':
            outDir,
            '###WORKDIR###':
            workdir,
            '###IN_BOX###':
            inbox,
            '###WN_INPUTFILES###':
            wnCodeToDownloadInputFiles,
            '###WN_INPUTDATA###':
            wnCodeToDownloadInputData,
            '###JOBCONFIG_ENV###':
            jobconfig.env if jobconfig.env is not None else dict(),
            '###EXE_STRING###':
            exeString,
            '###ARG_STRING###':
            argString,
            '###WN_POSTPROCESSING###':
            wnCodeForPostprocessing,
            '###PATTERNS_TO_ZIP###':
            patternsToZip,
            '###OUTPUT_SANDBOX_PATTERNS###':
            outputSandboxPatterns
        }

        script_location = os.path.join(
            os.path.dirname(
                os.path.abspath(inspect.getfile(inspect.currentframe()))),
            'InteractiveScriptTemplate.py.template')

        from Ganga.GPIDev.Lib.File import FileUtils
        commandString = FileUtils.loadScript(script_location, '')

        for k, v in replace_dict.iteritems():
            commandString = commandString.replace(str(k), str(v))

        return job.getInputWorkspace().writefile(FileBuffer(
            "__jobscript__", commandString),
                                                 executable=1)