Example #1
0
    def preparejob(self, jobconfig, master_input_sandbox):

        job = self.getJobObject()
        # print str(job.backend_output_postprocess)
        mon = job.getMonitoringService()
        import Ganga.Core.Sandbox as Sandbox
        subjob_input_sandbox = job.createPackedInputSandbox(jobconfig.getSandboxFiles() + Sandbox.getGangaModulesAsSandboxFiles(Sandbox.getDefaultModules()))

        appscriptpath = [jobconfig.getExeString()] + jobconfig.getArgStrings()
        if self.nice:
            appscriptpath = ['nice', '-n %d' % self.nice] + appscriptpath
        if self.nice < 0:
            logger.warning('increasing process priority is often not allowed, your job may fail due to this')

        sharedoutputpath = job.getOutputWorkspace().getPath()
        ## FIXME DON'T just use the blind list here, request the list of files to be in the output from a method.
        outputpatterns = jobconfig.outputbox
        environment = dict() if jobconfig.env is None else jobconfig.env

        import tempfile
        workdir = tempfile.mkdtemp(dir=config['location'])

        import inspect
        script_location = os.path.join(os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))),
                                                        'LocalHostExec.py')

        from Ganga.GPIDev.Lib.File import FileUtils
        script = FileUtils.loadScript(script_location, '')

        script = script.replace('###INLINEMODULES###', inspect.getsource(Sandbox.WNSandbox))

        from Ganga.GPIDev.Lib.File.OutputFileManager import getWNCodeForOutputSandbox, getWNCodeForOutputPostprocessing, getWNCodeForDownloadingInputFiles, getWNCodeForInputdataListCreation
        from Ganga.Utility.Config import getConfig
        jobidRepr = repr(job.getFQID('.'))


        script = script.replace('###OUTPUTSANDBOXPOSTPROCESSING###', getWNCodeForOutputSandbox(job, ['stdout', 'stderr', '__syslog__'], jobidRepr))
        script = script.replace('###OUTPUTUPLOADSPOSTPROCESSING###', getWNCodeForOutputPostprocessing(job, ''))
        script = script.replace('###DOWNLOADINPUTFILES###', getWNCodeForDownloadingInputFiles(job, ''))
        script = script.replace('###CREATEINPUTDATALIST###', getWNCodeForInputdataListCreation(job, ''))

        script = script.replace('###APPLICATION_NAME###', repr(job.application._name))
        script = script.replace('###INPUT_SANDBOX###', repr(subjob_input_sandbox + master_input_sandbox))
        script = script.replace('###SHAREDOUTPUTPATH###', repr(sharedoutputpath))
        script = script.replace('###APPSCRIPTPATH###', repr(appscriptpath))
        script = script.replace('###OUTPUTPATTERNS###', str(outputpatterns))
        script = script.replace('###JOBID###', jobidRepr)
        script = script.replace('###ENVIRONMENT###', repr(environment))
        script = script.replace('###WORKDIR###', repr(workdir))
        script = script.replace('###INPUT_DIR###', repr(job.getStringInputDir()))

        self.workdir = workdir

        script = script.replace('###GANGADIR###', repr(getConfig('System')['GANGA_PYTHONPATH']))

        wrkspace = job.getInputWorkspace()
        scriptPath = wrkspace.writefile(FileBuffer('__jobscript__', script), executable=1)

        return scriptPath
Example #2
0
    def preparejob(self, jobconfig, master_input_sandbox):

        job = self.getJobObject()
        # print str(job.backend_output_postprocess)
        mon = job.getMonitoringService()
        import Ganga.Core.Sandbox as Sandbox
        subjob_input_sandbox = job.createPackedInputSandbox(jobconfig.getSandboxFiles() + Sandbox.getGangaModulesAsSandboxFiles(Sandbox.getDefaultModules()))

        appscriptpath = [jobconfig.getExeString()] + jobconfig.getArgStrings()
        if self.nice:
            appscriptpath = ['nice', '-n %d' % self.nice] + appscriptpath
        if self.nice < 0:
            logger.warning('increasing process priority is often not allowed, your job may fail due to this')

        sharedoutputpath = job.getOutputWorkspace().getPath()
        ## FIXME DON'T just use the blind list here, request the list of files to be in the output from a method.
        outputpatterns = jobconfig.outputbox
        environment = dict() if jobconfig.env is None else jobconfig.env

        import tempfile
        workdir = tempfile.mkdtemp(dir=config['location'])

        import inspect
        script_location = os.path.join(os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))),
                                                        'LocalHostExec.py')

        from Ganga.GPIDev.Lib.File import FileUtils
        script = FileUtils.loadScript(script_location, '')

        script = script.replace('###INLINEMODULES###', inspect.getsource(Sandbox.WNSandbox))

        from Ganga.GPIDev.Lib.File.OutputFileManager import getWNCodeForOutputSandbox, getWNCodeForOutputPostprocessing, getWNCodeForDownloadingInputFiles, getWNCodeForInputdataListCreation
        from Ganga.Utility.Config import getConfig
        jobidRepr = repr(job.getFQID('.'))


        script = script.replace('###OUTPUTSANDBOXPOSTPROCESSING###', getWNCodeForOutputSandbox(job, ['stdout', 'stderr', '__syslog__'], jobidRepr))
        script = script.replace('###OUTPUTUPLOADSPOSTPROCESSING###', getWNCodeForOutputPostprocessing(job, ''))
        script = script.replace('###DOWNLOADINPUTFILES###', getWNCodeForDownloadingInputFiles(job, ''))
        script = script.replace('###CREATEINPUTDATALIST###', getWNCodeForInputdataListCreation(job, ''))

        script = script.replace('###APPLICATION_NAME###', repr(getName(job.application)))
        script = script.replace('###INPUT_SANDBOX###', repr(subjob_input_sandbox + master_input_sandbox))
        script = script.replace('###SHAREDOUTPUTPATH###', repr(sharedoutputpath))
        script = script.replace('###APPSCRIPTPATH###', repr(appscriptpath))
        script = script.replace('###OUTPUTPATTERNS###', str(outputpatterns))
        script = script.replace('###JOBID###', jobidRepr)
        script = script.replace('###ENVIRONMENT###', repr(environment))
        script = script.replace('###WORKDIR###', repr(workdir))
        script = script.replace('###INPUT_DIR###', repr(job.getStringInputDir()))

        self.workdir = workdir

        script = script.replace('###GANGADIR###', repr(getConfig('System')['GANGA_PYTHONPATH']))

        wrkspace = job.getInputWorkspace()
        scriptPath = wrkspace.writefile(FileBuffer('__jobscript__', script), executable=1)

        return scriptPath
Example #3
0
    def preparejob(self, jobconfig, master_job_sandbox):
        '''Prepare the JDL'''

        script = self.__jobWrapperTemplate__()

        job = self.getJobObject()
        inpw = job.getInputWorkspace()

        wrapperlog = '__jobscript__.log'

        import Ganga.Core.Sandbox as Sandbox

        # FIXME: check what happens if 'stdout','stderr' are specified here
        script = script.replace(
            '###OUTPUTSANDBOX###', repr(jobconfig.outputbox))

        script = script.replace(
            '###APPLICATION_NAME###', getName(job.application))
        script = script.replace(
            '###APPLICATIONEXEC###', repr(jobconfig.getExeString()))
        script = script.replace(
            '###APPLICATIONARGS###', repr(jobconfig.getArguments()))

        from Ganga.GPIDev.Lib.File.OutputFileManager import getWNCodeForOutputPostprocessing, getWNCodeForDownloadingInputFiles

        script = script.replace(
            '###OUTPUTUPLOADSPOSTPROCESSING###', getWNCodeForOutputPostprocessing(job, '    '))

        script = script.replace(
            '###DOWNLOADINPUTFILES###', getWNCodeForDownloadingInputFiles(job, '    '))

        if jobconfig.env:
            script = script.replace(
                '###APPLICATIONENVS###', repr(jobconfig.env))
        else:
            script = script.replace('###APPLICATIONENVS###', repr({}))

        script = script.replace('###WRAPPERLOG###', repr(wrapperlog))
        import inspect
        script = script.replace(
            '###INLINEMODULES###', inspect.getsource(Sandbox.WNSandbox))

        mon = job.getMonitoringService()

        self.monInfo = None

        # set the monitoring file by default to the stdout
        if isinstance(self.monInfo, dict):
            self.monInfo['remotefile'] = 'stdout'

        # try to print out the monitoring service information in debug mode
        try:
            logger.debug('job info of monitoring service: %s' %
                         str(self.monInfo))
        except:
            pass

#       prepare input/output sandboxes
        packed_files = jobconfig.getSandboxFiles() + Sandbox.getGangaModulesAsSandboxFiles(Sandbox.getDefaultModules())
        sandbox_files = job.createPackedInputSandbox(packed_files)

        # sandbox of child jobs should include master's sandbox
        sandbox_files.extend(master_job_sandbox)

        # check the input file size and pre-upload larger inputs to the iocache
        lfc_host = ''

        input_sandbox_uris = []
        input_sandbox_names = []

        ick = True

        max_prestaged_fsize = 0
        for f in sandbox_files:

            idx = self.__check_and_prestage_inputfile__(f)

            if not idx:
                logger.error('input sandbox preparation failed: %s' % f)
                ick = False
                break
            else:

                if idx['lfc_host']:
                    lfc_host = idx['lfc_host']

                if idx['remote']:
                    abspath = os.path.abspath(f)
                    fsize = os.path.getsize(abspath)

                    if fsize > max_prestaged_fsize:
                        max_prestaged_fsize = fsize

                    input_sandbox_uris.append(
                        idx['remote'][os.path.basename(f)])

                    input_sandbox_names.append(
                        os.path.basename(urlparse(f)[2]))

                if idx['local']:
                    input_sandbox_uris += idx['local']
                    input_sandbox_names.append(os.path.basename(f))

        if not ick:
            logger.error('stop job submission')
            return None

        # determin the lcg-cp timeout according to the max_prestaged_fsize
        # - using the assumption of 1 MB/sec.
        max_prestaged_fsize = 0
        lfc_host = ''
        transfer_timeout = config['SandboxTransferTimeout']
        predict_timeout = int(math.ceil(max_prestaged_fsize / 1000000.0))

        if predict_timeout > transfer_timeout:
            transfer_timeout = predict_timeout

        if transfer_timeout < 60:
            transfer_timeout = 60

        script = script.replace(
            '###TRANSFERTIMEOUT###', '%d' % transfer_timeout)

        # update the job wrapper with the inputsandbox list
        script = script.replace(
            '###INPUTSANDBOX###', repr({'remote': {}, 'local': input_sandbox_names}))

        # write out the job wrapper and put job wrapper into job's inputsandbox
        scriptPath = inpw.writefile(
            FileBuffer('__jobscript_%s__' % job.getFQID('.'), script), executable=1)
        input_sandbox = input_sandbox_uris + [scriptPath]

        for isb in input_sandbox:
            logger.debug('ISB URI: %s' % isb)

        # compose output sandbox to include by default the following files:
        # - gzipped stdout (transferred only when the JobLogHandler is WMS)
        # - gzipped stderr (transferred only when the JobLogHandler is WMS)
        # - __jobscript__.log (job wrapper's log)
        output_sandbox = [wrapperlog]

        from Ganga.GPIDev.Lib.File.OutputFileManager import getOutputSandboxPatterns
        for outputSandboxPattern in getOutputSandboxPatterns(job):
            output_sandbox.append(outputSandboxPattern)

        if config['JobLogHandler'] in ['WMS']:
            output_sandbox += ['stdout.gz', 'stderr.gz']

        if len(jobconfig.outputbox):
            output_sandbox += [Sandbox.OUTPUT_TARBALL_NAME]

        # compose LCG JDL
        jdl = {
            'VirtualOrganisation': config['VirtualOrganisation'],
            'Executable': os.path.basename(scriptPath),
            'Environment': {'GANGA_LCG_VO': config['VirtualOrganisation'], 'GANGA_LOG_HANDLER': config['JobLogHandler'], 'LFC_HOST': lfc_host},
            'StdOutput': 'stdout',
            'StdError': 'stderr',
            'InputSandbox': input_sandbox,
            'OutputSandbox': output_sandbox,
            'OutputSandboxBaseDestURI': 'gsiftp://localhost'
        }

        jdl['Environment'].update({'GANGA_LCG_CE': self.CE})
        jdl['Requirements'] = self.requirements.merge(
            jobconfig.requirements).convert()

        if self.jobtype.upper() in ['NORMAL', 'MPICH']:
            jdl['JobType'] = self.jobtype.upper()
            if self.jobtype.upper() == 'MPICH':
                #jdl['Requirements'].append('(other.GlueCEInfoTotalCPUs >= NodeNumber)')
                jdl['Requirements'].append(
                    'Member("MPICH",other.GlueHostApplicationSoftwareRunTimeEnvironment)')
                jdl['NodeNumber'] = self.requirements.nodenumber
        else:
            logger.warning('JobType "%s" not supported' % self.jobtype)
            return

#       additional settings from the job
#        if jobconfig.env:
#            jdl['Environment'].update(jobconfig.env)

        jdlText = Grid.expandjdl(jdl)
        logger.debug('subjob JDL: %s' % jdlText)
        return inpw.writefile(FileBuffer('__jdlfile__', jdlText))
Example #4
0
    def preparejob(self, jobconfig, master_input_sandbox):
        """Method for preparing job script"""

        job = self.getJobObject()

        inputfiles = jobconfig.getSandboxFiles()
        inbox = job.createPackedInputSandbox(inputfiles)

        inbox.extend(master_input_sandbox)
        inpDir = job.getInputWorkspace(create=True).getPath()
        outDir = job.getOutputWorkspace(create=True).getPath()
        workdir = tempfile.mkdtemp()
        self.workdir = workdir
        exeString = jobconfig.getExeString()
        argList = jobconfig.getArgStrings()
        argString = " ".join(map(lambda x: " %s " % x, argList))

        outputSandboxPatterns = jobconfig.outputbox
        patternsToZip = []
        wnCodeForPostprocessing = ''
        wnCodeToDownloadInputFiles = ''

        if (len(job.outputfiles) > 0):

            from Ganga.GPIDev.Lib.File.OutputFileManager import getOutputSandboxPatternsForInteractive, getWNCodeForOutputPostprocessing
            (outputSandboxPatterns,
                    patternsToZip) = getOutputSandboxPatternsForInteractive(job)

            wnCodeForPostprocessing = 'def printError(message):pass\ndef printInfo(message):pass' + \
                getWNCodeForOutputPostprocessing(job, '')
 
        all_inputfiles = [this_file for this_file in job.inputfiles]
        if job.master: all_inputfiles.extend([this_file for this_file in job.master.inputfiles])

        wnCodeToDownloadInputFiles = ''

        if(len(all_inputfiles) > 0):

            from Ganga.GPIDev.Lib.File.OutputFileManager import outputFilePostProcessingOnWN

            for inputFile in all_inputfiles:

                inputfileClassName = getName(inputFile)

                logger.debug("name: %s" % inputfileClassName)
                logger.debug("result: %s" % str(outputFilePostProcessingOnWN(job, inputfileClassName)))

                if outputFilePostProcessingOnWN(job, inputfileClassName):
                    inputFile.processWildcardMatches()
                    if inputFile.subfiles:
                        getfromFile = False
                        for subfile in inputFile.subfiles:
                            wnCodeToDownloadInputFiles += subfile.getWNScriptDownloadCommand('')
                        else:
                            getfromFile = True
                    else:
                        getFromFile = True

                    if getFromFile:
                        wnCodeToDownloadInputFiles += inputFile.getWNScriptDownloadCommand('')

        wnCodeToDownloadInputData = ''

        if job.inputdata and (len(job.inputdata) > 0):

            from Ganga.GPIDev.Lib.File.OutputFileManager import getWNCodeForDownloadingInputFiles

            wnCodeToDownloadInputData = getWNCodeForDownloadingInputFiles(job, '')

        

        import inspect

        replace_dict = {
        '###CONSTRUCT_TIME###' : (time.strftime("%c")),
        '###WNSANDBOX_SOURCE###' : inspect.getsource(Sandbox.WNSandbox),
        '###GANGA_PYTHONPATH###' : getConfig("System")["GANGA_PYTHONPATH"],
        '###OUTPUTDIR###' : outDir,
        '###WORKDIR###' : workdir,
        '###IN_BOX###' : inbox,
        '###WN_INPUTFILES###' : wnCodeToDownloadInputFiles,
        '###WN_INPUTDATA###' : wnCodeToDownloadInputData,
        '###JOBCONFIG_ENV###' : jobconfig.env if jobconfig.env is not None else dict(),
        '###EXE_STRING###' : exeString,
        '###ARG_STRING###' : argString,
        '###WN_POSTPROCESSING###' : wnCodeForPostprocessing,
        '###PATTERNS_TO_ZIP###' : patternsToZip,
        '###OUTPUT_SANDBOX_PATTERNS###' : outputSandboxPatterns
        }

        script_location = os.path.join(os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))),
                'InteractiveScriptTemplate.py')

        from Ganga.GPIDev.Lib.File import FileUtils
        commandString = FileUtils.loadScript(script_location, '')

        for k, v in replace_dict.iteritems():
            commandString = commandString.replace(str(k), str(v))

        return job.getInputWorkspace().writefile(FileBuffer("__jobscript__", commandString), executable=1)
Example #5
0
    def preparejob(self, jobconfig, master_job_sandbox):
        '''Prepare the JDL'''

        script = self.__jobWrapperTemplate__()

        job = self.getJobObject()
        inpw = job.getInputWorkspace()

        wrapperlog = '__jobscript__.log'

        import Ganga.Core.Sandbox as Sandbox

        # FIXME: check what happens if 'stdout','stderr' are specified here
        script = script.replace('###OUTPUTSANDBOX###',
                                repr(jobconfig.outputbox))

        script = script.replace('###APPLICATION_NAME###',
                                getName(job.application))
        script = script.replace('###APPLICATIONEXEC###',
                                repr(jobconfig.getExeString()))
        script = script.replace('###APPLICATIONARGS###',
                                repr(jobconfig.getArguments()))

        from Ganga.GPIDev.Lib.File.OutputFileManager import getWNCodeForOutputPostprocessing, getWNCodeForDownloadingInputFiles

        script = script.replace('###OUTPUTUPLOADSPOSTPROCESSING###',
                                getWNCodeForOutputPostprocessing(job, '    '))

        script = script.replace('###DOWNLOADINPUTFILES###',
                                getWNCodeForDownloadingInputFiles(job, '    '))

        if jobconfig.env:
            script = script.replace('###APPLICATIONENVS###',
                                    repr(jobconfig.env))
        else:
            script = script.replace('###APPLICATIONENVS###', repr({}))

        script = script.replace('###WRAPPERLOG###', repr(wrapperlog))
        import inspect
        script = script.replace('###INLINEMODULES###',
                                inspect.getsource(Sandbox.WNSandbox))

        mon = job.getMonitoringService()

        self.monInfo = None

        # set the monitoring file by default to the stdout
        if isinstance(self.monInfo, dict):
            self.monInfo['remotefile'] = 'stdout'

        # try to print out the monitoring service information in debug mode
        try:
            logger.debug('job info of monitoring service: %s' %
                         str(self.monInfo))
        except:
            pass

#       prepare input/output sandboxes
        import Ganga.Utility.files
        from Ganga.GPIDev.Lib.File import File
        from Ganga.Core.Sandbox.WNSandbox import PYTHON_DIR
        import inspect

        fileutils = File(inspect.getsourcefile(Ganga.Utility.files),
                         subdir=PYTHON_DIR)
        packed_files = jobconfig.getSandboxFiles() + [fileutils]
        sandbox_files = job.createPackedInputSandbox(packed_files)

        # sandbox of child jobs should include master's sandbox
        sandbox_files.extend(master_job_sandbox)

        # check the input file size and pre-upload larger inputs to the iocache
        lfc_host = ''

        input_sandbox_uris = []
        input_sandbox_names = []

        ick = True

        max_prestaged_fsize = 0
        for f in sandbox_files:

            idx = self.__check_and_prestage_inputfile__(f)

            if not idx:
                logger.error('input sandbox preparation failed: %s' % f)
                ick = False
                break
            else:

                if idx['lfc_host']:
                    lfc_host = idx['lfc_host']

                if idx['remote']:
                    abspath = os.path.abspath(f)
                    fsize = os.path.getsize(abspath)

                    if fsize > max_prestaged_fsize:
                        max_prestaged_fsize = fsize

                    input_sandbox_uris.append(
                        idx['remote'][os.path.basename(f)])

                    input_sandbox_names.append(os.path.basename(
                        urlparse(f)[2]))

                if idx['local']:
                    input_sandbox_uris += idx['local']
                    input_sandbox_names.append(os.path.basename(f))

        if not ick:
            logger.error('stop job submission')
            return None

        # determin the lcg-cp timeout according to the max_prestaged_fsize
        # - using the assumption of 1 MB/sec.
        max_prestaged_fsize = 0
        lfc_host = ''
        transfer_timeout = config['SandboxTransferTimeout']
        predict_timeout = int(math.ceil(max_prestaged_fsize / 1000000.0))

        if predict_timeout > transfer_timeout:
            transfer_timeout = predict_timeout

        if transfer_timeout < 60:
            transfer_timeout = 60

        script = script.replace('###TRANSFERTIMEOUT###',
                                '%d' % transfer_timeout)

        # update the job wrapper with the inputsandbox list
        script = script.replace(
            '###INPUTSANDBOX###',
            repr({
                'remote': {},
                'local': input_sandbox_names
            }))

        # write out the job wrapper and put job wrapper into job's inputsandbox
        scriptPath = inpw.writefile(FileBuffer(
            '__jobscript_%s__' % job.getFQID('.'), script),
                                    executable=1)
        input_sandbox = input_sandbox_uris + [scriptPath]

        for isb in input_sandbox:
            logger.debug('ISB URI: %s' % isb)

        # compose output sandbox to include by default the following files:
        # - gzipped stdout (transferred only when the JobLogHandler is WMS)
        # - gzipped stderr (transferred only when the JobLogHandler is WMS)
        # - __jobscript__.log (job wrapper's log)
        output_sandbox = [wrapperlog]

        from Ganga.GPIDev.Lib.File.OutputFileManager import getOutputSandboxPatterns
        for outputSandboxPattern in getOutputSandboxPatterns(job):
            output_sandbox.append(outputSandboxPattern)

        if config['JobLogHandler'] in ['WMS']:
            output_sandbox += ['stdout.gz', 'stderr.gz']

        if len(jobconfig.outputbox):
            output_sandbox += [Sandbox.OUTPUT_TARBALL_NAME]

        # compose ARC XRSL
        xrsl = {
            #'VirtualOrganisation' : config['VirtualOrganisation'],
            'executable': os.path.basename(scriptPath),
            'environment': {
                'GANGA_LCG_VO': config['VirtualOrganisation'],
                'GANGA_LOG_HANDLER': config['JobLogHandler'],
                'LFC_HOST': lfc_host
            },
            #'stdout'                : 'stdout',
            #'stderr'                : 'stderr',
            'inputFiles': input_sandbox,
            'outputFiles': output_sandbox,
            #'OutputSandboxBaseDestURI': 'gsiftp://localhost'
        }

        xrsl['environment'].update({'GANGA_LCG_CE': self.CE})
        #xrsl['Requirements'] = self.requirements.merge(jobconfig.requirements).convert()

        # if self.jobtype.upper() in ['NORMAL','MPICH']:
        #xrsl['JobType'] = self.jobtype.upper()
        # if self.jobtype.upper() == 'MPICH':
        #xrsl['Requirements'].append('(other.GlueCEInfoTotalCPUs >= NodeNumber)')
        # xrsl['Requirements'].append('Member("MPICH",other.GlueHostApplicationSoftwareRunTimeEnvironment)')
        #xrsl['NodeNumber'] = self.requirements.nodenumber
        # else:
        #    logger.warning('JobType "%s" not supported' % self.jobtype)
        #    return

        #       additional settings from the job
        if jobconfig.env:
            xrsl['environment'].update(jobconfig.env)

        xrslText = Grid.expandxrsl(xrsl)

        # append any additional requirements from the requirements object
        xrslText += '\n'.join(self.requirements.other)

        logger.debug('subjob XRSL: %s' % xrslText)
        return inpw.writefile(FileBuffer('__xrslfile__', xrslText))
Example #6
0
    def preparejob(self, jobconfig, master_input_sandbox):
        """Method for preparing job script"""

        job = self.getJobObject()

        from Ganga.GPIDev.Lib.File import File
        from Ganga.Core.Sandbox.WNSandbox import PYTHON_DIR
        import Ganga.Utility.files
        import inspect

        fileutils = File(inspect.getsourcefile(Ganga.Utility.files),
                         subdir=PYTHON_DIR)
        inputfiles = jobconfig.getSandboxFiles() + [fileutils]
        inbox = job.createPackedInputSandbox(inputfiles)

        inbox.extend(master_input_sandbox)
        inpDir = job.getInputWorkspace(create=True).getPath()
        outDir = job.getOutputWorkspace(create=True).getPath()
        workdir = tempfile.mkdtemp()
        self.workdir = workdir
        exeString = jobconfig.getExeString()
        argList = jobconfig.getArgStrings()
        argString = " ".join(map(lambda x: " %s " % x, argList))

        outputSandboxPatterns = jobconfig.outputbox
        patternsToZip = []
        wnCodeForPostprocessing = ''
        wnCodeToDownloadInputFiles = ''

        if (len(job.outputfiles) > 0):

            from Ganga.GPIDev.Lib.File.OutputFileManager import getOutputSandboxPatternsForInteractive, getWNCodeForOutputPostprocessing
            (outputSandboxPatterns,
             patternsToZip) = getOutputSandboxPatternsForInteractive(job)

            wnCodeForPostprocessing = 'def printError(message):pass\ndef printInfo(message):pass' + \
                getWNCodeForOutputPostprocessing(job, '')

        all_inputfiles = [this_file for this_file in job.inputfiles]
        if job.master is not None:
            all_inputfiles.extend(
                [this_file for this_file in job.master.inputfiles])

        wnCodeToDownloadInputFiles = ''

        if (len(all_inputfiles) > 0):

            from Ganga.GPIDev.Lib.File.OutputFileManager import outputFilePostProcessingOnWN

            for inputFile in all_inputfiles:

                inputfileClassName = getName(inputFile)

                logger.debug("name: %s" % inputfileClassName)
                logger.debug(
                    "result: %s" %
                    str(outputFilePostProcessingOnWN(job, inputfileClassName)))

                if outputFilePostProcessingOnWN(job, inputfileClassName):
                    inputFile.processWildcardMatches()
                    if inputFile.subfiles:
                        getfromFile = False
                        for subfile in inputFile.subfiles:
                            wnCodeToDownloadInputFiles += subfile.getWNScriptDownloadCommand(
                                '')
                        else:
                            getfromFile = True
                    else:
                        getFromFile = True

                    if getFromFile:
                        wnCodeToDownloadInputFiles += inputFile.getWNScriptDownloadCommand(
                            '')

        wnCodeToDownloadInputData = ''

        if job.inputdata and (len(job.inputdata) > 0):

            from Ganga.GPIDev.Lib.File.OutputFileManager import getWNCodeForDownloadingInputFiles

            wnCodeToDownloadInputData = getWNCodeForDownloadingInputFiles(
                job, '')

        import inspect

        replace_dict = {
            '###CONSTRUCT_TIME###': (time.strftime("%c")),
            '###WNSANDBOX_SOURCE###':
            inspect.getsource(Sandbox.WNSandbox),
            '###GANGA_PYTHONPATH###':
            getConfig("System")["GANGA_PYTHONPATH"],
            '###OUTPUTDIR###':
            outDir,
            '###WORKDIR###':
            workdir,
            '###IN_BOX###':
            inbox,
            '###WN_INPUTFILES###':
            wnCodeToDownloadInputFiles,
            '###WN_INPUTDATA###':
            wnCodeToDownloadInputData,
            '###JOBCONFIG_ENV###':
            jobconfig.env if jobconfig.env is not None else dict(),
            '###EXE_STRING###':
            exeString,
            '###ARG_STRING###':
            argString,
            '###WN_POSTPROCESSING###':
            wnCodeForPostprocessing,
            '###PATTERNS_TO_ZIP###':
            patternsToZip,
            '###OUTPUT_SANDBOX_PATTERNS###':
            outputSandboxPatterns
        }

        script_location = os.path.join(
            os.path.dirname(
                os.path.abspath(inspect.getfile(inspect.currentframe()))),
            'InteractiveScriptTemplate.py.template')

        from Ganga.GPIDev.Lib.File import FileUtils
        commandString = FileUtils.loadScript(script_location, '')

        for k, v in replace_dict.iteritems():
            commandString = commandString.replace(str(k), str(v))

        return job.getInputWorkspace().writefile(FileBuffer(
            "__jobscript__", commandString),
                                                 executable=1)
Example #7
0
    def preparejob(self, jobconfig, master_input_sandbox):

        job = self.getJobObject()
        mon = job.getMonitoringService()
        import Ganga.Core.Sandbox as Sandbox
        from Ganga.GPIDev.Lib.File import File
        from Ganga.Core.Sandbox.WNSandbox import PYTHON_DIR
        import inspect

        fileutils = File( inspect.getsourcefile(Ganga.Utility.files), subdir=PYTHON_DIR )
        subjob_input_sandbox = job.createPackedInputSandbox(jobconfig.getSandboxFiles() + [ fileutils ] )

        appscriptpath = [jobconfig.getExeString()] + jobconfig.getArgStrings()
        sharedoutputpath = job.getOutputWorkspace().getPath()
        ## FIXME Check this isn't a GangaList
        outputpatterns = jobconfig.outputbox
        environment = jobconfig.env if not jobconfig.env is None else {}


        import inspect
        script_location = os.path.join(os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))),
                                                       'BatchScriptTemplate.py')

        from Ganga.GPIDev.Lib.File import FileUtils
        text = FileUtils.loadScript(script_location, '')

        import Ganga.Core.Sandbox as Sandbox
        import Ganga.Utility as Utility
        from Ganga.Utility.Config import getConfig
        from Ganga.GPIDev.Lib.File.OutputFileManager import getWNCodeForOutputSandbox, getWNCodeForOutputPostprocessing, getWNCodeForDownloadingInputFiles
        jobidRepr = repr(self.getJobObject().getFQID('.'))

        replace_dict = {

        '###OUTPUTSANDBOXPOSTPROCESSING###' : getWNCodeForOutputSandbox(job, ['__syslog__'], jobidRepr),

        '###OUTPUTUPLOADSPOSTPROCESSING###' : getWNCodeForOutputPostprocessing(job, ''),

        '###DOWNLOADINPUTFILES###' : getWNCodeForDownloadingInputFiles(job, ''),

        '###INLINEMODULES###' : inspect.getsource(Sandbox.WNSandbox),
        '###INLINEHOSTNAMEFUNCTION###' : inspect.getsource(Utility.util.hostname),
        '###APPSCRIPTPATH###' : repr(appscriptpath),
        #'###SHAREDINPUTPATH###' : repr(sharedinputpath)),

        '###INPUT_SANDBOX###' : repr(subjob_input_sandbox + master_input_sandbox),
        '###SHAREDOUTPUTPATH###' : repr(sharedoutputpath),

        '###OUTPUTPATTERNS###' : repr(outputpatterns),
        '###JOBID###' : jobidRepr,
        '###ENVIRONMENT###' : repr(environment),
        '###PREEXECUTE###' : self.config['preexecute'],
        '###POSTEXECUTE###' : self.config['postexecute'],
        '###JOBIDNAME###' : self.config['jobid_name'],
        '###QUEUENAME###' : self.config['queue_name'],
        '###HEARTBEATFREQUENCE###' : self.config['heartbeat_frequency'],
        '###INPUT_DIR###' : repr(job.getStringInputDir()),

        '###GANGADIR###' : repr(getConfig('System')['GANGA_PYTHONPATH'])
        }

        for k, v in replace_dict.iteritems():
            text = text.replace(str(k), str(v))

        logger.debug('subjob input sandbox %s ', subjob_input_sandbox)
        logger.debug('master input sandbox %s ', master_input_sandbox)

        from Ganga.GPIDev.Lib.File import FileBuffer

        return job.getInputWorkspace().writefile(FileBuffer('__jobscript__', text), executable=1)
Example #8
0
    def preparejob(self, jobconfig, master_input_sandbox):

        job = self.getJobObject()
        # print str(job.backend_output_postprocess)
        mon = job.getMonitoringService()
        import Ganga.Core.Sandbox as Sandbox

        subjob_input_sandbox = job.createPackedInputSandbox(
            jobconfig.getSandboxFiles()
            + Sandbox.getGangaModulesAsSandboxFiles(Sandbox.getDefaultModules())
            + Sandbox.getGangaModulesAsSandboxFiles(mon.getSandboxModules())
        )

        appscriptpath = [jobconfig.getExeString()] + jobconfig.getArgStrings()
        if self.nice:
            appscriptpath = ["nice", "-n %d" % self.nice] + appscriptpath
        if self.nice < 0:
            logger.warning("increasing process priority is often not allowed, your job may fail due to this")

        sharedoutputpath = job.getOutputWorkspace().getPath()
        outputpatterns = jobconfig.outputbox
        environment = dict() if jobconfig.env is None else jobconfig.env

        import tempfile

        workdir = tempfile.mkdtemp(dir=config["location"])

        import inspect

        script_location = os.path.join(
            os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))), "LocalHostExec.py"
        )

        from Ganga.GPIDev.Lib.File import FileUtils

        script = FileUtils.loadScript(script_location, "")

        script = script.replace("###INLINEMODULES###", inspect.getsource(Sandbox.WNSandbox))

        from Ganga.GPIDev.Lib.File.OutputFileManager import (
            getWNCodeForOutputSandbox,
            getWNCodeForOutputPostprocessing,
            getWNCodeForDownloadingInputFiles,
            getWNCodeForInputdataListCreation,
        )
        from Ganga.Utility.Config import getConfig

        jobidRepr = repr(job.getFQID("."))

        script = script.replace(
            "###OUTPUTSANDBOXPOSTPROCESSING###",
            getWNCodeForOutputSandbox(job, ["stdout", "stderr", "__syslog__"], jobidRepr),
        )
        script = script.replace("###OUTPUTUPLOADSPOSTPROCESSING###", getWNCodeForOutputPostprocessing(job, ""))
        script = script.replace("###DOWNLOADINPUTFILES###", getWNCodeForDownloadingInputFiles(job, ""))
        script = script.replace("###CREATEINPUTDATALIST###", getWNCodeForInputdataListCreation(job, ""))

        script = script.replace("###APPLICATION_NAME###", repr(job.application._name))
        script = script.replace("###INPUT_SANDBOX###", repr(subjob_input_sandbox + master_input_sandbox))
        script = script.replace("###SHAREDOUTPUTPATH###", repr(sharedoutputpath))
        script = script.replace("###APPSCRIPTPATH###", repr(appscriptpath))
        script = script.replace("###OUTPUTPATTERNS###", repr(outputpatterns))
        script = script.replace("###JOBID###", jobidRepr)
        script = script.replace("###ENVIRONMENT###", repr(environment))
        script = script.replace("###WORKDIR###", repr(workdir))
        script = script.replace("###INPUT_DIR###", repr(job.getStringInputDir()))

        script = script.replace(
            "###MONITORING_SERVICE###", job.getMonitoringService().getWrapperScriptConstructorText()
        )

        self.workdir = workdir

        script = script.replace("###GANGADIR###", repr(getConfig("System")["GANGA_PYTHONPATH"]))

        wrkspace = job.getInputWorkspace()
        scriptPath = wrkspace.writefile(FileBuffer("__jobscript__", script), executable=1)

        return scriptPath
Example #9
0
    def preparejob(self, jobconfig, master_job_sandbox):
        """Prepare the JDL"""

        script = self.__jobWrapperTemplate__()

        job = self.getJobObject()
        inpw = job.getInputWorkspace()

        wrapperlog = "__jobscript__.log"

        import Ganga.Core.Sandbox as Sandbox

        # FIXME: check what happens if 'stdout','stderr' are specified here
        script = script.replace("###OUTPUTSANDBOX###", repr(jobconfig.outputbox))

        script = script.replace("###APPLICATION_NAME###", job.application._name)
        script = script.replace("###APPLICATIONEXEC###", repr(jobconfig.getExeString()))
        script = script.replace("###APPLICATIONARGS###", repr(jobconfig.getArguments()))

        from Ganga.GPIDev.Lib.File.OutputFileManager import (
            getWNCodeForOutputPostprocessing,
            getWNCodeForDownloadingInputFiles,
        )

        script = script.replace("###OUTPUTUPLOADSPOSTPROCESSING###", getWNCodeForOutputPostprocessing(job, "    "))

        script = script.replace("###DOWNLOADINPUTFILES###", getWNCodeForDownloadingInputFiles(job, "    "))

        if jobconfig.env:
            script = script.replace("###APPLICATIONENVS###", repr(jobconfig.env))
        else:
            script = script.replace("###APPLICATIONENVS###", repr({}))

        script = script.replace("###WRAPPERLOG###", repr(wrapperlog))
        import inspect

        script = script.replace("###INLINEMODULES###", inspect.getsource(Sandbox.WNSandbox))

        mon = job.getMonitoringService()

        self.monInfo = None

        # set the monitoring file by default to the stdout
        if isinstance(self.monInfo, dict):
            self.monInfo["remotefile"] = "stdout"

        # try to print out the monitoring service information in debug mode
        try:
            logger.debug("job info of monitoring service: %s" % str(self.monInfo))
        except:
            pass

        script = script.replace("###MONITORING_SERVICE###", mon.getWrapperScriptConstructorText())

        #       prepare input/output sandboxes
        packed_files = (
            jobconfig.getSandboxFiles()
            + Sandbox.getGangaModulesAsSandboxFiles(Sandbox.getDefaultModules())
            + Sandbox.getGangaModulesAsSandboxFiles(mon.getSandboxModules())
        )
        sandbox_files = job.createPackedInputSandbox(packed_files)

        # sandbox of child jobs should include master's sandbox
        sandbox_files.extend(master_job_sandbox)

        # check the input file size and pre-upload larger inputs to the iocache
        lfc_host = ""

        input_sandbox_uris = []
        input_sandbox_names = []

        ick = True

        max_prestaged_fsize = 0
        for f in sandbox_files:

            idx = self.__check_and_prestage_inputfile__(f)

            if not idx:
                logger.error("input sandbox preparation failed: %s" % f)
                ick = False
                break
            else:

                if idx["lfc_host"]:
                    lfc_host = idx["lfc_host"]

                if idx["remote"]:
                    abspath = os.path.abspath(f)
                    fsize = os.path.getsize(abspath)

                    if fsize > max_prestaged_fsize:
                        max_prestaged_fsize = fsize

                    input_sandbox_uris.append(idx["remote"][os.path.basename(f)])

                    input_sandbox_names.append(os.path.basename(urlparse(f)[2]))

                if idx["local"]:
                    input_sandbox_uris += idx["local"]
                    input_sandbox_names.append(os.path.basename(f))

        if not ick:
            logger.error("stop job submission")
            return None

        # determin the lcg-cp timeout according to the max_prestaged_fsize
        # - using the assumption of 1 MB/sec.
        max_prestaged_fsize = 0
        lfc_host = ""
        transfer_timeout = config["SandboxTransferTimeout"]
        predict_timeout = int(math.ceil(max_prestaged_fsize / 1000000.0))

        if predict_timeout > transfer_timeout:
            transfer_timeout = predict_timeout

        if transfer_timeout < 60:
            transfer_timeout = 60

        script = script.replace("###TRANSFERTIMEOUT###", "%d" % transfer_timeout)

        # update the job wrapper with the inputsandbox list
        script = script.replace("###INPUTSANDBOX###", repr({"remote": {}, "local": input_sandbox_names}))

        # write out the job wrapper and put job wrapper into job's inputsandbox
        scriptPath = inpw.writefile(FileBuffer("__jobscript_%s__" % job.getFQID("."), script), executable=1)
        input_sandbox = input_sandbox_uris + [scriptPath]

        for isb in input_sandbox:
            logger.debug("ISB URI: %s" % isb)

        # compose output sandbox to include by default the following files:
        # - gzipped stdout (transferred only when the JobLogHandler is WMS)
        # - gzipped stderr (transferred only when the JobLogHandler is WMS)
        # - __jobscript__.log (job wrapper's log)
        output_sandbox = [wrapperlog]

        from Ganga.GPIDev.Lib.File.OutputFileManager import getOutputSandboxPatterns

        for outputSandboxPattern in getOutputSandboxPatterns(job):
            output_sandbox.append(outputSandboxPattern)

        if config["JobLogHandler"] in ["WMS"]:
            output_sandbox += ["stdout.gz", "stderr.gz"]

        if len(jobconfig.outputbox):
            output_sandbox += [Sandbox.OUTPUT_TARBALL_NAME]

        # compose LCG JDL
        jdl = {
            "VirtualOrganisation": config["VirtualOrganisation"],
            "Executable": os.path.basename(scriptPath),
            "Environment": {
                "GANGA_LCG_VO": config["VirtualOrganisation"],
                "GANGA_LOG_HANDLER": config["JobLogHandler"],
                "LFC_HOST": lfc_host,
            },
            "StdOutput": "stdout",
            "StdError": "stderr",
            "InputSandbox": input_sandbox,
            "OutputSandbox": output_sandbox,
            "OutputSandboxBaseDestURI": "gsiftp://localhost",
        }

        jdl["Environment"].update({"GANGA_LCG_CE": self.CE})
        jdl["Requirements"] = self.requirements.merge(jobconfig.requirements).convert()

        if self.jobtype.upper() in ["NORMAL", "MPICH"]:
            jdl["JobType"] = self.jobtype.upper()
            if self.jobtype.upper() == "MPICH":
                # jdl['Requirements'].append('(other.GlueCEInfoTotalCPUs >= NodeNumber)')
                jdl["Requirements"].append('Member("MPICH",other.GlueHostApplicationSoftwareRunTimeEnvironment)')
                jdl["NodeNumber"] = self.requirements.nodenumber
        else:
            logger.warning('JobType "%s" not supported' % self.jobtype)
            return

        #       additional settings from the job
        #        if jobconfig.env:
        #            jdl['Environment'].update(jobconfig.env)

        jdlText = Grid.expandjdl(jdl)
        logger.debug("subjob JDL: %s" % jdlText)
        return inpw.writefile(FileBuffer("__jdlfile__", jdlText))