Ejemplo n.º 1
0
def __get_app_exitcode__(outputdir):
    import Ganga.Core.Sandbox as Sandbox

    Sandbox.getPackedOutputSandbox(outputdir, outputdir)

    # check the application exit code
    app_exitcode = -1
    runtime_log = os.path.join(outputdir, '__jobscript__.log')
    pat = re.compile(r'.*exit code (\d+).')

    if not os.path.exists(runtime_log):
        logger.warning('job runtime log not found: %s' % runtime_log)
        return False, 'job runtime log not found: %s' % runtime_log

    for line in open(runtime_log, 'r'):
        mat = pat.match(line)
        if mat:
            app_exitcode = eval(mat.groups()[0])
            break

    # returns False if the exit code of the real executable is not zero
    # the job status of GANGA will be changed to 'failed' if the return
    # value is False
    if app_exitcode != 0:
        logger.debug(
            'job\'s executable returns non-zero exit code: %d' % app_exitcode)
        return False, app_exitcode
    else:
        return True, 0
Ejemplo n.º 2
0
def __get_app_exitcode__(outputdir):
    import Ganga.Core.Sandbox as Sandbox

    Sandbox.getPackedOutputSandbox(outputdir, outputdir)

    # check the application exit code
    app_exitcode = -1
    runtime_log = os.path.join(outputdir, '__jobscript__.log')
    pat = re.compile(r'.*exit code (\d+).')

    if not os.path.exists(runtime_log):
        logger.warning('job runtime log not found: %s' % runtime_log)
        return False, 'job runtime log not found: %s' % runtime_log

    for line in open(runtime_log, 'r'):
        mat = pat.match(line)
        if mat:
            app_exitcode = eval(mat.groups()[0])
            break

    # returns False if the exit code of the real executable is not zero
    # the job status of GANGA will be changed to 'failed' if the return
    # value is False
    if app_exitcode != 0:
        logger.debug('job\'s executable returns non-zero exit code: %d' %
                     app_exitcode)
        return False, app_exitcode
    else:
        return True, 0
Ejemplo n.º 3
0
    def preparejob(self, jobconfig, master_input_sandbox):

        job = self.getJobObject()
        # print str(job.backend_output_postprocess)
        mon = job.getMonitoringService()
        import Ganga.Core.Sandbox as Sandbox
        subjob_input_sandbox = job.createPackedInputSandbox(jobconfig.getSandboxFiles() + Sandbox.getGangaModulesAsSandboxFiles(Sandbox.getDefaultModules()))

        appscriptpath = [jobconfig.getExeString()] + jobconfig.getArgStrings()
        if self.nice:
            appscriptpath = ['nice', '-n %d' % self.nice] + appscriptpath
        if self.nice < 0:
            logger.warning('increasing process priority is often not allowed, your job may fail due to this')

        sharedoutputpath = job.getOutputWorkspace().getPath()
        ## FIXME DON'T just use the blind list here, request the list of files to be in the output from a method.
        outputpatterns = jobconfig.outputbox
        environment = dict() if jobconfig.env is None else jobconfig.env

        import tempfile
        workdir = tempfile.mkdtemp(dir=config['location'])

        import inspect
        script_location = os.path.join(os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))),
                                                        'LocalHostExec.py')

        from Ganga.GPIDev.Lib.File import FileUtils
        script = FileUtils.loadScript(script_location, '')

        script = script.replace('###INLINEMODULES###', inspect.getsource(Sandbox.WNSandbox))

        from Ganga.GPIDev.Lib.File.OutputFileManager import getWNCodeForOutputSandbox, getWNCodeForOutputPostprocessing, getWNCodeForDownloadingInputFiles, getWNCodeForInputdataListCreation
        from Ganga.Utility.Config import getConfig
        jobidRepr = repr(job.getFQID('.'))


        script = script.replace('###OUTPUTSANDBOXPOSTPROCESSING###', getWNCodeForOutputSandbox(job, ['stdout', 'stderr', '__syslog__'], jobidRepr))
        script = script.replace('###OUTPUTUPLOADSPOSTPROCESSING###', getWNCodeForOutputPostprocessing(job, ''))
        script = script.replace('###DOWNLOADINPUTFILES###', getWNCodeForDownloadingInputFiles(job, ''))
        script = script.replace('###CREATEINPUTDATALIST###', getWNCodeForInputdataListCreation(job, ''))

        script = script.replace('###APPLICATION_NAME###', repr(getName(job.application)))
        script = script.replace('###INPUT_SANDBOX###', repr(subjob_input_sandbox + master_input_sandbox))
        script = script.replace('###SHAREDOUTPUTPATH###', repr(sharedoutputpath))
        script = script.replace('###APPSCRIPTPATH###', repr(appscriptpath))
        script = script.replace('###OUTPUTPATTERNS###', str(outputpatterns))
        script = script.replace('###JOBID###', jobidRepr)
        script = script.replace('###ENVIRONMENT###', repr(environment))
        script = script.replace('###WORKDIR###', repr(workdir))
        script = script.replace('###INPUT_DIR###', repr(job.getStringInputDir()))

        self.workdir = workdir

        script = script.replace('###GANGADIR###', repr(getConfig('System')['GANGA_PYTHONPATH']))

        wrkspace = job.getInputWorkspace()
        scriptPath = wrkspace.writefile(FileBuffer('__jobscript__', script), executable=1)

        return scriptPath
Ejemplo n.º 4
0
    def prestagesandbox(self, jobconfig):

        job = self.getJobObject()
        mon = job.getMonitoringService()
        inw = job.getInputWorkspace()

        sandbox_files = jobconfig.getSandboxFiles()

        logger.info("pre-staging files for saga job with id: %s", job.id)

        ## Compression is ENABLED. All input sandbox files are archived
        ## into an uncompressed tarball which is then  transfered to the
        ## execution host. This speeds up things a lot in many scenarios.
        ##
        if self.enable_compression:
            logger.info("  * adding %s user defined files to input sandbox", len(sandbox_files))

            import Ganga.Core.Sandbox as Sandbox

            compressed_input_sandbox = job.createPackedInputSandbox(
                jobconfig.getSandboxFiles()
                + Sandbox.getGangaModulesAsSandboxFiles(Sandbox.getDefaultModules())
                + Sandbox.getGangaModulesAsSandboxFiles(mon.getSandboxModules())
            )

            try:
                for f in compressed_input_sandbox:
                    archive_url = saga.url("file://localhost/" + f)
                    logger.info(
                        "  * copying %s -> %s ", archive_url.url, self.filesystem_url + "/" + self.workdir_uuid + "/"
                    )
                    sf = saga.filesystem.file(archive_url.url)
                    sf.copy(self.filesystem_url + "/" + self.workdir_uuid + "/", saga.filesystem.Overwrite)

            except saga.exception as e:
                logger.error("exception caught while transfering file: %s", e.get_all_exceptions())
                job.updateStatus("failed")
                return False

        ## Compression is DISABLED. All input sandbox files are transfered
        ## one-by-one to the execution host. This can be very slow, especially
        ## if SAGA is using the Globus GridFTP adaptor.
        else:
            logger.info("prestaging %s input sandbox files", len(sandbox_files))
            for f in sandbox_files:
                try:
                    source = saga.url(f.name)
                    if len(source.scheme) == 0:
                        source.scheme = "file"
                    if len(source.host) == 0:
                        source.host = "localhost"

                    if f.subdir != ".":  # create subdirectory & copy
                        target = saga.url(self.filesystem_url + "/" + self.workdir_uuid + "/" + f.subdir + "/")

                        sd = saga.filesystem.directory(target, saga.filesystem.Create)
                        sf = saga.filesystem.file(source)

                        logger.info("  * copying %s -> %s ", source, target)
                        sf.copy(target, saga.filesystem.Overwrite)  # copy the file

                    else:  # copy to remote work dir
                        logger.info(
                            "  * copying %s -> %s ", source, self.filesystem_url + "/" + self.workdir_uuid + "/"
                        )
                        sf = saga.filesystem.file(source)
                        sf.copy(self.filesystem_url + "/" + self.workdir_uuid + "/", saga.filesystem.Overwrite)

                except saga.exception as e:
                    logger.error("exception caught while transfering file: %s", e.get_all_exceptions())
                    job.updateStatus("failed")
                    return False

        ##
        ## copy the job script
        ##
        try:
            jobscript_path = saga.url("file://localhost/" + inw.getPath() + "/__jobscript__")
            if os.path.exists(jobscript_path.path):
                logger.info(
                    "  * copying %s -> %s ", jobscript_path.url, self.filesystem_url + "/" + self.workdir_uuid + "/"
                )
                sf = saga.filesystem.file(jobscript_path.url)
                sf.copy(self.filesystem_url + "/" + self.workdir_uuid + "/", saga.filesystem.Overwrite)

        except saga.exception as e:
            logger.error("exception caught while transfering file: %s", e.get_all_exceptions())
            job.updateStatus("failed")
            return False

        return True
Ejemplo n.º 5
0
    def preparejob(self, jobconfig, master_job_sandbox):
        '''Prepare the JDL'''

        script = self.__jobWrapperTemplate__()

        job = self.getJobObject()
        inpw = job.getInputWorkspace()

        wrapperlog = '__jobscript__.log'

        import Ganga.Core.Sandbox as Sandbox

        # FIXME: check what happens if 'stdout','stderr' are specified here
        script = script.replace(
            '###OUTPUTSANDBOX###', repr(jobconfig.outputbox))

        script = script.replace(
            '###APPLICATION_NAME###', getName(job.application))
        script = script.replace(
            '###APPLICATIONEXEC###', repr(jobconfig.getExeString()))
        script = script.replace(
            '###APPLICATIONARGS###', repr(jobconfig.getArguments()))

        from Ganga.GPIDev.Lib.File.OutputFileManager import getWNCodeForOutputPostprocessing, getWNCodeForDownloadingInputFiles

        script = script.replace(
            '###OUTPUTUPLOADSPOSTPROCESSING###', getWNCodeForOutputPostprocessing(job, '    '))

        script = script.replace(
            '###DOWNLOADINPUTFILES###', getWNCodeForDownloadingInputFiles(job, '    '))

        if jobconfig.env:
            script = script.replace(
                '###APPLICATIONENVS###', repr(jobconfig.env))
        else:
            script = script.replace('###APPLICATIONENVS###', repr({}))

        script = script.replace('###WRAPPERLOG###', repr(wrapperlog))
        import inspect
        script = script.replace(
            '###INLINEMODULES###', inspect.getsource(Sandbox.WNSandbox))

        mon = job.getMonitoringService()

        self.monInfo = None

        # set the monitoring file by default to the stdout
        if isinstance(self.monInfo, dict):
            self.monInfo['remotefile'] = 'stdout'

        # try to print out the monitoring service information in debug mode
        try:
            logger.debug('job info of monitoring service: %s' %
                         str(self.monInfo))
        except:
            pass

#       prepare input/output sandboxes
        packed_files = jobconfig.getSandboxFiles() + Sandbox.getGangaModulesAsSandboxFiles(Sandbox.getDefaultModules())
        sandbox_files = job.createPackedInputSandbox(packed_files)

        # sandbox of child jobs should include master's sandbox
        sandbox_files.extend(master_job_sandbox)

        # check the input file size and pre-upload larger inputs to the iocache
        lfc_host = ''

        input_sandbox_uris = []
        input_sandbox_names = []

        ick = True

        max_prestaged_fsize = 0
        for f in sandbox_files:

            idx = self.__check_and_prestage_inputfile__(f)

            if not idx:
                logger.error('input sandbox preparation failed: %s' % f)
                ick = False
                break
            else:

                if idx['lfc_host']:
                    lfc_host = idx['lfc_host']

                if idx['remote']:
                    abspath = os.path.abspath(f)
                    fsize = os.path.getsize(abspath)

                    if fsize > max_prestaged_fsize:
                        max_prestaged_fsize = fsize

                    input_sandbox_uris.append(
                        idx['remote'][os.path.basename(f)])

                    input_sandbox_names.append(
                        os.path.basename(urlparse(f)[2]))

                if idx['local']:
                    input_sandbox_uris += idx['local']
                    input_sandbox_names.append(os.path.basename(f))

        if not ick:
            logger.error('stop job submission')
            return None

        # determin the lcg-cp timeout according to the max_prestaged_fsize
        # - using the assumption of 1 MB/sec.
        max_prestaged_fsize = 0
        lfc_host = ''
        transfer_timeout = config['SandboxTransferTimeout']
        predict_timeout = int(math.ceil(max_prestaged_fsize / 1000000.0))

        if predict_timeout > transfer_timeout:
            transfer_timeout = predict_timeout

        if transfer_timeout < 60:
            transfer_timeout = 60

        script = script.replace(
            '###TRANSFERTIMEOUT###', '%d' % transfer_timeout)

        # update the job wrapper with the inputsandbox list
        script = script.replace(
            '###INPUTSANDBOX###', repr({'remote': {}, 'local': input_sandbox_names}))

        # write out the job wrapper and put job wrapper into job's inputsandbox
        scriptPath = inpw.writefile(
            FileBuffer('__jobscript_%s__' % job.getFQID('.'), script), executable=1)
        input_sandbox = input_sandbox_uris + [scriptPath]

        for isb in input_sandbox:
            logger.debug('ISB URI: %s' % isb)

        # compose output sandbox to include by default the following files:
        # - gzipped stdout (transferred only when the JobLogHandler is WMS)
        # - gzipped stderr (transferred only when the JobLogHandler is WMS)
        # - __jobscript__.log (job wrapper's log)
        output_sandbox = [wrapperlog]

        from Ganga.GPIDev.Lib.File.OutputFileManager import getOutputSandboxPatterns
        for outputSandboxPattern in getOutputSandboxPatterns(job):
            output_sandbox.append(outputSandboxPattern)

        if config['JobLogHandler'] in ['WMS']:
            output_sandbox += ['stdout.gz', 'stderr.gz']

        if len(jobconfig.outputbox):
            output_sandbox += [Sandbox.OUTPUT_TARBALL_NAME]

        # compose LCG JDL
        jdl = {
            'VirtualOrganisation': config['VirtualOrganisation'],
            'Executable': os.path.basename(scriptPath),
            'Environment': {'GANGA_LCG_VO': config['VirtualOrganisation'], 'GANGA_LOG_HANDLER': config['JobLogHandler'], 'LFC_HOST': lfc_host},
            'StdOutput': 'stdout',
            'StdError': 'stderr',
            'InputSandbox': input_sandbox,
            'OutputSandbox': output_sandbox,
            'OutputSandboxBaseDestURI': 'gsiftp://localhost'
        }

        jdl['Environment'].update({'GANGA_LCG_CE': self.CE})
        jdl['Requirements'] = self.requirements.merge(
            jobconfig.requirements).convert()

        if self.jobtype.upper() in ['NORMAL', 'MPICH']:
            jdl['JobType'] = self.jobtype.upper()
            if self.jobtype.upper() == 'MPICH':
                #jdl['Requirements'].append('(other.GlueCEInfoTotalCPUs >= NodeNumber)')
                jdl['Requirements'].append(
                    'Member("MPICH",other.GlueHostApplicationSoftwareRunTimeEnvironment)')
                jdl['NodeNumber'] = self.requirements.nodenumber
        else:
            logger.warning('JobType "%s" not supported' % self.jobtype)
            return

#       additional settings from the job
#        if jobconfig.env:
#            jdl['Environment'].update(jobconfig.env)

        jdlText = Grid.expandjdl(jdl)
        logger.debug('subjob JDL: %s' % jdlText)
        return inpw.writefile(FileBuffer('__jdlfile__', jdlText))
Ejemplo n.º 6
0
    def preparejob(self, jobconfig, master_input_sandbox):

        job = self.getJobObject()
        # print str(job.backend_output_postprocess)
        mon = job.getMonitoringService()
        import Ganga.Core.Sandbox as Sandbox
        subjob_input_sandbox = job.createPackedInputSandbox(jobconfig.getSandboxFiles()
                                                            +
                                                            Sandbox.getGangaModulesAsSandboxFiles(
                                                                Sandbox.getDefaultModules())
                                                            + Sandbox.getGangaModulesAsSandboxFiles(mon.getSandboxModules()))

        appscriptpath = [jobconfig.getExeString()] + jobconfig.getArgStrings()
        if self.nice:
            appscriptpath = ['nice', '-n %d' % self.nice] + appscriptpath
        if self.nice < 0:
            logger.warning('increasing process priority is often not allowed, your job may fail due to this')

        sharedoutputpath = job.getOutputWorkspace().getPath()
        outputpatterns = jobconfig.outputbox
        environment = dict() if jobconfig.env is None else jobconfig.env

        import tempfile
        workdir = tempfile.mkdtemp(dir=config['location'])

        import inspect
        script_location = os.path.join(os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))),
                                                        'LocalHostExec.py')

        from Ganga.GPIDev.Lib.File import FileUtils
        script = FileUtils.loadScript(script_location, '')

        script = script.replace('###INLINEMODULES###', inspect.getsource(Sandbox.WNSandbox))

        from Ganga.GPIDev.Lib.File.OutputFileManager import getWNCodeForOutputSandbox, getWNCodeForOutputPostprocessing, getWNCodeForDownloadingInputFiles, getWNCodeForInputdataListCreation
        from Ganga.Utility.Config import getConfig
        jobidRepr = repr(job.getFQID('.'))


        script = script.replace('###OUTPUTSANDBOXPOSTPROCESSING###', getWNCodeForOutputSandbox(job, ['stdout', 'stderr', '__syslog__'], jobidRepr))
        script = script.replace('###OUTPUTUPLOADSPOSTPROCESSING###', getWNCodeForOutputPostprocessing(job, ''))
        script = script.replace('###DOWNLOADINPUTFILES###', getWNCodeForDownloadingInputFiles(job, ''))
        script = script.replace('###CREATEINPUTDATALIST###', getWNCodeForInputdataListCreation(job, ''))

        script = script.replace('###APPLICATION_NAME###', repr(job.application._name))
        script = script.replace('###INPUT_SANDBOX###', repr(subjob_input_sandbox + master_input_sandbox))
        script = script.replace('###SHAREDOUTPUTPATH###', repr(sharedoutputpath))
        script = script.replace('###APPSCRIPTPATH###', repr(appscriptpath))
        script = script.replace('###OUTPUTPATTERNS###', repr(outputpatterns))
        script = script.replace('###JOBID###', jobidRepr)
        script = script.replace('###ENVIRONMENT###', repr(environment))
        script = script.replace('###WORKDIR###', repr(workdir))
        script = script.replace('###INPUT_DIR###', repr(job.getStringInputDir()))

        script = script.replace('###MONITORING_SERVICE###', job.getMonitoringService().getWrapperScriptConstructorText())

        self.workdir = workdir

        script = script.replace('###GANGADIR###', repr(getConfig('System')['GANGA_PYTHONPATH']))

        wrkspace = job.getInputWorkspace()
        scriptPath = wrkspace.writefile(
            FileBuffer('__jobscript__', script), executable=1)

        return scriptPath
Ejemplo n.º 7
0
    def preparejob(self, jobconfig, master_input_sandbox):

        job = self.getJobObject()
        mon = job.getMonitoringService()
        import Ganga.Core.Sandbox as Sandbox
        subjob_input_sandbox = job.createPackedInputSandbox(jobconfig.getSandboxFiles() + Sandbox.getGangaModulesAsSandboxFiles(Sandbox.getDefaultModules()))

        appscriptpath = [jobconfig.getExeString()] + jobconfig.getArgStrings()
        sharedoutputpath = job.getOutputWorkspace().getPath()
        ## FIXME Check this isn't a GangaList
        outputpatterns = jobconfig.outputbox
        environment = jobconfig.env if not jobconfig.env is None else {}


        import inspect
        script_location = os.path.join(os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))),
                                                       'BatchScriptTemplate.py')

        from Ganga.GPIDev.Lib.File import FileUtils
        text = FileUtils.loadScript(script_location, '')

        import Ganga.Core.Sandbox as Sandbox
        import Ganga.Utility as Utility
        from Ganga.Utility.Config import getConfig
        from Ganga.GPIDev.Lib.File.OutputFileManager import getWNCodeForOutputSandbox, getWNCodeForOutputPostprocessing, getWNCodeForDownloadingInputFiles
        jobidRepr = repr(self.getJobObject().getFQID('.'))

        replace_dict = {

        '###OUTPUTSANDBOXPOSTPROCESSING###' : getWNCodeForOutputSandbox(job, ['__syslog__'], jobidRepr),

        '###OUTPUTUPLOADSPOSTPROCESSING###' : getWNCodeForOutputPostprocessing(job, ''),

        '###DOWNLOADINPUTFILES###' : getWNCodeForDownloadingInputFiles(job, ''),

        '###INLINEMODULES###' : inspect.getsource(Sandbox.WNSandbox),
        '###INLINEHOSTNAMEFUNCTION###' : inspect.getsource(Utility.util.hostname),
        '###APPSCRIPTPATH###' : repr(appscriptpath),
        #'###SHAREDINPUTPATH###' : repr(sharedinputpath)),

        '###INPUT_SANDBOX###' : repr(subjob_input_sandbox + master_input_sandbox),
        '###SHAREDOUTPUTPATH###' : repr(sharedoutputpath),

        '###OUTPUTPATTERNS###' : repr(outputpatterns),
        '###JOBID###' : jobidRepr,
        '###ENVIRONMENT###' : repr(environment),
        '###PREEXECUTE###' : self.config['preexecute'],
        '###POSTEXECUTE###' : self.config['postexecute'],
        '###JOBIDNAME###' : self.config['jobid_name'],
        '###QUEUENAME###' : self.config['queue_name'],
        '###HEARTBEATFREQUENCE###' : self.config['heartbeat_frequency'],
        '###INPUT_DIR###' : repr(job.getStringInputDir()),

        '###GANGADIR###' : repr(getConfig('System')['GANGA_PYTHONPATH'])
        }

        for k, v in replace_dict.iteritems():
            text = text.replace(str(k), str(v))

        logger.debug('subjob input sandbox %s ', subjob_input_sandbox)
        logger.debug('master input sandbox %s ', master_input_sandbox)

        from Ganga.GPIDev.Lib.File import FileBuffer

        return job.getInputWorkspace().writefile(FileBuffer('__jobscript__', text), executable=1)
Ejemplo n.º 8
0
    def preparejob(self, jobconfig, master_job_sandbox):
        '''Prepare the JDL'''

        script = self.__jobWrapperTemplate__()

        job = self.getJobObject()
        inpw = job.getInputWorkspace()

        wrapperlog = '__jobscript__.log'

        import Ganga.Core.Sandbox as Sandbox

        # FIXME: check what happens if 'stdout','stderr' are specified here
        script = script.replace(
            '###OUTPUTSANDBOX###', repr(jobconfig.outputbox))

        script = script.replace(
            '###APPLICATION_NAME###', getName(job.application))
        script = script.replace(
            '###APPLICATIONEXEC###', repr(jobconfig.getExeString()))
        script = script.replace(
            '###APPLICATIONARGS###', repr(jobconfig.getArguments()))

        from Ganga.GPIDev.Lib.File.OutputFileManager import getWNCodeForOutputPostprocessing, getWNCodeForDownloadingInputFiles

        script = script.replace(
            '###OUTPUTUPLOADSPOSTPROCESSING###', getWNCodeForOutputPostprocessing(job, '    '))

        script = script.replace(
            '###DOWNLOADINPUTFILES###', getWNCodeForDownloadingInputFiles(job, '    '))

        if jobconfig.env:
            script = script.replace(
                '###APPLICATIONENVS###', repr(jobconfig.env))
        else:
            script = script.replace('###APPLICATIONENVS###', repr({}))

        script = script.replace('###WRAPPERLOG###', repr(wrapperlog))
        import inspect
        script = script.replace(
            '###INLINEMODULES###', inspect.getsource(Sandbox.WNSandbox))

        mon = job.getMonitoringService()

        self.monInfo = None

        # set the monitoring file by default to the stdout
        if isinstance(self.monInfo, dict):
            self.monInfo['remotefile'] = 'stdout'

        # try to print out the monitoring service information in debug mode
        try:
            logger.debug('job info of monitoring service: %s' %
                         str(self.monInfo))
        except:
            pass

#       prepare input/output sandboxes
        packed_files = jobconfig.getSandboxFiles() + Sandbox.getGangaModulesAsSandboxFiles(Sandbox.getDefaultModules())
        sandbox_files = job.createPackedInputSandbox(packed_files)

        # sandbox of child jobs should include master's sandbox
        sandbox_files.extend(master_job_sandbox)

        # check the input file size and pre-upload larger inputs to the iocache
        lfc_host = ''

        input_sandbox_uris = []
        input_sandbox_names = []

        ick = True

        max_prestaged_fsize = 0
        for f in sandbox_files:

            idx = self.__check_and_prestage_inputfile__(f)

            if not idx:
                logger.error('input sandbox preparation failed: %s' % f)
                ick = False
                break
            else:

                if idx['lfc_host']:
                    lfc_host = idx['lfc_host']

                if idx['remote']:
                    abspath = os.path.abspath(f)
                    fsize = os.path.getsize(abspath)

                    if fsize > max_prestaged_fsize:
                        max_prestaged_fsize = fsize

                    input_sandbox_uris.append(
                        idx['remote'][os.path.basename(f)])

                    input_sandbox_names.append(
                        os.path.basename(urlparse(f)[2]))

                if idx['local']:
                    input_sandbox_uris += idx['local']
                    input_sandbox_names.append(os.path.basename(f))

        if not ick:
            logger.error('stop job submission')
            return None

        # determin the lcg-cp timeout according to the max_prestaged_fsize
        # - using the assumption of 1 MB/sec.
        max_prestaged_fsize = 0
        lfc_host = ''
        transfer_timeout = config['SandboxTransferTimeout']
        predict_timeout = int(math.ceil(max_prestaged_fsize / 1000000.0))

        if predict_timeout > transfer_timeout:
            transfer_timeout = predict_timeout

        if transfer_timeout < 60:
            transfer_timeout = 60

        script = script.replace(
            '###TRANSFERTIMEOUT###', '%d' % transfer_timeout)

        # update the job wrapper with the inputsandbox list
        script = script.replace(
            '###INPUTSANDBOX###', repr({'remote': {}, 'local': input_sandbox_names}))

        # write out the job wrapper and put job wrapper into job's inputsandbox
        scriptPath = inpw.writefile(
            FileBuffer('__jobscript_%s__' % job.getFQID('.'), script), executable=1)
        input_sandbox = input_sandbox_uris + [scriptPath]

        for isb in input_sandbox:
            logger.debug('ISB URI: %s' % isb)

        # compose output sandbox to include by default the following files:
        # - gzipped stdout (transferred only when the JobLogHandler is WMS)
        # - gzipped stderr (transferred only when the JobLogHandler is WMS)
        # - __jobscript__.log (job wrapper's log)
        output_sandbox = [wrapperlog]

        from Ganga.GPIDev.Lib.File.OutputFileManager import getOutputSandboxPatterns
        for outputSandboxPattern in getOutputSandboxPatterns(job):
            output_sandbox.append(outputSandboxPattern)

        if config['JobLogHandler'] in ['WMS']:
            output_sandbox += ['stdout.gz', 'stderr.gz']

        if len(jobconfig.outputbox):
            output_sandbox += [Sandbox.OUTPUT_TARBALL_NAME]

        # compose ARC XRSL
        xrsl = {
            #'VirtualOrganisation' : config['VirtualOrganisation'],
            'executable': os.path.basename(scriptPath),
            'environment': {'GANGA_LCG_VO': config['VirtualOrganisation'], 'GANGA_LOG_HANDLER': config['JobLogHandler'], 'LFC_HOST': lfc_host},
            #'stdout'                : 'stdout',
            #'stderr'                : 'stderr',
            'inputFiles': input_sandbox,
            'outputFiles': output_sandbox,
            #'OutputSandboxBaseDestURI': 'gsiftp://localhost'
        }

        xrsl['environment'].update({'GANGA_LCG_CE': self.CE})
        #xrsl['Requirements'] = self.requirements.merge(jobconfig.requirements).convert()

        # if self.jobtype.upper() in ['NORMAL','MPICH']:
        #xrsl['JobType'] = self.jobtype.upper()
        # if self.jobtype.upper() == 'MPICH':
        #xrsl['Requirements'].append('(other.GlueCEInfoTotalCPUs >= NodeNumber)')
        # xrsl['Requirements'].append('Member("MPICH",other.GlueHostApplicationSoftwareRunTimeEnvironment)')
        #xrsl['NodeNumber'] = self.requirements.nodenumber
        # else:
        #    logger.warning('JobType "%s" not supported' % self.jobtype)
        #    return

#       additional settings from the job
        if jobconfig.env:
            xrsl['environment'].update(jobconfig.env)

        xrslText = Grid.expandxrsl(xrsl)
        
        # append any additional requirements from the requirements object
        xrslText += '\n'.join(self.requirements.other)

        logger.debug('subjob XRSL: %s' % xrslText)
        return inpw.writefile(FileBuffer('__xrslfile__', xrslText))
Ejemplo n.º 9
0
    def preparejob(self, jobconfig, master_input_sandbox):

        job = self.getJobObject()
        # print str(job.backend_output_postprocess)
        mon = job.getMonitoringService()
        import Ganga.Core.Sandbox as Sandbox

        subjob_input_sandbox = job.createPackedInputSandbox(
            jobconfig.getSandboxFiles() + Sandbox.getGangaModulesAsSandboxFiles(Sandbox.getDefaultModules())
        )

        appscriptpath = [jobconfig.getExeString()] + jobconfig.getArgStrings()
        if self.nice:
            appscriptpath = ["nice", "-n %d" % self.nice] + appscriptpath
        if self.nice < 0:
            logger.warning("increasing process priority is often not allowed, your job may fail due to this")

        sharedoutputpath = job.getOutputWorkspace().getPath()
        ## FIXME DON'T just use the blind list here, request the list of files to be in the output from a method.
        outputpatterns = jobconfig.outputbox
        environment = dict() if jobconfig.env is None else jobconfig.env

        import tempfile

        workdir = tempfile.mkdtemp(dir=config["location"])

        import inspect

        script_location = os.path.join(
            os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))), "LocalHostExec.py"
        )

        from Ganga.GPIDev.Lib.File import FileUtils

        script = FileUtils.loadScript(script_location, "")

        script = script.replace("###INLINEMODULES###", inspect.getsource(Sandbox.WNSandbox))

        from Ganga.GPIDev.Lib.File.OutputFileManager import (
            getWNCodeForOutputSandbox,
            getWNCodeForOutputPostprocessing,
            getWNCodeForDownloadingInputFiles,
            getWNCodeForInputdataListCreation,
        )
        from Ganga.Utility.Config import getConfig

        jobidRepr = repr(job.getFQID("."))

        script = script.replace(
            "###OUTPUTSANDBOXPOSTPROCESSING###",
            getWNCodeForOutputSandbox(job, ["stdout", "stderr", "__syslog__"], jobidRepr),
        )
        script = script.replace("###OUTPUTUPLOADSPOSTPROCESSING###", getWNCodeForOutputPostprocessing(job, ""))
        script = script.replace("###DOWNLOADINPUTFILES###", getWNCodeForDownloadingInputFiles(job, ""))
        script = script.replace("###CREATEINPUTDATALIST###", getWNCodeForInputdataListCreation(job, ""))

        script = script.replace("###APPLICATION_NAME###", repr(getName(job.application)))
        script = script.replace("###INPUT_SANDBOX###", repr(subjob_input_sandbox + master_input_sandbox))
        script = script.replace("###SHAREDOUTPUTPATH###", repr(sharedoutputpath))
        script = script.replace("###APPSCRIPTPATH###", repr(appscriptpath))
        script = script.replace("###OUTPUTPATTERNS###", str(outputpatterns))
        script = script.replace("###JOBID###", jobidRepr)
        script = script.replace("###ENVIRONMENT###", repr(environment))
        script = script.replace("###WORKDIR###", repr(workdir))
        script = script.replace("###INPUT_DIR###", repr(job.getStringInputDir()))

        self.workdir = workdir

        script = script.replace("###GANGADIR###", repr(getConfig("System")["GANGA_PYTHONPATH"]))

        wrkspace = job.getInputWorkspace()
        scriptPath = wrkspace.writefile(FileBuffer("__jobscript__", script), executable=1)

        return scriptPath
Ejemplo n.º 10
0
    def preparejob(self, jobconfig, master_input_sandbox):

        job = self.getJobObject()
        mon = job.getMonitoringService()
        import Ganga.Core.Sandbox as Sandbox

        subjob_input_sandbox = job.createPackedInputSandbox(
            jobconfig.getSandboxFiles()
            + Sandbox.getGangaModulesAsSandboxFiles(Sandbox.getDefaultModules())
            + Sandbox.getGangaModulesAsSandboxFiles(mon.getSandboxModules())
        )

        appscriptpath = [jobconfig.getExeString()] + jobconfig.getArgStrings()
        sharedoutputpath = job.getOutputWorkspace().getPath()
        outputpatterns = jobconfig.outputbox
        environment = jobconfig.env

        text = """#!/usr/bin/env python
import shutil
import os
import time
import popen2
import glob

############################################################################################

###INLINEMODULES###
###INLINEHOSTNAMEFUNCTION###

############################################################################################

input_sandbox = ###INPUT_SANDBOX###
sharedoutputpath = ###SHAREDOUTPUTPATH###
outputpatterns = ###OUTPUTPATTERNS###
appscriptpath = ###APPSCRIPTPATH###
environment = ###ENVIRONMENT###

# jobid is a string
jobid = ###JOBID###

###PREEXECUTE###

def flush_file(f):
   f.flush()
   os.fsync(f.fileno()) #this forces a global flush (cache synchronization on AFS)

def open_file(fname):
  try:
    filehandle=file(fname,'w')
  except IOError,x:
    print 'ERROR: not able to write a status file: ', fname
    print 'ERROR: ',x
    raise
  return filehandle

statusfilename = os.path.join(sharedoutputpath,'__jobstatus__')
heartbeatfilename = os.path.join(sharedoutputpath,'__heartbeat__')

statusfile=open_file(statusfilename)
heartbeatfile=open_file(heartbeatfilename)

line='START: '+ time.strftime('%a %b %d %H:%M:%S %Y',time.gmtime(time.time())) + os.linesep
try:
  line+='PID: ' + os.getenv('###JOBIDNAME###') + os.linesep
  line+='QUEUE: ' + os.getenv('###QUEUENAME###') + os.linesep
  line+='ACTUALCE: ' + hostname() + os.linesep
except:
  pass
statusfile.writelines(line)
flush_file(statusfile)

try:
    import tarfile
except ImportError,x:
    sys.path.insert(0,###TARFILE_PYTHONPATH###)
    import tarfile

# -- WARNING: get the input files including the python modules BEFORE sys.path.insert()
# -- SINCE PYTHON 2.6 THERE WAS A SUBTLE CHANGE OF SEMANTICS IN THIS AREA

for f in input_sandbox:
  getPackedInputSandbox(f)

# -- END OF MOVED CODE BLOCK

import sys
sys.path.insert(0, ###GANGADIR###)
sys.path.insert(0,os.path.join(os.getcwd(),PYTHON_DIR))

try:
    import subprocess
except ImportError,x:
    sys.path.insert(0,###SUBPROCESS_PYTHONPATH###)
    import subprocess

for key,value in environment.iteritems():
    os.environ[key] = value

sysout2 = os.dup(sys.stdout.fileno())
syserr2 = os.dup(sys.stderr.fileno())

print >>sys.stdout,"--- GANGA APPLICATION OUTPUT BEGIN ---"
print >>sys.stderr,"--- GANGA APPLICATION ERROR BEGIN ---"
flush_file(sys.stdout)
flush_file(sys.stderr)

sys.stdout=file('./__syslog__','w')
sys.stderr=sys.stdout

###MONITORING_SERVICE###
monitor = createMonitoringObject()
monitor.start()

result = 255



try:
  child = subprocess.Popen(appscriptpath, shell=False, stdout=sysout2, stderr=syserr2)

  while 1:
    result = child.poll()
    if result is not None:
        break
    monitor.progress()
    heartbeatfile.write('.')
    flush_file(heartbeatfile)
    time.sleep(###HEARTBEATFREQUENCE###)
except Exception,x:
  print 'ERROR: %s'%str(x)

monitor.progress()
flush_file(sys.stdout)
flush_file(sys.stderr)
sys.stdout=sys.__stdout__
sys.stderr=sys.__stderr__
print >>sys.stdout,"--- GANGA APPLICATION OUTPUT END ---"


monitor.stop(result)

try:
    filefilter
except:
    filefilter = None

from Ganga.Utility.files import multi_glob, recursive_copy

createOutputSandbox(outputpatterns,filefilter,sharedoutputpath)

def printError(message):
    print >>sys.stderr, message

def printInfo(message):
    print >>sys.stdout, message

###OUTPUTUPLOADSPOSTPROCESSING###

print >>sys.stderr,"--- GANGA APPLICATION ERROR END ---"

###OUTPUTSANDBOXPOSTPROCESSING###

###POSTEXECUTE###

line='EXITCODE: ' + repr(result) + os.linesep
line+='STOP: '+time.strftime('%a %b %d %H:%M:%S %Y',time.gmtime(time.time())) + os.linesep
statusfile.writelines(line)

statusfile.close()
heartbeatfile.close()
os.unlink(heartbeatfilename)

sys.exit(result)
"""

        import inspect
        import Ganga.Core.Sandbox as Sandbox
        import Ganga.Utility as Utility
        from Ganga.Utility.Config import getConfig
        from Ganga.GPIDev.Lib.File.OutputFileManager import getWNCodeForOutputSandbox, getWNCodeForOutputPostprocessing

        jobidRepr = repr(self.getJobObject().getFQID("."))
        text = text.replace(
            "###OUTPUTSANDBOXPOSTPROCESSING###",
            getWNCodeForOutputSandbox(
                job, ["__syslog__", getConfig("Output")["PostProcessLocationsFileName"]], jobidRepr
            ),
        )

        text = text.replace("###OUTPUTUPLOADSPOSTPROCESSING###", getWNCodeForOutputPostprocessing(job, ""))

        text = text.replace("###INLINEMODULES###", inspect.getsource(Sandbox.WNSandbox))
        text = text.replace("###INLINEHOSTNAMEFUNCTION###", inspect.getsource(Utility.util.hostname))
        text = text.replace("###APPSCRIPTPATH###", repr(appscriptpath))
        # text = text.replace('###SHAREDINPUTPATH###',repr(sharedinputpath))

        logger.debug("subjob input sandbox %s ", subjob_input_sandbox)
        logger.debug("master input sandbox %s ", master_input_sandbox)

        text = text.replace("###INPUT_SANDBOX###", repr(subjob_input_sandbox + master_input_sandbox))
        text = text.replace("###SHAREDOUTPUTPATH###", repr(sharedoutputpath))

        text = text.replace("###OUTPUTPATTERNS###", repr(outputpatterns))
        text = text.replace("###JOBID###", jobidRepr)
        text = text.replace("###ENVIRONMENT###", repr(environment))
        text = text.replace("###PREEXECUTE###", self.config["preexecute"])
        text = text.replace("###POSTEXECUTE###", self.config["postexecute"])
        text = text.replace("###JOBIDNAME###", self.config["jobid_name"])
        text = text.replace("###QUEUENAME###", self.config["queue_name"])
        text = text.replace("###HEARTBEATFREQUENCE###", self.config["heartbeat_frequency"])
        text = text.replace("###INPUT_DIR###", repr(job.getStringInputDir()))

        text = text.replace("###MONITORING_SERVICE###", job.getMonitoringService().getWrapperScriptConstructorText())

        text = text.replace("###GANGADIR###", repr(getConfig("System")["GANGA_PYTHONPATH"]))

        import Ganga.PACKAGE

        text = text.replace(
            "###SUBPROCESS_PYTHONPATH###",
            repr(Ganga.PACKAGE.setup.getPackagePath2("subprocess", "syspath", force=True)),
        )
        text = text.replace(
            "###TARFILE_PYTHONPATH###", repr(Ganga.PACKAGE.setup.getPackagePath2("tarfile", "syspath", force=True))
        )

        from Ganga.GPIDev.Lib.File import FileBuffer

        return job.getInputWorkspace().writefile(FileBuffer("__jobscript__", text), executable=1)
Ejemplo n.º 11
0
    def prestagesandbox(self, jobconfig):

        job = self.getJobObject()
        mon = job.getMonitoringService()
        inw = job.getInputWorkspace()

        sandbox_files = jobconfig.getSandboxFiles()

        logger.info("pre-staging files for saga job with id: %s", job.id)

        ## Compression is ENABLED. All input sandbox files are archived
        ## into an uncompressed tarball which is then  transfered to the
        ## execution host. This speeds up things a lot in many scenarios.
        ##
        if self.enable_compression:
            logger.info("  * adding %s user defined files to input sandbox",
                        len(sandbox_files))

            import Ganga.Core.Sandbox as Sandbox

            compressed_input_sandbox = job.createPackedInputSandbox(
                jobconfig.getSandboxFiles() +
                Sandbox.getGangaModulesAsSandboxFiles(
                    Sandbox.getDefaultModules()))

            try:
                for f in compressed_input_sandbox:
                    archive_url = saga.url("file://localhost/" + f)
                    logger.info(
                        "  * copying %s -> %s ", archive_url.url,
                        self.filesystem_url + "/" + self.workdir_uuid + "/")
                    sf = saga.filesystem.file(archive_url.url)
                    sf.copy(
                        self.filesystem_url + "/" + self.workdir_uuid + "/",
                        saga.filesystem.Overwrite)

            except saga.exception as e:
                logger.error('exception caught while transfering file: %s',
                             e.get_all_exceptions())
                job.updateStatus("failed")
                return False

        ## Compression is DISABLED. All input sandbox files are transfered
        ## one-by-one to the execution host. This can be very slow, especially
        ## if SAGA is using the Globus GridFTP adaptor.
        else:
            logger.info("prestaging %s input sandbox files",
                        len(sandbox_files))
            for f in sandbox_files:
                try:
                    source = saga.url(f.name)
                    if (len(source.scheme) == 0):
                        source.scheme = "file"
                    if (len(source.host) == 0):
                        source.host = "localhost"

                    if (f.subdir != '.'):  # create subdirectory & copy
                        target = saga.url(self.filesystem_url + "/" +
                                          self.workdir_uuid + "/" + f.subdir +
                                          "/")

                        sd = saga.filesystem.directory(target,
                                                       saga.filesystem.Create)
                        sf = saga.filesystem.file(source)

                        logger.info("  * copying %s -> %s ", source, target)
                        sf.copy(target,
                                saga.filesystem.Overwrite)  # copy the file

                    else:  # copy to remote work dir
                        logger.info(
                            "  * copying %s -> %s ", source,
                            self.filesystem_url + "/" + self.workdir_uuid +
                            "/")
                        sf = saga.filesystem.file(source)
                        sf.copy(
                            self.filesystem_url + "/" + self.workdir_uuid +
                            "/", saga.filesystem.Overwrite)

                except saga.exception as e:
                    logger.error('exception caught while transfering file: %s',
                                 e.get_all_exceptions())
                    job.updateStatus("failed")
                    return False

        ##
        ## copy the job script
        ##
        try:
            jobscript_path = saga.url("file://localhost/" + inw.getPath() +
                                      "/__jobscript__")
            if os.path.exists(jobscript_path.path):
                logger.info(
                    "  * copying %s -> %s ", jobscript_path.url,
                    self.filesystem_url + "/" + self.workdir_uuid + "/")
                sf = saga.filesystem.file(jobscript_path.url)
                sf.copy(self.filesystem_url + "/" + self.workdir_uuid + "/",
                        saga.filesystem.Overwrite)

        except saga.exception as e:
            logger.error('exception caught while transfering file: %s',
                         e.get_all_exceptions())
            job.updateStatus("failed")
            return False

        return True
Ejemplo n.º 12
0
class SAGASubmitter(Submitter):

    def submit_worker(self,j):
        import Ganga.GPI
        """This function uses Ganga.GPI to submit the job created by the submitter script."""
        j.application = Ganga.GPI.Executable() 
        wf = tempfile.NamedTemporaryFile(mode="w",prefix='diane_worker_wrapper-')
        
        # if diane dir is not explicitly given, default to 
        # the master wrapper
        if(self.options.dianedir == ''):
            wf.write(self.wrapper)

        # if it is given, use a modified wrapper script
        else:
            sh_run_wrapper = '''#!/bin/sh
                                $('''+self.options.dianedir+'''/bin/diane-env)
                                diane-worker-start $*
                             '''
 
            wf.write(sh_run_wrapper)

        wf.flush()
        uuid_path = tempfile.mkdtemp()
        uuid = open(os.path.join(uuid_path, 'ganga_job_uuid'), 'w')
        uuid.write(str(j.info.uuid))
        uuid.flush()
        uuid.close()
        diane.util.chmod_executable(wf.name)
        j.application.exe = "/bin/sh" #Ganga.GPI.File(wf.name)
        j.application.args=[Ganga.GPI.File(wf.name)]
        j.application.args+=['--ior-file=MasterOID']
        if self.enable_GSI:
            j.application.args+=['--enable-GSI']

        j.application.args+=['--omniorb-config-file=%s'%os.path.basename(self.config_file.name)]

        # Point the worker agent to the working directory
        workdir = saga.url(j.backend.filesystem_url).path+"/"
        j.application.args += ['--workdir=%s'%workdir]


        from Ganga.Utility.util import hostname

        ganga_config = Ganga.GPI.config
        
        # works only for local repository type
        j.application.env = {'GANGA_JOB_ID': "%s@%s:%s:%d"%(ganga_config.Configuration.user,ganga_config.System.GANGA_HOSTNAME, ganga_config.Configuration.gangadir, j.id),
                             'GANGA_JOB_UUID': str(j.info.uuid)}
        
        j.inputsandbox = j.inputsandbox + [Ganga.GPI.File(self.master_oid_filename),
                                           Ganga.GPI.File(self.config_file.name),
                                           Ganga.GPI.File(uuid.name)]
        j.name = "SAGA/DIANEWorkerAgent"

        if self.options.ENABLE_DOCTOR:
            j.outputsandbox = j.outputsandbox + ['*DUMMY_PLATFORM_STRING*']

        #try:
            self.runfile.call('worker_submit',j)
        #except DianeException:
        #    pass

        ###############
        ### EMULATION OF WRAPPER SCRIPT

        ganga_wrapper = """
try:
    import subprocess
except ImportError,x:
    sys.path.insert(0,###SUBPROCESS_PYTHONPATH###)
    import subprocess

appscriptpath = ###APPSCRIPTPATH###

###MONITORING_SERVICE###

monitor = createMonitoringObject()
monitor.start()

import subprocess

try:
 child = subprocess.Popen(appscriptpath, shell=False)
except OSError,x:
 print 'EXITCODE: %d'%-9999
 print 'PROBLEM STARTING THE APPLICATION SCRIPT: %s %s'%(appscriptpath,str(x))
 sys.exit()
 
result = -1

try:
  while 1:
    result = child.poll()
    if result is not None:
        break
    monitor.progress()
    time.sleep(0.3)
finally:
    monitor.progress()

monitor.stop(result)
"""        

      job = j._impl
      mon = job.getMonitoringService()

      import Ganga.Core.Sandbox as Sandbox
      import Ganga.PACKAGE
      j.inputsandbox += Sandbox.getGangaModulesAsSandboxFiles(Sandbox.getDefaultModules())
      j.inputsandbox += Sandbox.getGangaModulesAsSandboxFiles(mon.getSandboxModules()))

      ganga_wrapper = ganga_wrapper.replace('###APPSCRIPTPATH###',repr([j.application.exe,wf.name]+j.application.args[1:])) #HACK SPECIFIC TO THIS SCRIPT
      #ganga_wrapper = ganga_wrapper.replace('###APPSCRIPTPATH###',repr([j.application.exe]+j.args)) # NEED TO CONVERT ALL ARGS TO STRINGS (SMARTLY)
      ganga_wrapper = ganga_wrapper.replace('###JOBID###',repr(job.getFQID('.')))
      ganga_wrapper = ganga_wrapper.replace('###MONITORING_SERVICE###',job.getMonitoringService().getWrapperScriptConstructorText())
      import Ganga.PACKAGE
      ganga_wrapper = ganga_wrapper.replace('###SUBPROCESS_PYTHONPATH###',repr(Ganga.PACKAGE.setup.getPackagePath2('subprocess','syspath',force=True)))


      gwf = tempfile.NamedTemporaryFile(mode="w",prefix='ganga-emulated-wrapper-')
      gwf.write(ganga_wrapper)
      gwf.close()
   
      # replace the job args
      j.application.exe='/usr/bin/python'
      j.application.args=[Ganga.GPI.File(gwf.name)]             
      j.submit()
Ejemplo n.º 13
0
    def preparejob(self, jobconfig, master_input_sandbox):

        job = self.getJobObject()
        mon = job.getMonitoringService()
        import Ganga.Core.Sandbox as Sandbox
        subjob_input_sandbox = job.createPackedInputSandbox(
            jobconfig.getSandboxFiles() +
            Sandbox.getGangaModulesAsSandboxFiles(Sandbox.getDefaultModules()))

        appscriptpath = [jobconfig.getExeString()] + jobconfig.getArgStrings()
        sharedoutputpath = job.getOutputWorkspace().getPath()
        ## FIXME Check this isn't a GangaList
        outputpatterns = jobconfig.outputbox
        environment = jobconfig.env if not jobconfig.env is None else {}

        import inspect
        script_location = os.path.join(
            os.path.dirname(
                os.path.abspath(inspect.getfile(inspect.currentframe()))),
            'BatchScriptTemplate.py')

        from Ganga.GPIDev.Lib.File import FileUtils
        text = FileUtils.loadScript(script_location, '')

        import Ganga.Core.Sandbox as Sandbox
        import Ganga.Utility as Utility
        from Ganga.Utility.Config import getConfig
        from Ganga.GPIDev.Lib.File.OutputFileManager import getWNCodeForOutputSandbox, getWNCodeForOutputPostprocessing, getWNCodeForDownloadingInputFiles
        jobidRepr = repr(self.getJobObject().getFQID('.'))

        replace_dict = {
            '###OUTPUTSANDBOXPOSTPROCESSING###':
            getWNCodeForOutputSandbox(job, ['__syslog__'], jobidRepr),
            '###OUTPUTUPLOADSPOSTPROCESSING###':
            getWNCodeForOutputPostprocessing(job, ''),
            '###DOWNLOADINPUTFILES###':
            getWNCodeForDownloadingInputFiles(job, ''),
            '###INLINEMODULES###':
            inspect.getsource(Sandbox.WNSandbox),
            '###INLINEHOSTNAMEFUNCTION###':
            inspect.getsource(Utility.util.hostname),
            '###APPSCRIPTPATH###':
            repr(appscriptpath),
            #'###SHAREDINPUTPATH###' : repr(sharedinputpath)),
            '###INPUT_SANDBOX###':
            repr(subjob_input_sandbox + master_input_sandbox),
            '###SHAREDOUTPUTPATH###':
            repr(sharedoutputpath),
            '###OUTPUTPATTERNS###':
            repr(outputpatterns),
            '###JOBID###':
            jobidRepr,
            '###ENVIRONMENT###':
            repr(environment),
            '###PREEXECUTE###':
            self.config['preexecute'],
            '###POSTEXECUTE###':
            self.config['postexecute'],
            '###JOBIDNAME###':
            self.config['jobid_name'],
            '###QUEUENAME###':
            self.config['queue_name'],
            '###HEARTBEATFREQUENCE###':
            self.config['heartbeat_frequency'],
            '###INPUT_DIR###':
            repr(job.getStringInputDir()),
            '###GANGADIR###':
            repr(getConfig('System')['GANGA_PYTHONPATH'])
        }

        for k, v in replace_dict.iteritems():
            text = text.replace(str(k), str(v))

        logger.debug('subjob input sandbox %s ', subjob_input_sandbox)
        logger.debug('master input sandbox %s ', master_input_sandbox)

        from Ganga.GPIDev.Lib.File import FileBuffer

        return job.getInputWorkspace().writefile(FileBuffer(
            '__jobscript__', text),
                                                 executable=1)
Ejemplo n.º 14
0
    def preparejob(self, jobconfig, master_job_sandbox):
        """Prepare the JDL"""

        script = self.__jobWrapperTemplate__()

        job = self.getJobObject()
        inpw = job.getInputWorkspace()

        wrapperlog = "__jobscript__.log"

        import Ganga.Core.Sandbox as Sandbox

        # FIXME: check what happens if 'stdout','stderr' are specified here
        script = script.replace("###OUTPUTSANDBOX###", repr(jobconfig.outputbox))

        script = script.replace("###APPLICATION_NAME###", job.application._name)
        script = script.replace("###APPLICATIONEXEC###", repr(jobconfig.getExeString()))
        script = script.replace("###APPLICATIONARGS###", repr(jobconfig.getArguments()))

        from Ganga.GPIDev.Lib.File.OutputFileManager import (
            getWNCodeForOutputPostprocessing,
            getWNCodeForDownloadingInputFiles,
        )

        script = script.replace("###OUTPUTUPLOADSPOSTPROCESSING###", getWNCodeForOutputPostprocessing(job, "    "))

        script = script.replace("###DOWNLOADINPUTFILES###", getWNCodeForDownloadingInputFiles(job, "    "))

        if jobconfig.env:
            script = script.replace("###APPLICATIONENVS###", repr(jobconfig.env))
        else:
            script = script.replace("###APPLICATIONENVS###", repr({}))

        script = script.replace("###WRAPPERLOG###", repr(wrapperlog))
        import inspect

        script = script.replace("###INLINEMODULES###", inspect.getsource(Sandbox.WNSandbox))

        mon = job.getMonitoringService()

        self.monInfo = None

        # set the monitoring file by default to the stdout
        if isinstance(self.monInfo, dict):
            self.monInfo["remotefile"] = "stdout"

        # try to print out the monitoring service information in debug mode
        try:
            logger.debug("job info of monitoring service: %s" % str(self.monInfo))
        except:
            pass

        script = script.replace("###MONITORING_SERVICE###", mon.getWrapperScriptConstructorText())

        #       prepare input/output sandboxes
        packed_files = (
            jobconfig.getSandboxFiles()
            + Sandbox.getGangaModulesAsSandboxFiles(Sandbox.getDefaultModules())
            + Sandbox.getGangaModulesAsSandboxFiles(mon.getSandboxModules())
        )
        sandbox_files = job.createPackedInputSandbox(packed_files)

        # sandbox of child jobs should include master's sandbox
        sandbox_files.extend(master_job_sandbox)

        # check the input file size and pre-upload larger inputs to the iocache
        lfc_host = ""

        input_sandbox_uris = []
        input_sandbox_names = []

        ick = True

        max_prestaged_fsize = 0
        for f in sandbox_files:

            idx = self.__check_and_prestage_inputfile__(f)

            if not idx:
                logger.error("input sandbox preparation failed: %s" % f)
                ick = False
                break
            else:

                if idx["lfc_host"]:
                    lfc_host = idx["lfc_host"]

                if idx["remote"]:
                    abspath = os.path.abspath(f)
                    fsize = os.path.getsize(abspath)

                    if fsize > max_prestaged_fsize:
                        max_prestaged_fsize = fsize

                    input_sandbox_uris.append(idx["remote"][os.path.basename(f)])

                    input_sandbox_names.append(os.path.basename(urlparse(f)[2]))

                if idx["local"]:
                    input_sandbox_uris += idx["local"]
                    input_sandbox_names.append(os.path.basename(f))

        if not ick:
            logger.error("stop job submission")
            return None

        # determin the lcg-cp timeout according to the max_prestaged_fsize
        # - using the assumption of 1 MB/sec.
        max_prestaged_fsize = 0
        lfc_host = ""
        transfer_timeout = config["SandboxTransferTimeout"]
        predict_timeout = int(math.ceil(max_prestaged_fsize / 1000000.0))

        if predict_timeout > transfer_timeout:
            transfer_timeout = predict_timeout

        if transfer_timeout < 60:
            transfer_timeout = 60

        script = script.replace("###TRANSFERTIMEOUT###", "%d" % transfer_timeout)

        # update the job wrapper with the inputsandbox list
        script = script.replace("###INPUTSANDBOX###", repr({"remote": {}, "local": input_sandbox_names}))

        # write out the job wrapper and put job wrapper into job's inputsandbox
        scriptPath = inpw.writefile(FileBuffer("__jobscript_%s__" % job.getFQID("."), script), executable=1)
        input_sandbox = input_sandbox_uris + [scriptPath]

        for isb in input_sandbox:
            logger.debug("ISB URI: %s" % isb)

        # compose output sandbox to include by default the following files:
        # - gzipped stdout (transferred only when the JobLogHandler is WMS)
        # - gzipped stderr (transferred only when the JobLogHandler is WMS)
        # - __jobscript__.log (job wrapper's log)
        output_sandbox = [wrapperlog]

        from Ganga.GPIDev.Lib.File.OutputFileManager import getOutputSandboxPatterns

        for outputSandboxPattern in getOutputSandboxPatterns(job):
            output_sandbox.append(outputSandboxPattern)

        if config["JobLogHandler"] in ["WMS"]:
            output_sandbox += ["stdout.gz", "stderr.gz"]

        if len(jobconfig.outputbox):
            output_sandbox += [Sandbox.OUTPUT_TARBALL_NAME]

        # compose LCG JDL
        jdl = {
            "VirtualOrganisation": config["VirtualOrganisation"],
            "Executable": os.path.basename(scriptPath),
            "Environment": {
                "GANGA_LCG_VO": config["VirtualOrganisation"],
                "GANGA_LOG_HANDLER": config["JobLogHandler"],
                "LFC_HOST": lfc_host,
            },
            "StdOutput": "stdout",
            "StdError": "stderr",
            "InputSandbox": input_sandbox,
            "OutputSandbox": output_sandbox,
            "OutputSandboxBaseDestURI": "gsiftp://localhost",
        }

        jdl["Environment"].update({"GANGA_LCG_CE": self.CE})
        jdl["Requirements"] = self.requirements.merge(jobconfig.requirements).convert()

        if self.jobtype.upper() in ["NORMAL", "MPICH"]:
            jdl["JobType"] = self.jobtype.upper()
            if self.jobtype.upper() == "MPICH":
                # jdl['Requirements'].append('(other.GlueCEInfoTotalCPUs >= NodeNumber)')
                jdl["Requirements"].append('Member("MPICH",other.GlueHostApplicationSoftwareRunTimeEnvironment)')
                jdl["NodeNumber"] = self.requirements.nodenumber
        else:
            logger.warning('JobType "%s" not supported' % self.jobtype)
            return

        #       additional settings from the job
        #        if jobconfig.env:
        #            jdl['Environment'].update(jobconfig.env)

        jdlText = Grid.expandjdl(jdl)
        logger.debug("subjob JDL: %s" % jdlText)
        return inpw.writefile(FileBuffer("__jdlfile__", jdlText))
Ejemplo n.º 15
0
    def preparejob(self,jobconfig,master_input_sandbox):

      job = self.getJobObject()
      #print str(job.backend_output_postprocess)        
      mon = job.getMonitoringService()
      import Ganga.Core.Sandbox as Sandbox
      subjob_input_sandbox = job.createPackedInputSandbox(jobconfig.getSandboxFiles()
        + Sandbox.getGangaModulesAsSandboxFiles(Sandbox.getDefaultModules())
        + Sandbox.getGangaModulesAsSandboxFiles(mon.getSandboxModules()))
      
      appscriptpath = [jobconfig.getExeString()]+jobconfig.getArgStrings()
      if self.nice:
          appscriptpath = ['nice','-n %d'%self.nice] + appscriptpath
      if self.nice < 0:
          logger.warning('increasing process priority is often not allowed, your job may fail due to this')
          
      sharedoutputpath=job.getOutputWorkspace().getPath()
      outputpatterns=jobconfig.outputbox
      environment=jobconfig.env

      from Ganga.Utility import tempfile
      workdir = tempfile.mkdtemp()
      
      script= """#!/usr/bin/env python

import os,os.path,shutil,tempfile
import sys,time
import glob
import sys

# FIXME: print as DEBUG: to __syslog__ file
#print sys.path
#print os.environ['PATH']
#print sys.version

# bugfix #13314 : make sure that the wrapper (spawned process) is detached from Ganga session
# the process will not receive Control-C signals
# using fork  and doing setsid() before  exec would probably  be a bit
# better (to avoid  slim chance that the signal  is propagated to this
# process before setsid is reached)
# this is only enabled if the first argument is 'subprocess' in order to enable
# running this script by hand from outside ganga (which is sometimes useful)
if len(sys.argv)>1 and sys.argv[1] == 'subprocess':
 os.setsid()

############################################################################################

###INLINEMODULES###

############################################################################################

input_sandbox = ###INPUT_SANDBOX###
sharedoutputpath= ###SHAREDOUTPUTPATH###
outputpatterns = ###OUTPUTPATTERNS###
appscriptpath = ###APPSCRIPTPATH###
environment = ###ENVIRONMENT###
workdir = ###WORKDIR###

statusfilename = os.path.join(sharedoutputpath,'__jobstatus__')

try:
  statusfile=file(statusfilename,'w')
except IOError,x:
  print 'ERROR: not able to write a status file: ', statusfilename
  print 'ERROR: ',x
  raise
  
line='START: '+ time.strftime('%a %b %d %H:%M:%S %Y',time.gmtime(time.time())) + os.linesep
statusfile.writelines(line)
statusfile.flush()

os.chdir(workdir)

# -- WARNING: get the input files including the python modules BEFORE sys.path.insert()
# -- SINCE PYTHON 2.6 THERE WAS A SUBTLE CHANGE OF SEMANTICS IN THIS AREA

for f in input_sandbox:
  getPackedInputSandbox(f)

# -- END OF MOVED CODE BLOCK

import sys
sys.path.insert(0, ###GANGADIR###)
sys.path.insert(0,os.path.join(os.getcwd(),PYTHON_DIR))

try:
    import subprocess
except ImportError,x:
    sys.path.insert(0,###SUBPROCESS_PYTHONPATH###)
    import subprocess
try:
    import tarfile
except ImportError,x:
    sys.path.insert(0,###TARFILE_PYTHONPATH###)
    import tarfile


for key,value in environment.iteritems():
    os.environ[key] = value

outfile=file('stdout','w')
errorfile=file('stderr','w')

sys.stdout=file('./__syslog__','w')
sys.stderr=sys.stdout

###MONITORING_SERVICE###
monitor = createMonitoringObject()
monitor.start()

import subprocess

import time #datetime #disabled for python2.2 compatiblity

try:
 child = subprocess.Popen(appscriptpath, shell=False, stdout=outfile, stderr=errorfile)
except OSError,x:
 file('tt','w').close()
 print >> statusfile, 'EXITCODE: %d'%-9999
 print >> statusfile, 'FAILED: %s'%time.strftime('%a %b %d %H:%M:%S %Y') #datetime.datetime.utcnow().strftime('%a %b %d %H:%M:%S %Y')
 print >> statusfile, 'PROBLEM STARTING THE APPLICATION SCRIPT: %s %s'%(appscriptpath,str(x))
 statusfile.close()
 sys.exit()
 
print >> statusfile, 'PID: %d'%child.pid
statusfile.flush()

result = -1

try:
  while 1:
    result = child.poll()
    if result is not None:
        break
    outfile.flush()
    errorfile.flush()
    monitor.progress()
    time.sleep(0.3)
finally:
    monitor.progress()
    sys.stdout=sys.__stdout__
    sys.stderr=sys.__stderr__

monitor.stop(result)

outfile.flush()
errorfile.flush()

createOutputSandbox(outputpatterns,None,sharedoutputpath)

def printError(message):
    errorfile.write(message + os.linesep)
    errorfile.flush()  

def printInfo(message):
    outfile.write(message + os.linesep)
    outfile.flush()  


###OUTPUTUPLOADSPOSTPROCESSING###

outfile.close()
errorfile.close()

###OUTPUTSANDBOXPOSTPROCESSING###

line="EXITCODE: " + repr(result) + os.linesep
line+='STOP: '+time.strftime('%a %b %d %H:%M:%S %Y',time.gmtime(time.time())) + os.linesep
statusfile.writelines(line)
sys.exit()

"""

      import inspect
      script = script.replace('###INLINEMODULES###',inspect.getsource(Sandbox.WNSandbox))

      from Ganga.GPIDev.Lib.File.OutputFileManager import getWNCodeForOutputSandbox, getWNCodeForOutputPostprocessing
      from Ganga.Utility.Config import getConfig
      jobidRepr = repr(job.getFQID('.'))
      script = script.replace('###OUTPUTSANDBOXPOSTPROCESSING###',getWNCodeForOutputSandbox(job, ['stdout', 'stderr', '__syslog__', getConfig('Output')['PostProcessLocationsFileName']], jobidRepr))

      script = script.replace('###OUTPUTUPLOADSPOSTPROCESSING###',getWNCodeForOutputPostprocessing(job, ''))

      script = script.replace('###APPLICATION_NAME###',repr(job.application._name))
      script = script.replace('###INPUT_SANDBOX###',repr(subjob_input_sandbox+master_input_sandbox))
      script = script.replace('###SHAREDOUTPUTPATH###',repr(sharedoutputpath))
      script = script.replace('###APPSCRIPTPATH###',repr(appscriptpath))
      script = script.replace('###OUTPUTPATTERNS###',repr(outputpatterns))
      script = script.replace('###JOBID###',jobidRepr)
      script = script.replace('###ENVIRONMENT###',repr(environment))
      script = script.replace('###WORKDIR###',repr(workdir))
      script = script.replace('###INPUT_DIR###',repr(job.getStringInputDir()))
      
      script = script.replace('###MONITORING_SERVICE###',job.getMonitoringService().getWrapperScriptConstructorText())

      self.workdir = workdir

      script = script.replace('###GANGADIR###',repr(getConfig('System')['GANGA_PYTHONPATH']))

      import Ganga.PACKAGE
      script = script.replace('###SUBPROCESS_PYTHONPATH###',repr(Ganga.PACKAGE.setup.getPackagePath2('subprocess','syspath',force=True)))
      script = script.replace('###TARFILE_PYTHONPATH###',repr(Ganga.PACKAGE.setup.getPackagePath2('tarfile','syspath',force=True)))             

      return job.getInputWorkspace().writefile(FileBuffer('__jobscript__',script),executable=1)