Ejemplo n.º 1
0
    def master_resubmit(self, rjobs):
        '''Resubmit the master job to the grid'''

        profiler = ElapsedTimeProfiler(getLogger(name='Profile.LCG'))
        profiler.start()

        job = self.getJobObject()

        ick = False

        if not job.master and len(job.subjobs) == 0:
            # case 1: master job normal resubmission
            logger.debug('rjobs: %s' % str(rjobs))
            logger.debug('mode: master job normal resubmission')
            ick = IBackend.master_resubmit(self, rjobs)

        elif job.master:
            # case 2: individual subjob resubmission
            logger.debug('mode: individual subjob resubmission')
            ick = IBackend.master_resubmit(self, rjobs)

        else:
            # case 3: master job bulk resubmission
            logger.debug('mode: master job resubmission')

            ick = self.master_bulk_resubmit(rjobs)
            if not ick:
                raise GangaException('ARC bulk submission failure')

        profiler.check('job re-submission elapsed time')

        return ick
Ejemplo n.º 2
0
def get_md5sum(fname, ignoreGzipTimestamp=False):
    ''' Calculates the MD5 checksum of a file '''

    profiler = ElapsedTimeProfiler(getLogger(name='Profile.LCG'))
    profiler.start()

    # if the file is a zipped format (determined by extension),
    # try to get checksum from it's content. The reason is that
    # gzip file contains a timestamp in the header, which causes
    # different md5sum value even the contents are the same.
    #re_gzipfile = re.compile('.*[\.tgz|\.gz].*$')

    f = None

    if ignoreGzipTimestamp and (fname.find('.tgz') > 0
                                or fname.find('.gz') > 0):
        f = gzip.open(fname, 'rb')
    else:
        f = open(fname, 'rb')

    m = hashlib.md5()

    while True:
        d = f.read(8096)
        if not d:
            break
        m.update(d)
    f.close()

    md5sum = m.hexdigest()

    profiler.check('md5sum calculation time')

    return md5sum
Ejemplo n.º 3
0
def get_md5sum(fname, ignoreGzipTimestamp=False):
    ''' Calculates the MD5 checksum of a file '''

    profiler = ElapsedTimeProfiler(getLogger(name='Profile.LCG'))
    profiler.start()


    ## if the file is a zipped format (determined by extension),
    ## try to get checksum from it's content. The reason is that
    ## gzip file contains a timestamp in the header, which causes
    ## different md5sum value even the contents are the same.
    #re_gzipfile = re.compile('.*[\.tgz|\.gz].*$')

    f = None

    if ignoreGzipTimestamp and (fname.find('.tgz') > 0 or fname.find('.gz') > 0):
        f = gzip.open(fname,'rb')
    else:
        f = open(fname, 'rb')

    m = get_md5_obj()

    while True:
        d = f.read(8096)
        if not d:
            break
        m.update(d)
    f.close()

    md5sum = m.hexdigest()

    profiler.check('md5sum calculation time')

    return md5sum
Ejemplo n.º 4
0
    def master_resubmit(self, rjobs):
        '''Resubmit the master job to the grid'''

        profiler = ElapsedTimeProfiler(getLogger(name='Profile.LCG'))
        profiler.start()

        job = self.getJobObject()

        ick = False

        if not job.master and len(job.subjobs) == 0:
            # case 1: master job normal resubmission
            logger.debug('rjobs: %s' % str(rjobs))
            logger.debug('mode: master job normal resubmission')
            ick = IBackend.master_resubmit(self, rjobs)

        elif job.master:
            # case 2: individual subjob resubmission
            logger.debug('mode: individual subjob resubmission')
            ick = IBackend.master_resubmit(self, rjobs)

        else:
            # case 3: master job bulk resubmission
            logger.debug('mode: master job resubmission')

            ick = self.master_bulk_resubmit(rjobs)
            if not ick:
                raise GangaException('ARC bulk submission failure')

        profiler.check('job re-submission elapsed time')

        return ick
Ejemplo n.º 5
0
    def master_submit(self, rjobs, subjobconfigs, masterjobconfig):
        '''Submit the master job to the grid'''

        profiler = ElapsedTimeProfiler(getLogger(name='Profile.LCG'))
        profiler.start()

        job = self.getJobObject()

        # finding ARC CE endpoint for job submission
        #allowed_celist = []
        # try:
        #    allowed_celist = self.requirements.getce()
        #    if not self.CE and allowed_celist:
        #        self.CE = allowed_celist[0]
        # except:
        #    logger.warning('ARC CE assigment from ARCRequirements failed.')

        # if self.CE and allowed_celist:
        #    if self.CE not in allowed_celist:
        #        logger.warning('submission to CE not allowed: %s, use %s instead' % ( self.CE, allowed_celist[0] ) )
        #        self.CE = allowed_celist[0]

        # use arc info to check for any endpoints recorded in the config file
        rc, output = grids['GLITE'].arc_info()

        if not self.CE and rc != 0:
            raise GangaException(
                "ARC CE endpoint not set and no default settings in '%s'. " % config['ArcConfigFile'])
        elif self.CE:
            logger.info('ARC CE endpoint set to: ' + str(self.CE))
        else:
            logger.info("Using ARC CE endpoints defined in '%s'" %
                        config['ArcConfigFile'])

        # delegate proxy to ARC CE
        # if not grids['GLITE'].arc_proxy_delegation(self.CE):
        #    logger.warning('proxy delegation to %s failed' % self.CE)

        # doing massive job preparation
        if len(job.subjobs) == 0:
            ick = IBackend.master_submit(
                self, rjobs, subjobconfigs, masterjobconfig)
        else:
            ick = self.master_bulk_submit(
                rjobs, subjobconfigs, masterjobconfig)

        profiler.check('==> master_submit() elapsed time')

        return ick
Ejemplo n.º 6
0
    def master_submit(self, rjobs, subjobconfigs, masterjobconfig):
        '''Submit the master job to the grid'''

        profiler = ElapsedTimeProfiler(getLogger(name='Profile.LCG'))
        profiler.start()

        job = self.getJobObject()

        # finding CREAM CE endpoint for job submission
        allowed_celist = []
        try:
            allowed_celist = self.requirements.getce()
            if not self.CE and allowed_celist:
                self.CE = allowed_celist[0]
        except:
            logger.warning(
                'CREAM CE assigment from AtlasCREAMRequirements failed.')

        if self.CE and allowed_celist:
            if self.CE not in allowed_celist:
                logger.warning(
                    'submission to CE not allowed: %s, use %s instead' %
                    (self.CE, allowed_celist[0]))
                self.CE = allowed_celist[0]

        if not self.CE:
            raise GangaException('CREAM CE endpoint not set')

        # delegate proxy to CREAM CE
        self.delegation_id = Grid.cream_proxy_delegation(
            self.CE, self.delegation_id, self.credential_requirements)
        if not self.delegation_id:
            logger.warning('proxy delegation to %s failed' % self.CE)

        # doing massive job preparation
        if len(job.subjobs) == 0:
            ick = IBackend.master_submit(self, rjobs, subjobconfigs,
                                         masterjobconfig)
        else:
            ick = self.master_bulk_submit(rjobs, subjobconfigs,
                                          masterjobconfig)

        profiler.check('==> master_submit() elapsed time')

        return ick
Ejemplo n.º 7
0
    def master_submit(self, rjobs, subjobconfigs, masterjobconfig):
        '''Submit the master job to the grid'''

        profiler = ElapsedTimeProfiler(getLogger(name='Profile.LCG'))
        profiler.start()

        job = self.getJobObject()

        # finding CREAM CE endpoint for job submission
        allowed_celist = []
        try:
            allowed_celist = self.requirements.getce()
            if not self.CE and allowed_celist:
                self.CE = allowed_celist[0]
        except:
            logger.warning(
                'CREAM CE assigment from AtlasCREAMRequirements failed.')

        if self.CE and allowed_celist:
            if self.CE not in allowed_celist:
                logger.warning('submission to CE not allowed: %s, use %s instead' % (
                    self.CE, allowed_celist[0]))
                self.CE = allowed_celist[0]

        if not self.CE:
            raise GangaException('CREAM CE endpoint not set')

        # delegate proxy to CREAM CE
        self.delegation_id = Grid.cream_proxy_delegation(self.CE, self.delegation_id)
        if not self.delegation_id:
            logger.warning('proxy delegation to %s failed' % self.CE)

        # doing massive job preparation
        if len(job.subjobs) == 0:
            ick = IBackend.master_submit(
                self, rjobs, subjobconfigs, masterjobconfig)
        else:
            ick = self.master_bulk_submit(
                rjobs, subjobconfigs, masterjobconfig)

        profiler.check('==> master_submit() elapsed time')

        return ick
Ejemplo n.º 8
0
    def master_resubmit(self, rjobs):
        '''Resubmit the master job to the grid'''

        profiler = ElapsedTimeProfiler(getLogger(name='Profile.LCG'))
        profiler.start()

        job = self.getJobObject()

        ick = False

        # delegate proxy to CREAM CE
        self.delegation_id = Grid.cream_proxy_delegation(
            self.CE, self.delegation_id, self.credential_requirements)
        if not self.delegation_id:
            logger.warning('proxy delegation to %s failed' % self.CE)

        if not job.master and len(job.subjobs) == 0:
            # case 1: master job normal resubmission
            logger.debug('rjobs: %s' % str(rjobs))
            logger.debug('mode: master job normal resubmission')
            ick = IBackend.master_resubmit(self, rjobs)

        elif job.master:
            # case 2: individual subjob resubmission
            logger.debug('mode: individual subjob resubmission')
            ick = IBackend.master_resubmit(self, rjobs)

        else:
            # case 3: master job bulk resubmission
            logger.debug('mode: master job resubmission')

            ick = self.master_bulk_resubmit(rjobs)
            if not ick:
                raise GangaException('CREAM bulk submission failure')

        profiler.check('job re-submission elapsed time')

        return ick
Ejemplo n.º 9
0
    def master_submit(self, rjobs, subjobconfigs, masterjobconfig):
        """Submit the master job to the grid"""

        profiler = ElapsedTimeProfiler(getLogger(name="Profile.LCG"))
        profiler.start()

        job = self.getJobObject()

        # finding CREAM CE endpoint for job submission
        allowed_celist = []
        try:
            allowed_celist = self.requirements.getce()
            if not self.CE and allowed_celist:
                self.CE = allowed_celist[0]
        except:
            logger.warning("CREAM CE assigment from AtlasCREAMRequirements failed.")

        if self.CE and allowed_celist:
            if self.CE not in allowed_celist:
                logger.warning("submission to CE not allowed: %s, use %s instead" % (self.CE, allowed_celist[0]))
                self.CE = allowed_celist[0]

        if not self.CE:
            raise GangaException("CREAM CE endpoint not set")

        # delegate proxy to CREAM CE
        if not grids["GLITE"].cream_proxy_delegation(self.CE):
            logger.warning("proxy delegation to %s failed" % self.CE)

        # doing massive job preparation
        if len(job.subjobs) == 0:
            ick = IBackend.master_submit(self, rjobs, subjobconfigs, masterjobconfig)
        else:
            ick = self.master_bulk_submit(rjobs, subjobconfigs, masterjobconfig)

        profiler.check("==> master_submit() elapsed time")

        return ick
Ejemplo n.º 10
0
    def master_resubmit(self, rjobs):
        '''Resubmit the master job to the grid'''

        profiler = ElapsedTimeProfiler(getLogger(name='Profile.LCG'))
        profiler.start()

        job = self.getJobObject()

        ick = False

        # delegate proxy to CREAM CE
        self.delegation_id = Grid.cream_proxy_delegation(self.CE, self.delegation_id)
        if not self.delegation_id:
            logger.warning('proxy delegation to %s failed' % self.CE)

        if not job.master and len(job.subjobs) == 0:
            # case 1: master job normal resubmission
            logger.debug('rjobs: %s' % str(rjobs))
            logger.debug('mode: master job normal resubmission')
            ick = IBackend.master_resubmit(self, rjobs)

        elif job.master:
            # case 2: individual subjob resubmission
            logger.debug('mode: individual subjob resubmission')
            ick = IBackend.master_resubmit(self, rjobs)

        else:
            # case 3: master job bulk resubmission
            logger.debug('mode: master job resubmission')

            ick = self.master_bulk_resubmit(rjobs)
            if not ick:
                raise GangaException('CREAM bulk submission failure')

        profiler.check('job re-submission elapsed time')

        return ick
Ejemplo n.º 11
0
    def master_resubmit(self, rjobs):
        """Resubmit the master job to the grid"""

        profiler = ElapsedTimeProfiler(getLogger(name="Profile.LCG"))
        profiler.start()

        job = self.getJobObject()

        ick = False

        # delegate proxy to CREAM CE
        if not grids["GLITE"].cream_proxy_delegation(self.CE):
            logger.warning("proxy delegation to %s failed" % self.CE)

        if not job.master and len(job.subjobs) == 0:
            # case 1: master job normal resubmission
            logger.debug("rjobs: %s" % str(rjobs))
            logger.debug("mode: master job normal resubmission")
            ick = IBackend.master_resubmit(self, rjobs)

        elif job.master:
            # case 2: individual subjob resubmission
            logger.debug("mode: individual subjob resubmission")
            ick = IBackend.master_resubmit(self, rjobs)

        else:
            # case 3: master job bulk resubmission
            logger.debug("mode: master job resubmission")

            ick = self.master_bulk_resubmit(rjobs)
            if not ick:
                raise GangaException("CREAM bulk submission failure")

        profiler.check("job re-submission elapsed time")

        return ick