Beispiel #1
0
    def reportRebounce( self, job ):
        """
        __reportRebounce__

        """

        logging.info("Job %s : report rebounce" % self.fullId(job) )

        # loading task
        task = self.bossLiteSession.loadTask( job['taskId'], deep=False )

        # fwjr name
        reportName = 'crab_fjr_' + str(job['jobId']) + '.xml'

        # remote source name
        outputDirectory = task['outputDirectory']
        out = self.ft.match( outputDirectory )
        if out is not None :
            outputDirectory = outputDirectory[out.end()-1:]
        source = os.path.join( outputDirectory, reportName )

        # local destination name
        dest = os.path.join( job.runningJob['outputDirectory'], reportName )

        # initialize tranfer protocol
        seEl = SElement( self.configs["storageName"], \
                         self.configs["Protocol"],    \
                         self.configs["storagePort"] )
        loc = SElement("localhost", "local")
        sbi = SBinterface( seEl, loc )
        credential = task['user_proxy'] 
        if self.configs["Protocol"].upper() == 'RFIO':
            username = task['name'].split('_')[0]
            credential = '%s::%s' % (username, credential)

        # transfer fwjr
        try:
            logging.debug( 'Job %s REBOUNCE DBG : %s, %s' % \
                           (self.fullId(job), source, dest) )
#        try not to transfer via gridftp
            if  os.access (source, os.R_OK) :
                copy (source, dest)
            else:
                sbi.copy( source, dest, credential )

        except Exception, e:
            logging.info("Job %s Report rebounce transfer fail : %s " \
                         % ( self.fullId(job), str(e) ) )
Beispiel #2
0
    def rebounceLoggingInfo( self, job ):
        """
        __rebounceLoggingInfo__

        """

        logging.info("Job %s : loggingInfo.log rebounce" % self.fullId(job) )
        
        localOutDir = job.runningJob['outputDirectory']

        source = os.path.join( localOutDir, 'loggingInfo.log' )
        if os.path.exists( source ):
            task = self.bossLiteSession.loadTask( job['taskId'], deep=False )

            seEl = SElement( self.configs["storageName"], \
                             self.configs["Protocol"],    \
                             self.configs["storagePort"] )
            loc = SElement("localhost", "local")

            ### copy ISB ###
            sbi = SBinterface( loc, seEl )

            # remote source name
            outputDirectory = task['outputDirectory']
            out = self.ft.match( outputDirectory )
            if out is not None :
                outputDirectory = outputDirectory[out.end()-1:]
            dest = os.path.join(
                outputDirectory, 'loggingInfo_' + str(job['jobId']) + '.log' )

            credential = task['user_proxy'] 
            if self.configs["Protocol"].upper() == 'RFIO':
                username = task['name'].split('_')[0]
                credential = '%s::%s' % (username, credential)

            try:
                logging.info( 'Job %s : REBOUNCE DBG %s, %s' % \
                              (self.fullId(job), source, dest) )
                sbi.copy( source, dest, credential)
                #filesToClean.append(source)
            except Exception, e:
                logging.error(
                    "Job %s : loggingInfo.log rebounce transfer fail: %s " \
                             % ( self.fullId(job), str(e) ) )

            logging.info("Job %s : loggingInfo.log rebounce completed" \
                         % self.fullId(job) )
Beispiel #3
0
    def rebounceOSB( self, job ):
        """
        __rebounceOSB__

        """
         
        logging.info("Job %s : Output rebounce" % self.fullId(job) )

        localOutDir = job.runningJob['outputDirectory']
        localOutputTgz = [ localOutDir +'/'+ f.split('/')[-1]
                           for f in job['outputFiles'] if '.tgz' in f ]
        localOutputTgz = [ f for f in localOutputTgz if os.path.exists(f) ]

        logging.info( 'Job %s : REBOUNCE DBG %s, %s, %s' \
                      % (self.fullId(job), localOutDir, localOutputTgz, \
                         [ localOutDir +'/'+ f.split('/')[-1]
                           for f in job['outputFiles'] ] ) )

        if len(localOutputTgz)==0:
            return   

        task = self.bossLiteSession.loadTask( job['taskId'], deep=False )
        seEl = SElement( self.configs["storageName"], \
                         self.configs["Protocol"],    \
                         self.configs["storagePort"] )
        loc = SElement("localhost", "local")

        ## copy OSB ##
        sbi = SBinterface( loc, seEl )
        filesToClean = []
        for filetocopy in localOutputTgz:
            source = os.path.abspath(filetocopy)
            dest = os.path.join(
                task['outputDirectory'], os.path.basename(filetocopy) )
            try: 
                ## logging.info( 'REBOUNCE DBG %s, %s'%(source, dest) ) 
                sbi.copy( source, dest, task['user_proxy'])
                filesToClean.append(source)
            except Exception, e:
                logging.info("Job %s : Output rebounce transfer fail: %s " \
                             % ( self.fullId(job), str(e) ) )
                continue 
Beispiel #4
0
        try:
            seEl = SElement(self.storage_name, self.storage_proto, self.storage_port)
        except Exception, ex:
            common.logger.debug( str(ex))
            msg = "ERROR : Unable to create SE source interface \n"
            raise CrabException(msg)
        try:
            loc = SElement("localhost", "local")
        except Exception, ex:
            common.logger.debug( str(ex))
            msg = "ERROR : Unable to create destination  interface \n"
            raise CrabException(msg)

        ## copy ISB ##
        sbi = SBinterface(seEl, loc, logger = common.logger.logger)

        filesAndJodId = {}

        if self.storage_proto in ['globus']:
            # construct a list of absolute paths of input files
            # and the destinations to copy them to
            sourcesList = []
            destsList = []
            for i in xrange(len(osbFiles)):
                sourcesList.append(osbFiles[i])
                destsList.append(destFiles[i])
                #if i < len(filesToRetrieve):
                #    filesAndJodId[ filesToRetrieve[i] ] = osbFiles[i]

            # construct logging information
Beispiel #5
0
        try:
            seEl = SElement(self.storage_name, self.storage_proto,
                            self.storage_port)
        except Exception, ex:
            common.logger.debug(str(ex))
            msg = "ERROR: Unable to create SE source interface \n"
            raise CrabException(msg)
        try:
            loc = SElement("localhost", "local")
        except Exception, ex:
            common.logger.debug(str(ex))
            msg = "ERROR: Unable to create destination interface \n"
            raise CrabException(msg)

        ## coupling se interfaces
        sbi = SBinterface(seEl, loc, logger=common.logger.logger)

        ## get the list of jobs to get logging.info skimmed by failed status
        logginable = self.skimDeadList()

        if self.storage_proto in ['globus']:
            for id in self.nj_list:
                if id not in self.all_jobs:
                    common.logger.info(
                        'Warning: job # ' + str(id) +
                        ' does not exist! Not possible to ask for postMortem ')
                elif id not in logginable:
                    common.logger.info(
                        'Warning: job # ' + str(id) +
                        ' not killed or aborted! Will get loggingInfo manually '
                    )
        try:
            loc = SElement("localhost", "local")
        except Exception, ex:
            common.logger.debug(str(ex))
            msg = "ERROR : Unable to create SE source interface \n"
            msg +="Project "+ self.taskuuid +" not Submitted \n"
            raise CrabException(msg)


        ### it should not be there... To move into SE API. DS

        # create remote dir for gsiftp
        if self.storage_proto in ['gridftp','rfio','uberftp']:
            try:
                action = SBinterface(seEl, logger = common.logger.logger)
                action.createDir( self.remotedir )
            except AlreadyExistsException, ex:
                msg = "Project %s already exist on the Storage Element \n"%self.taskuuid
                msg +='\t%s'%str(ex)
                common.logger.debug(msg)
            except OperationException, ex:
                common.logger.debug(str(ex.detail))
                msg = "ERROR: Unable to create project destination on the Storage Element %s\n"%str(ex)
                msg +="Project "+ self.taskuuid +" not Submitted \n"
                raise CrabException(msg)
            except AuthorizationException, ex:
                common.logger.debug(str(ex.detail))
                msg = "ERROR: Unable to create project destination on the Storage Element: %s\n"%str(ex)
                msg +="Project "+ self.taskuuid +" not Submitted \n"
                raise CrabException(msg)
Beispiel #7
0
            raise CrabException(msg)

        try:
            loc = SElement("localhost", "local")
        except Exception, ex:
            common.logger.debug(str(ex))
            msg = "ERROR : Unable to create SE source interface \n"
            msg += "Project " + self.taskuuid + " not Submitted \n"
            raise CrabException(msg)

        ### it should not be there... To move into SE API. DS

        # create remote dir for gsiftp
        if self.storage_proto in ['gridftp', 'rfio', 'uberftp']:
            try:
                action = SBinterface(seEl, logger=common.logger.logger)
                action.createDir(self.remotedir)
            except AlreadyExistsException, ex:
                msg = "Project %s already exist on the Storage Element \n" % self.taskuuid
                msg += '\t%s' % str(ex)
                common.logger.debug(msg)
            except OperationException, ex:
                common.logger.debug(str(ex.detail))
                msg = "ERROR: Unable to create project destination on the Storage Element %s\n" % str(
                    ex)
                msg += "Project " + self.taskuuid + " not Submitted \n"
                raise CrabException(msg)
            except AuthorizationException, ex:
                common.logger.debug(str(ex.detail))
                msg = "ERROR: Unable to create project destination on the Storage Element: %s\n" % str(
                    ex)
        loc = None
        try:
            seEl = SElement(self.storage_name, self.storage_proto, self.storage_port)
        except Exception, ex:
            common.logger.debug( str(ex))
            msg = "ERROR: Unable to create SE source interface \n"
            raise CrabException(msg)
        try:
            loc = SElement("localhost", "local")
        except Exception, ex:
            common.logger.debug( str(ex))
            msg = "ERROR: Unable to create destination interface \n"
            raise CrabException(msg)

        ## coupling se interfaces
        sbi = SBinterface(seEl, loc, logger = common.logger.logger)

        ## get the list of jobs to get logging.info skimmed by failed status
        logginable = self.skimDeadList()

        if self.storage_proto in ['globus']:
            for id in self.nj_list:
                if id not in self.all_jobs:
                    common.logger.info('Warning: job # ' + str(id) + ' does not exist! Not possible to ask for postMortem ')
                elif id not in logginable:
                    common.logger.info('Warning: job # ' + str(id) + ' not killed or aborted! Will get loggingInfo manually ')
                    PostMortem.collectOneLogging(self,id)
            # construct a list of absolute paths of input files
            # and the destinations to copy them to
            sourcesList = []
            destsList = []
Beispiel #9
0
        try:
            seEl = SElement(self.storage_name, self.storage_proto,
                            self.storage_port)
        except Exception, ex:
            common.logger.debug(str(ex))
            msg = "ERROR : Unable to create SE source interface \n"
            raise CrabException(msg)
        try:
            loc = SElement("localhost", "local")
        except Exception, ex:
            common.logger.debug(str(ex))
            msg = "ERROR : Unable to create destination  interface \n"
            raise CrabException(msg)

        ## copy ISB ##
        sbi = SBinterface(seEl, loc, logger=common.logger.logger)

        filesAndJodId = {}

        if self.storage_proto in ['globus']:
            # construct a list of absolute paths of input files
            # and the destinations to copy them to
            sourcesList = []
            destsList = []
            for i in xrange(len(osbFiles)):
                sourcesList.append(osbFiles[i])
                destsList.append(destFiles[i])
                #if i < len(filesToRetrieve):
                #    filesAndJodId[ filesToRetrieve[i] ] = osbFiles[i]

            # construct logging information