Ejemplo n.º 1
0
    def reportRebounce( self, job ):
        """
        __reportRebounce__

        """

        logging.info("Job %s : report rebounce" % self.fullId(job) )

        # loading task
        task = self.bossLiteSession.loadTask( job['taskId'], deep=False )

        # fwjr name
        reportName = 'crab_fjr_' + str(job['jobId']) + '.xml'

        # remote source name
        outputDirectory = task['outputDirectory']
        out = self.ft.match( outputDirectory )
        if out is not None :
            outputDirectory = outputDirectory[out.end()-1:]
        source = os.path.join( outputDirectory, reportName )

        # local destination name
        dest = os.path.join( job.runningJob['outputDirectory'], reportName )

        # initialize tranfer protocol
        seEl = SElement( self.configs["storageName"], \
                         self.configs["Protocol"],    \
                         self.configs["storagePort"] )
        loc = SElement("localhost", "local")
        sbi = SBinterface( seEl, loc )
        credential = task['user_proxy'] 
        if self.configs["Protocol"].upper() == 'RFIO':
            username = task['name'].split('_')[0]
            credential = '%s::%s' % (username, credential)

        # transfer fwjr
        try:
            logging.debug( 'Job %s REBOUNCE DBG : %s, %s' % \
                           (self.fullId(job), source, dest) )
#        try not to transfer via gridftp
            if  os.access (source, os.R_OK) :
                copy (source, dest)
            else:
                sbi.copy( source, dest, credential )

        except Exception, e:
            logging.info("Job %s Report rebounce transfer fail : %s " \
                         % ( self.fullId(job), str(e) ) )
Ejemplo n.º 2
0
    def rebounceLoggingInfo( self, job ):
        """
        __rebounceLoggingInfo__

        """

        logging.info("Job %s : loggingInfo.log rebounce" % self.fullId(job) )
        
        localOutDir = job.runningJob['outputDirectory']

        source = os.path.join( localOutDir, 'loggingInfo.log' )
        if os.path.exists( source ):
            task = self.bossLiteSession.loadTask( job['taskId'], deep=False )

            seEl = SElement( self.configs["storageName"], \
                             self.configs["Protocol"],    \
                             self.configs["storagePort"] )
            loc = SElement("localhost", "local")

            ### copy ISB ###
            sbi = SBinterface( loc, seEl )

            # remote source name
            outputDirectory = task['outputDirectory']
            out = self.ft.match( outputDirectory )
            if out is not None :
                outputDirectory = outputDirectory[out.end()-1:]
            dest = os.path.join(
                outputDirectory, 'loggingInfo_' + str(job['jobId']) + '.log' )

            credential = task['user_proxy'] 
            if self.configs["Protocol"].upper() == 'RFIO':
                username = task['name'].split('_')[0]
                credential = '%s::%s' % (username, credential)

            try:
                logging.info( 'Job %s : REBOUNCE DBG %s, %s' % \
                              (self.fullId(job), source, dest) )
                sbi.copy( source, dest, credential)
                #filesToClean.append(source)
            except Exception, e:
                logging.error(
                    "Job %s : loggingInfo.log rebounce transfer fail: %s " \
                             % ( self.fullId(job), str(e) ) )

            logging.info("Job %s : loggingInfo.log rebounce completed" \
                         % self.fullId(job) )
Ejemplo n.º 3
0
    def rebounceOSB( self, job ):
        """
        __rebounceOSB__

        """
         
        logging.info("Job %s : Output rebounce" % self.fullId(job) )

        localOutDir = job.runningJob['outputDirectory']
        localOutputTgz = [ localOutDir +'/'+ f.split('/')[-1]
                           for f in job['outputFiles'] if '.tgz' in f ]
        localOutputTgz = [ f for f in localOutputTgz if os.path.exists(f) ]

        logging.info( 'Job %s : REBOUNCE DBG %s, %s, %s' \
                      % (self.fullId(job), localOutDir, localOutputTgz, \
                         [ localOutDir +'/'+ f.split('/')[-1]
                           for f in job['outputFiles'] ] ) )

        if len(localOutputTgz)==0:
            return   

        task = self.bossLiteSession.loadTask( job['taskId'], deep=False )
        seEl = SElement( self.configs["storageName"], \
                         self.configs["Protocol"],    \
                         self.configs["storagePort"] )
        loc = SElement("localhost", "local")

        ## copy OSB ##
        sbi = SBinterface( loc, seEl )
        filesToClean = []
        for filetocopy in localOutputTgz:
            source = os.path.abspath(filetocopy)
            dest = os.path.join(
                task['outputDirectory'], os.path.basename(filetocopy) )
            try: 
                ## logging.info( 'REBOUNCE DBG %s, %s'%(source, dest) ) 
                sbi.copy( source, dest, task['user_proxy'])
                filesToClean.append(source)
            except Exception, e:
                logging.info("Job %s : Output rebounce transfer fail: %s " \
                             % ( self.fullId(job), str(e) ) )
                continue 
Ejemplo n.º 4
0
            sourcesList = []
            destsList = []
            for i in xrange(len(osbFiles)):
                sourcesList.append(osbFiles[i])
                destsList.append(destFiles[i])
                #if i < len(filesToRetrieve):
                #    filesAndJodId[ filesToRetrieve[i] ] = osbFiles[i]

            # construct logging information
            toCopy = "\n".join([t[0] + " to " + t[1] for t in map(None, sourcesList, destsList)]) + "\n"
            common.logger.debug("Retrieving:\n " + toCopy)

            # try to do the copy
            copy_res = None
            try:
                copy_res = sbi.copy( sourcesList, destsList, opt="tout=300")
            except Exception, ex:
                msg = "WARNING: Unable to retrieve output file %s \n" % osbFiles[i]
                msg += str(ex)
                common.logger.debug(msg)
                import traceback
                common.logger.debug( str(traceback.format_exc()) )
            if copy_res is not None:
                ## evaluating copy results
                copy_err_list = []
                count = 0
                for ll in map(None, copy_res, sourcesList):
                    exitcode = int(ll[0][0])
                    if exitcode == 0:
                        filesAndJodId[ filesToRetrieve[count] ] = osbFiles[count]
                    else:
Ejemplo n.º 5
0
            sourcesList = []
            destsList = []
            self.taskuuid = str(common._db.queryTask('name'))
            common.logger.debug("Starting globus retrieval for task name: " +
                                self.taskuuid)
            remotedir = os.path.join(self.storage_path, self.taskuuid)
            for i in logginable:
                remotelog = remotedir + '/loggingInfo_' + str(i) + '.log'
                sourcesList.append(remotelog)
                fname = self.fname_base + str(i) + '.LoggingInfo'
                destsList.append(fname)

            # try to do the copy
            copy_res = None
            try:
                copy_res = sbi.copy(sourcesList, destsList, opt=self.copyTout)
            except Exception, ex:
                msg = "WARNING: Unable to retrieve logging"  #ainfo file %s \n" % osbFiles[i]
                msg += str(ex)
                common.logger.debug(msg)
                import traceback
                common.logger.debug(str(traceback.format_exc()))
            if copy_res is not None:
                ## evaluating copy results
                copy_err_list = []
                count = 0
                for ll in map(None, copy_res, sourcesList):
                    exitcode = int(ll[0][0])
                    if exitcode == 0:
                        ## decode logging info
                        fl = open(destsList[count], 'r')
Ejemplo n.º 6
0
            # and the destinations to copy them to
            sourcesList = []
            destsList = []
            self.taskuuid = str(common._db.queryTask('name'))
            common.logger.debug( "Starting globus retrieval for task name: " + self.taskuuid)
            remotedir = os.path.join(self.storage_path, self.taskuuid)
            for i in logginable:
                remotelog = remotedir + '/loggingInfo_'+str(i)+'.log'
                sourcesList.append(remotelog)
                fname = self.fname_base + str(i) + '.LoggingInfo'
                destsList.append(fname)

            # try to do the copy
            copy_res = None
            try:
                copy_res = sbi.copy( sourcesList, destsList, opt=self.copyTout)
            except Exception, ex:
                msg = "WARNING: Unable to retrieve logging" #ainfo file %s \n" % osbFiles[i]
                msg += str(ex)
                common.logger.debug(msg)
                import traceback
                common.logger.debug( str(traceback.format_exc()) )
            if copy_res is not None:
                ## evaluating copy results
                copy_err_list = []
                count = 0
                for ll in map(None, copy_res, sourcesList):
                    exitcode = int(ll[0][0])
                    if exitcode == 0:
                        ## decode logging info
                        fl = open(destsList[count], 'r')
Ejemplo n.º 7
0
            for i in xrange(len(osbFiles)):
                sourcesList.append(osbFiles[i])
                destsList.append(destFiles[i])
                #if i < len(filesToRetrieve):
                #    filesAndJodId[ filesToRetrieve[i] ] = osbFiles[i]

            # construct logging information
            toCopy = "\n".join([
                t[0] + " to " + t[1] for t in map(None, sourcesList, destsList)
            ]) + "\n"
            common.logger.debug("Retrieving:\n " + toCopy)

            # try to do the copy
            copy_res = None
            try:
                copy_res = sbi.copy(sourcesList, destsList, opt="tout=300")
            except Exception, ex:
                msg = "WARNING: Unable to retrieve output file %s \n" % osbFiles[
                    i]
                msg += str(ex)
                common.logger.debug(msg)
                import traceback
                common.logger.debug(str(traceback.format_exc()))
            if copy_res is not None:
                ## evaluating copy results
                copy_err_list = []
                count = 0
                for ll in map(None, copy_res, sourcesList):
                    exitcode = int(ll[0][0])
                    if exitcode == 0:
                        filesAndJodId[filesToRetrieve[count]] = osbFiles[count]