def reportRebounce( self, job ): """ __reportRebounce__ """ logging.info("Job %s : report rebounce" % self.fullId(job) ) # loading task task = self.bossLiteSession.loadTask( job['taskId'], deep=False ) # fwjr name reportName = 'crab_fjr_' + str(job['jobId']) + '.xml' # remote source name outputDirectory = task['outputDirectory'] out = self.ft.match( outputDirectory ) if out is not None : outputDirectory = outputDirectory[out.end()-1:] source = os.path.join( outputDirectory, reportName ) # local destination name dest = os.path.join( job.runningJob['outputDirectory'], reportName ) # initialize tranfer protocol seEl = SElement( self.configs["storageName"], \ self.configs["Protocol"], \ self.configs["storagePort"] ) loc = SElement("localhost", "local") sbi = SBinterface( seEl, loc ) credential = task['user_proxy'] if self.configs["Protocol"].upper() == 'RFIO': username = task['name'].split('_')[0] credential = '%s::%s' % (username, credential) # transfer fwjr try: logging.debug( 'Job %s REBOUNCE DBG : %s, %s' % \ (self.fullId(job), source, dest) ) # try not to transfer via gridftp if os.access (source, os.R_OK) : copy (source, dest) else: sbi.copy( source, dest, credential ) except Exception, e: logging.info("Job %s Report rebounce transfer fail : %s " \ % ( self.fullId(job), str(e) ) )
def rebounceLoggingInfo( self, job ): """ __rebounceLoggingInfo__ """ logging.info("Job %s : loggingInfo.log rebounce" % self.fullId(job) ) localOutDir = job.runningJob['outputDirectory'] source = os.path.join( localOutDir, 'loggingInfo.log' ) if os.path.exists( source ): task = self.bossLiteSession.loadTask( job['taskId'], deep=False ) seEl = SElement( self.configs["storageName"], \ self.configs["Protocol"], \ self.configs["storagePort"] ) loc = SElement("localhost", "local") ### copy ISB ### sbi = SBinterface( loc, seEl ) # remote source name outputDirectory = task['outputDirectory'] out = self.ft.match( outputDirectory ) if out is not None : outputDirectory = outputDirectory[out.end()-1:] dest = os.path.join( outputDirectory, 'loggingInfo_' + str(job['jobId']) + '.log' ) credential = task['user_proxy'] if self.configs["Protocol"].upper() == 'RFIO': username = task['name'].split('_')[0] credential = '%s::%s' % (username, credential) try: logging.info( 'Job %s : REBOUNCE DBG %s, %s' % \ (self.fullId(job), source, dest) ) sbi.copy( source, dest, credential) #filesToClean.append(source) except Exception, e: logging.error( "Job %s : loggingInfo.log rebounce transfer fail: %s " \ % ( self.fullId(job), str(e) ) ) logging.info("Job %s : loggingInfo.log rebounce completed" \ % self.fullId(job) )
def rebounceOSB( self, job ): """ __rebounceOSB__ """ logging.info("Job %s : Output rebounce" % self.fullId(job) ) localOutDir = job.runningJob['outputDirectory'] localOutputTgz = [ localOutDir +'/'+ f.split('/')[-1] for f in job['outputFiles'] if '.tgz' in f ] localOutputTgz = [ f for f in localOutputTgz if os.path.exists(f) ] logging.info( 'Job %s : REBOUNCE DBG %s, %s, %s' \ % (self.fullId(job), localOutDir, localOutputTgz, \ [ localOutDir +'/'+ f.split('/')[-1] for f in job['outputFiles'] ] ) ) if len(localOutputTgz)==0: return task = self.bossLiteSession.loadTask( job['taskId'], deep=False ) seEl = SElement( self.configs["storageName"], \ self.configs["Protocol"], \ self.configs["storagePort"] ) loc = SElement("localhost", "local") ## copy OSB ## sbi = SBinterface( loc, seEl ) filesToClean = [] for filetocopy in localOutputTgz: source = os.path.abspath(filetocopy) dest = os.path.join( task['outputDirectory'], os.path.basename(filetocopy) ) try: ## logging.info( 'REBOUNCE DBG %s, %s'%(source, dest) ) sbi.copy( source, dest, task['user_proxy']) filesToClean.append(source) except Exception, e: logging.info("Job %s : Output rebounce transfer fail: %s " \ % ( self.fullId(job), str(e) ) ) continue
sourcesList = [] destsList = [] for i in xrange(len(osbFiles)): sourcesList.append(osbFiles[i]) destsList.append(destFiles[i]) #if i < len(filesToRetrieve): # filesAndJodId[ filesToRetrieve[i] ] = osbFiles[i] # construct logging information toCopy = "\n".join([t[0] + " to " + t[1] for t in map(None, sourcesList, destsList)]) + "\n" common.logger.debug("Retrieving:\n " + toCopy) # try to do the copy copy_res = None try: copy_res = sbi.copy( sourcesList, destsList, opt="tout=300") except Exception, ex: msg = "WARNING: Unable to retrieve output file %s \n" % osbFiles[i] msg += str(ex) common.logger.debug(msg) import traceback common.logger.debug( str(traceback.format_exc()) ) if copy_res is not None: ## evaluating copy results copy_err_list = [] count = 0 for ll in map(None, copy_res, sourcesList): exitcode = int(ll[0][0]) if exitcode == 0: filesAndJodId[ filesToRetrieve[count] ] = osbFiles[count] else:
sourcesList = [] destsList = [] self.taskuuid = str(common._db.queryTask('name')) common.logger.debug("Starting globus retrieval for task name: " + self.taskuuid) remotedir = os.path.join(self.storage_path, self.taskuuid) for i in logginable: remotelog = remotedir + '/loggingInfo_' + str(i) + '.log' sourcesList.append(remotelog) fname = self.fname_base + str(i) + '.LoggingInfo' destsList.append(fname) # try to do the copy copy_res = None try: copy_res = sbi.copy(sourcesList, destsList, opt=self.copyTout) except Exception, ex: msg = "WARNING: Unable to retrieve logging" #ainfo file %s \n" % osbFiles[i] msg += str(ex) common.logger.debug(msg) import traceback common.logger.debug(str(traceback.format_exc())) if copy_res is not None: ## evaluating copy results copy_err_list = [] count = 0 for ll in map(None, copy_res, sourcesList): exitcode = int(ll[0][0]) if exitcode == 0: ## decode logging info fl = open(destsList[count], 'r')
# and the destinations to copy them to sourcesList = [] destsList = [] self.taskuuid = str(common._db.queryTask('name')) common.logger.debug( "Starting globus retrieval for task name: " + self.taskuuid) remotedir = os.path.join(self.storage_path, self.taskuuid) for i in logginable: remotelog = remotedir + '/loggingInfo_'+str(i)+'.log' sourcesList.append(remotelog) fname = self.fname_base + str(i) + '.LoggingInfo' destsList.append(fname) # try to do the copy copy_res = None try: copy_res = sbi.copy( sourcesList, destsList, opt=self.copyTout) except Exception, ex: msg = "WARNING: Unable to retrieve logging" #ainfo file %s \n" % osbFiles[i] msg += str(ex) common.logger.debug(msg) import traceback common.logger.debug( str(traceback.format_exc()) ) if copy_res is not None: ## evaluating copy results copy_err_list = [] count = 0 for ll in map(None, copy_res, sourcesList): exitcode = int(ll[0][0]) if exitcode == 0: ## decode logging info fl = open(destsList[count], 'r')
for i in xrange(len(osbFiles)): sourcesList.append(osbFiles[i]) destsList.append(destFiles[i]) #if i < len(filesToRetrieve): # filesAndJodId[ filesToRetrieve[i] ] = osbFiles[i] # construct logging information toCopy = "\n".join([ t[0] + " to " + t[1] for t in map(None, sourcesList, destsList) ]) + "\n" common.logger.debug("Retrieving:\n " + toCopy) # try to do the copy copy_res = None try: copy_res = sbi.copy(sourcesList, destsList, opt="tout=300") except Exception, ex: msg = "WARNING: Unable to retrieve output file %s \n" % osbFiles[ i] msg += str(ex) common.logger.debug(msg) import traceback common.logger.debug(str(traceback.format_exc())) if copy_res is not None: ## evaluating copy results copy_err_list = [] count = 0 for ll in map(None, copy_res, sourcesList): exitcode = int(ll[0][0]) if exitcode == 0: filesAndJodId[filesToRetrieve[count]] = osbFiles[count]