def reportRebounce( self, job ): """ __reportRebounce__ """ logging.info("Job %s : report rebounce" % self.fullId(job) ) # loading task task = self.bossLiteSession.loadTask( job['taskId'], deep=False ) # fwjr name reportName = 'crab_fjr_' + str(job['jobId']) + '.xml' # remote source name outputDirectory = task['outputDirectory'] out = self.ft.match( outputDirectory ) if out is not None : outputDirectory = outputDirectory[out.end()-1:] source = os.path.join( outputDirectory, reportName ) # local destination name dest = os.path.join( job.runningJob['outputDirectory'], reportName ) # initialize tranfer protocol seEl = SElement( self.configs["storageName"], \ self.configs["Protocol"], \ self.configs["storagePort"] ) loc = SElement("localhost", "local") sbi = SBinterface( seEl, loc ) credential = task['user_proxy'] if self.configs["Protocol"].upper() == 'RFIO': username = task['name'].split('_')[0] credential = '%s::%s' % (username, credential) # transfer fwjr try: logging.debug( 'Job %s REBOUNCE DBG : %s, %s' % \ (self.fullId(job), source, dest) ) # try not to transfer via gridftp if os.access (source, os.R_OK) : copy (source, dest) else: sbi.copy( source, dest, credential ) except Exception, e: logging.info("Job %s Report rebounce transfer fail : %s " \ % ( self.fullId(job), str(e) ) )
def rebounceLoggingInfo( self, job ): """ __rebounceLoggingInfo__ """ logging.info("Job %s : loggingInfo.log rebounce" % self.fullId(job) ) localOutDir = job.runningJob['outputDirectory'] source = os.path.join( localOutDir, 'loggingInfo.log' ) if os.path.exists( source ): task = self.bossLiteSession.loadTask( job['taskId'], deep=False ) seEl = SElement( self.configs["storageName"], \ self.configs["Protocol"], \ self.configs["storagePort"] ) loc = SElement("localhost", "local") ### copy ISB ### sbi = SBinterface( loc, seEl ) # remote source name outputDirectory = task['outputDirectory'] out = self.ft.match( outputDirectory ) if out is not None : outputDirectory = outputDirectory[out.end()-1:] dest = os.path.join( outputDirectory, 'loggingInfo_' + str(job['jobId']) + '.log' ) credential = task['user_proxy'] if self.configs["Protocol"].upper() == 'RFIO': username = task['name'].split('_')[0] credential = '%s::%s' % (username, credential) try: logging.info( 'Job %s : REBOUNCE DBG %s, %s' % \ (self.fullId(job), source, dest) ) sbi.copy( source, dest, credential) #filesToClean.append(source) except Exception, e: logging.error( "Job %s : loggingInfo.log rebounce transfer fail: %s " \ % ( self.fullId(job), str(e) ) ) logging.info("Job %s : loggingInfo.log rebounce completed" \ % self.fullId(job) )
def rebounceOSB( self, job ): """ __rebounceOSB__ """ logging.info("Job %s : Output rebounce" % self.fullId(job) ) localOutDir = job.runningJob['outputDirectory'] localOutputTgz = [ localOutDir +'/'+ f.split('/')[-1] for f in job['outputFiles'] if '.tgz' in f ] localOutputTgz = [ f for f in localOutputTgz if os.path.exists(f) ] logging.info( 'Job %s : REBOUNCE DBG %s, %s, %s' \ % (self.fullId(job), localOutDir, localOutputTgz, \ [ localOutDir +'/'+ f.split('/')[-1] for f in job['outputFiles'] ] ) ) if len(localOutputTgz)==0: return task = self.bossLiteSession.loadTask( job['taskId'], deep=False ) seEl = SElement( self.configs["storageName"], \ self.configs["Protocol"], \ self.configs["storagePort"] ) loc = SElement("localhost", "local") ## copy OSB ## sbi = SBinterface( loc, seEl ) filesToClean = [] for filetocopy in localOutputTgz: source = os.path.abspath(filetocopy) dest = os.path.join( task['outputDirectory'], os.path.basename(filetocopy) ) try: ## logging.info( 'REBOUNCE DBG %s, %s'%(source, dest) ) sbi.copy( source, dest, task['user_proxy']) filesToClean.append(source) except Exception, e: logging.info("Job %s : Output rebounce transfer fail: %s " \ % ( self.fullId(job), str(e) ) ) continue
try: seEl = SElement(self.storage_name, self.storage_proto, self.storage_port) except Exception, ex: common.logger.debug( str(ex)) msg = "ERROR : Unable to create SE source interface \n" raise CrabException(msg) try: loc = SElement("localhost", "local") except Exception, ex: common.logger.debug( str(ex)) msg = "ERROR : Unable to create destination interface \n" raise CrabException(msg) ## copy ISB ## sbi = SBinterface(seEl, loc, logger = common.logger.logger) filesAndJodId = {} if self.storage_proto in ['globus']: # construct a list of absolute paths of input files # and the destinations to copy them to sourcesList = [] destsList = [] for i in xrange(len(osbFiles)): sourcesList.append(osbFiles[i]) destsList.append(destFiles[i]) #if i < len(filesToRetrieve): # filesAndJodId[ filesToRetrieve[i] ] = osbFiles[i] # construct logging information
try: seEl = SElement(self.storage_name, self.storage_proto, self.storage_port) except Exception, ex: common.logger.debug(str(ex)) msg = "ERROR: Unable to create SE source interface \n" raise CrabException(msg) try: loc = SElement("localhost", "local") except Exception, ex: common.logger.debug(str(ex)) msg = "ERROR: Unable to create destination interface \n" raise CrabException(msg) ## coupling se interfaces sbi = SBinterface(seEl, loc, logger=common.logger.logger) ## get the list of jobs to get logging.info skimmed by failed status logginable = self.skimDeadList() if self.storage_proto in ['globus']: for id in self.nj_list: if id not in self.all_jobs: common.logger.info( 'Warning: job # ' + str(id) + ' does not exist! Not possible to ask for postMortem ') elif id not in logginable: common.logger.info( 'Warning: job # ' + str(id) + ' not killed or aborted! Will get loggingInfo manually ' )
try: loc = SElement("localhost", "local") except Exception, ex: common.logger.debug(str(ex)) msg = "ERROR : Unable to create SE source interface \n" msg +="Project "+ self.taskuuid +" not Submitted \n" raise CrabException(msg) ### it should not be there... To move into SE API. DS # create remote dir for gsiftp if self.storage_proto in ['gridftp','rfio','uberftp']: try: action = SBinterface(seEl, logger = common.logger.logger) action.createDir( self.remotedir ) except AlreadyExistsException, ex: msg = "Project %s already exist on the Storage Element \n"%self.taskuuid msg +='\t%s'%str(ex) common.logger.debug(msg) except OperationException, ex: common.logger.debug(str(ex.detail)) msg = "ERROR: Unable to create project destination on the Storage Element %s\n"%str(ex) msg +="Project "+ self.taskuuid +" not Submitted \n" raise CrabException(msg) except AuthorizationException, ex: common.logger.debug(str(ex.detail)) msg = "ERROR: Unable to create project destination on the Storage Element: %s\n"%str(ex) msg +="Project "+ self.taskuuid +" not Submitted \n" raise CrabException(msg)
raise CrabException(msg) try: loc = SElement("localhost", "local") except Exception, ex: common.logger.debug(str(ex)) msg = "ERROR : Unable to create SE source interface \n" msg += "Project " + self.taskuuid + " not Submitted \n" raise CrabException(msg) ### it should not be there... To move into SE API. DS # create remote dir for gsiftp if self.storage_proto in ['gridftp', 'rfio', 'uberftp']: try: action = SBinterface(seEl, logger=common.logger.logger) action.createDir(self.remotedir) except AlreadyExistsException, ex: msg = "Project %s already exist on the Storage Element \n" % self.taskuuid msg += '\t%s' % str(ex) common.logger.debug(msg) except OperationException, ex: common.logger.debug(str(ex.detail)) msg = "ERROR: Unable to create project destination on the Storage Element %s\n" % str( ex) msg += "Project " + self.taskuuid + " not Submitted \n" raise CrabException(msg) except AuthorizationException, ex: common.logger.debug(str(ex.detail)) msg = "ERROR: Unable to create project destination on the Storage Element: %s\n" % str( ex)
loc = None try: seEl = SElement(self.storage_name, self.storage_proto, self.storage_port) except Exception, ex: common.logger.debug( str(ex)) msg = "ERROR: Unable to create SE source interface \n" raise CrabException(msg) try: loc = SElement("localhost", "local") except Exception, ex: common.logger.debug( str(ex)) msg = "ERROR: Unable to create destination interface \n" raise CrabException(msg) ## coupling se interfaces sbi = SBinterface(seEl, loc, logger = common.logger.logger) ## get the list of jobs to get logging.info skimmed by failed status logginable = self.skimDeadList() if self.storage_proto in ['globus']: for id in self.nj_list: if id not in self.all_jobs: common.logger.info('Warning: job # ' + str(id) + ' does not exist! Not possible to ask for postMortem ') elif id not in logginable: common.logger.info('Warning: job # ' + str(id) + ' not killed or aborted! Will get loggingInfo manually ') PostMortem.collectOneLogging(self,id) # construct a list of absolute paths of input files # and the destinations to copy them to sourcesList = [] destsList = []
try: seEl = SElement(self.storage_name, self.storage_proto, self.storage_port) except Exception, ex: common.logger.debug(str(ex)) msg = "ERROR : Unable to create SE source interface \n" raise CrabException(msg) try: loc = SElement("localhost", "local") except Exception, ex: common.logger.debug(str(ex)) msg = "ERROR : Unable to create destination interface \n" raise CrabException(msg) ## copy ISB ## sbi = SBinterface(seEl, loc, logger=common.logger.logger) filesAndJodId = {} if self.storage_proto in ['globus']: # construct a list of absolute paths of input files # and the destinations to copy them to sourcesList = [] destsList = [] for i in xrange(len(osbFiles)): sourcesList.append(osbFiles[i]) destsList.append(destFiles[i]) #if i < len(filesToRetrieve): # filesAndJodId[ filesToRetrieve[i] ] = osbFiles[i] # construct logging information