def monitorProxy(self,glexecLocation,pilotProxy,payloadProxy): """ Monitor the payload proxy and renew as necessary. """ if not os.path.exists(pilotProxy): return S_ERROR('Pilot proxy not found at %s' %pilotProxy) if not os.path.exists(payloadProxy): return S_ERROR('Payload proxy not found at %s' %payloadProxy) result = getProxyInfoAsString(payloadProxy) if not result['OK']: self.log.error('Could not get payload proxy info',result) return result self.log.info('Payload proxy information seen from pilot:\n%s' %result['Value']) gProxyManager.renewProxy(minLifeTime=self.minProxyTime, newProxyLifeTime=self.defaultProxyTime, proxyToConnect=pilotProxy) if glexecLocation: self.log.info('Rerunning glexec without arguments to renew payload proxy') result = self.glexecExecute(None,glexecLocation) if not result['OK']: self.log.error(result) else: self.log.info('Running without glexec, checking local proxy') return S_OK('Proxy checked')
def __sendErrorMail( self, rb, name, command, result, proxy ): """ In case or error with RB/WM: - check if RB/WMS still in use - remove RB/WMS from current list - check if RB/WMS not in cache - add RB/WMS to cache - send Error mail """ if rb in self.resourceBrokers: try: self.resourceBrokers.remove( rb ) self.log.info( 'Removed RB from list', rb ) except: pass if not self.__failingWMSCache.exists( rb ): self.__failingWMSCache.add( rb, self.errorClearTime ) # disable for 30 minutes mailAddress = self.errorMailAddress msg = '' if not result['OK']: subject = "%s: timeout executing %s" % ( rb, name ) msg += '\n%s' % result['Message'] elif result['Value'][0] != 0: if re.search( 'the server is temporarily drained', ' '.join( result['Value'][1:3] ) ): return if re.search( 'System load is too high:', ' '.join( result['Value'][1:3] ) ): return subject = "%s: error executing %s" % ( rb, name ) else: return msg += ' '.join( command ) msg += '\nreturns: %s\n' % str( result['Value'][0] ) + '\n'.join( result['Value'][1:3] ) msg += '\nUsing Proxy:\n' + getProxyInfoAsString( proxy )['Value'] #msg += '\nUsing Proxy:\n' + gProxyManager. ticketTime = self.errorClearTime + self.errorTicketTime if self.__ticketsWMSCache.exists( rb ): mailAddress = self.alarmMailAddress # the RB was already detected failing a short time ago msg = 'Submit GGUS Ticket for this error if not already opened\n' + \ 'It has been failing at least for %s hours\n' % ( ticketTime / 60 / 60 ) + msg else: self.__ticketsWMSCache.add( rb, ticketTime ) if mailAddress: result = NotificationClient().sendMail( mailAddress, subject, msg, fromAddress = self.mailFromAddress ) if not result[ 'OK' ]: self.log.error( "Mail could not be sent" ) return
def writeProxyToFile( self, proxy ): """CE helper function to write a CE proxy string to a file. """ result = File.writeToProxyFile( proxy ) if not result[ 'OK' ]: self.log.error( 'Could not write proxy to file', result[ 'Message' ] ) return result proxyLocation = result[ 'Value' ] result = getProxyInfoAsString( proxyLocation ) if not result['OK']: self.log.error( 'Could not get proxy info', result ) return result else: self.log.info( 'Payload proxy information:' ) print result['Value'] return S_OK( proxyLocation )
def monitorProxy(self,pilotProxy,payloadProxy): """ Monitor the payload proxy and renew as necessary. """ if not pilotProxy: return S_OK('Using server Certificate') if not os.path.exists(pilotProxy): return S_ERROR('Pilot proxy not found at %s' %pilotProxy) if not os.path.exists(payloadProxy): return S_ERROR('Payload proxy not found at %s' %payloadProxy) result = getProxyInfoAsString(payloadProxy) if not result['OK']: self.log.error('Could not get payload proxy info',result) return result self.log.verbose('Payload proxy information:\n%s' %result['Value']) gProxyManager.renewProxy(minLifeTime=self.minProxyTime, newProxyLifeTime=self.defaultProxyTime, proxyToConnect=pilotProxy) return S_OK('Proxy checked')