def monitorProxy(self,glexecLocation,pilotProxy,payloadProxy):
""" Monitor the payload proxy and renew as necessary.
"""
if not os.path.exists(pilotProxy):
return S_ERROR('Pilot proxy not found at %s' %pilotProxy)
if not os.path.exists(payloadProxy):
return S_ERROR('Payload proxy not found at %s' %payloadProxy)
result = getProxyInfoAsString(payloadProxy)
if not result['OK']:
self.log.error('Could not get payload proxy info',result)
return result
self.log.info('Payload proxy information seen from pilot:\n%s' %result['Value'])
gProxyManager.renewProxy(minLifeTime=self.minProxyTime,
newProxyLifeTime=self.defaultProxyTime,
proxyToConnect=pilotProxy)
if glexecLocation:
self.log.info('Rerunning glexec without arguments to renew payload proxy')
result = self.glexecExecute(None,glexecLocation)
if not result['OK']:
self.log.error(result)
else:
self.log.info('Running without glexec, checking local proxy')
return S_OK('Proxy checked')
def __sendErrorMail( self, rb, name, command, result, proxy ):
"""
In case or error with RB/WM:
- check if RB/WMS still in use
- remove RB/WMS from current list
- check if RB/WMS not in cache
- add RB/WMS to cache
- send Error mail
"""
if rb in self.resourceBrokers:
try:
self.resourceBrokers.remove( rb )
self.log.info( 'Removed RB from list', rb )
except:
pass
if not self.__failingWMSCache.exists( rb ):
self.__failingWMSCache.add( rb, self.errorClearTime ) # disable for 30 minutes
mailAddress = self.errorMailAddress
msg = ''
if not result['OK']:
subject = "%s: timeout executing %s" % ( rb, name )
msg += '\n%s' % result['Message']
elif result['Value'][0] != 0:
if re.search( 'the server is temporarily drained', ' '.join( result['Value'][1:3] ) ):
return
if re.search( 'System load is too high:', ' '.join( result['Value'][1:3] ) ):
return
subject = "%s: error executing %s" % ( rb, name )
else:
return
msg += ' '.join( command )
msg += '\nreturns: %s\n' % str( result['Value'][0] ) + '\n'.join( result['Value'][1:3] )
msg += '\nUsing Proxy:\n' + getProxyInfoAsString( proxy )['Value']
#msg += '\nUsing Proxy:\n' + gProxyManager.
ticketTime = self.errorClearTime + self.errorTicketTime
if self.__ticketsWMSCache.exists( rb ):
mailAddress = self.alarmMailAddress
# the RB was already detected failing a short time ago
msg = 'Submit GGUS Ticket for this error if not already opened\n' + \
'It has been failing at least for %s hours\n' % ( ticketTime / 60 / 60 ) + msg
else:
self.__ticketsWMSCache.add( rb, ticketTime )
if mailAddress:
result = NotificationClient().sendMail( mailAddress, subject, msg, fromAddress = self.mailFromAddress )
if not result[ 'OK' ]:
self.log.error( "Mail could not be sent" )
return
def writeProxyToFile( self, proxy ):
"""CE helper function to write a CE proxy string to a file.
"""
result = File.writeToProxyFile( proxy )
if not result[ 'OK' ]:
self.log.error( 'Could not write proxy to file', result[ 'Message' ] )
return result
proxyLocation = result[ 'Value' ]
result = getProxyInfoAsString( proxyLocation )
if not result['OK']:
self.log.error( 'Could not get proxy info', result )
return result
else:
self.log.info( 'Payload proxy information:' )
print result['Value']
return S_OK( proxyLocation )
def monitorProxy(self,pilotProxy,payloadProxy):
""" Monitor the payload proxy and renew as necessary.
"""
if not pilotProxy:
return S_OK('Using server Certificate')
if not os.path.exists(pilotProxy):
return S_ERROR('Pilot proxy not found at %s' %pilotProxy)
if not os.path.exists(payloadProxy):
return S_ERROR('Payload proxy not found at %s' %payloadProxy)
result = getProxyInfoAsString(payloadProxy)
if not result['OK']:
self.log.error('Could not get payload proxy info',result)
return result
self.log.verbose('Payload proxy information:\n%s' %result['Value'])
gProxyManager.renewProxy(minLifeTime=self.minProxyTime,
newProxyLifeTime=self.defaultProxyTime,
proxyToConnect=pilotProxy)
return S_OK('Proxy checked')
