def __init__(self, *args, **kwargs):
"""Initialize the agent, clients, default values."""
AgentModule.__init__(self, *args, **kwargs)
self.name = "ComponentSupervisionAgent"
self.setup = "DIRAC-Production"
self.enabled = False
self.restartAgents = False
self.restartExecutors = False
self.restartServices = False
self.controlComponents = False
self.commitURLs = False
self.doNotRestartInstancePattern = ["RequestExecutingAgent"]
self.diracLocation = rootPath
self.sysAdminClient = SystemAdministratorClient(socket.getfqdn())
self.jobMonClient = JobMonitoringClient()
self.nClient = NotificationClient()
self.csAPI = None
self.agents = dict()
self.executors = dict()
self.services = dict()
self._tornadoPort = "8443"
self.errors = list()
self.accounting = defaultdict(dict)
self.addressTo = []
self.addressFrom = ""
self.emailSubject = "ComponentSupervisionAgent on %s" % socket.getfqdn(
)
def initialize( self ):
""" Standard constructor
"""
try:
self.rsDB = ResourceStatusDB()
self.rmDB = ResourceManagementDB()
self.StorageElementToBeChecked = Queue.Queue()
self.StorageElementInCheck = []
self.maxNumberOfThreads = self.am_getOption( 'maxThreadsInPool', 1 )
self.threadPool = ThreadPool( self.maxNumberOfThreads,
self.maxNumberOfThreads )
if not self.threadPool:
self.log.error( 'Can not create Thread Pool' )
return S_ERROR( 'Can not create Thread Pool' )
self.setup = getSetup()[ 'Value' ]
self.VOExtension = getExt()
self.StorageElsWriteFreqs = CheckingFreqs[ 'StorageElsWriteFreqs' ]
self.nc = NotificationClient()
self.diracAdmin = DiracAdmin()
self.csAPI = CSAPI()
for _i in xrange( self.maxNumberOfThreads ):
self.threadPool.generateJobAndQueueIt( self._executeCheck, args = ( None, ) )
return S_OK()
except Exception:
errorStr = "StElWriteInspectorAgent initialization"
gLogger.exception( errorStr )
return S_ERROR( errorStr )
def notify(self, request, params):
"""Send notification for a given request with its params to the diracAdmin"""
address = Operations().getValue(
"ResourceStatus/Notification/DebugGroup/Users")
msg = "Matching parameters: " + str(params)
sbj = "[NOTIFICATION] DIRAC ResourceStatusDB: " + request + " entry"
NotificationClient().sendMail(address, sbj, msg, address)
def __registerNewUser(self, provider, payload):
"""Register new user
:param str provider: provider
:param dict payload: user information dictionary
:return: S_OK()/S_ERROR()
"""
from DIRAC.FrameworkSystem.Client.NotificationClient import NotificationClient
username = payload["sub"]
mail = {}
mail["subject"] = "[DIRAC AS] User %s to be added." % username
mail["body"] = "User %s was authenticated by %s." % (username, provider)
mail["body"] += "\n\nNew user to be added with the following information:\n%s" % pprint.pformat(payload)
mail["body"] += "\n\nPlease, add '%s' to /Register/Users/<username>/DN option.\n" % wrapIDAsDN(username)
mail["body"] += "\n\n------"
mail["body"] += "\n This is a notification from the DIRAC authorization service, please do not reply.\n"
result = S_OK()
for addresses in getEmailsForGroup("dirac_admin"):
result = NotificationClient().sendMail(addresses, mail["subject"], mail["body"], localAttempt=False)
if not result["OK"]:
sLog.error(result["Message"])
if result["OK"]:
sLog.info(result["Value"], "administrators have been notified about a new user.")
return result
def initialize(self):
self.SystemLoggingDB = SystemLoggingDB()
self.notification=NotificationClient()
userString = self.am_getOption( "Reviewer", 'mseco' )
self.log.debug( "Users to be notified", ": " + userString )
userList = List.fromChar( userString, ",")
mailList = []
for user in userList:
retval = gConfig.getOption( "/Registry/Users/" + user + "/Email" )
if not retval['OK']:
self.log.warn( "Could not get user's mail", retval['Message'] )
else:
mailList.append( retval['Value'] )
if not mailList:
mailList = gConfig.getValue( '/Operations/EMail/Logging', [] )
if not len(mailList):
errString = "There are no valid users in the list"
varString = "[" + ','.join( userList ) + "]"
self.log.error( errString, varString )
return S_ERROR( errString + varString )
self.log.info( "List of mails to be notified", ','.join( mailList ) )
self._mailAddress = mailList
self._subject = 'New error messages were entered in the SystemLoggingDB'
return S_OK()
def __lookForNewCEs( self ):
""" Look up BDII for CEs not yet present in the DIRAC CS
"""
bannedCEs = self.am_getOption( 'BannedCEs', [] )
result = getCEsFromCS()
if not result['OK']:
return result
knownCEs = set( result['Value'] )
knownCEs = knownCEs.union( set( bannedCEs ) )
for vo in self.voName:
result = self.__getBdiiCEInfo( vo )
if not result['OK']:
continue
bdiiInfo = result['Value']
result = getGridCEs( vo, bdiiInfo = bdiiInfo, ceBlackList = knownCEs )
if not result['OK']:
self.log.error( 'Failed to get unused CEs', result['Message'] )
siteDict = result['Value']
body = ''
for site in siteDict:
newCEs = set( siteDict[site].keys() )
if not newCEs:
continue
ceString = ''
for ce in newCEs:
queueString = ''
ceInfo = bdiiInfo[site]['CEs'][ce]
ceString = "CE: %s, GOCDB Site Name: %s" % ( ce, site )
systemTuple = siteDict[site][ce]['System']
osString = "%s_%s_%s" % ( systemTuple )
newCEString = "\n%s\n%s\n" % ( ceString, osString )
for queue in ceInfo['Queues']:
queueStatus = ceInfo['Queues'][queue].get( 'GlueCEStateStatus', 'UnknownStatus' )
if 'production' in queueStatus.lower():
ceType = ceInfo['Queues'][queue].get( 'GlueCEImplementationName', '' )
queueString += " %s %s %s\n" % ( queue, queueStatus, ceType )
if queueString:
ceString = newCEString
ceString += "Queues:\n"
ceString += queueString
if ceString:
body += ceString
if body:
body = "\nWe are glad to inform You about new CE(s) possibly suitable for %s:\n" % vo + body
body += "\n\nTo suppress information about CE add its name to BannedCEs list.\n"
body += "Add new Sites/CEs for vo %s with the command:\n" % vo
body += "dirac-admin-add-resources --vo %s --ce\n" % vo
self.log.info( body )
if self.addressTo and self.addressFrom:
notification = NotificationClient()
result = notification.sendMail( self.addressTo, self.subject, body, self.addressFrom, localAttempt = False )
if not result['OK']:
self.log.error( 'Can not send new site notification mail', result['Message'] )
return S_OK()
def initialize(self):
self.systemLoggingDB = SystemLoggingDB()
self.notification = NotificationClient()
userList = self.am_getOption("Reviewer", [])
self.log.debug("Users to be notified:", ', '.join(userList))
mailList = []
for user in userList:
mail = getUserOption(user, 'Email', '')
if not mail:
self.log.warn("Could not get user's mail", user)
else:
mailList.append(mail)
if not mailList:
mailList = Operations().getValue('EMail/Logging', [])
if not len(mailList):
errString = "There are no valid users in the mailing list"
varString = "[" + ','.join(userList) + "]"
self.log.warn(errString, varString)
self.log.info("List of mails to be notified", ','.join(mailList))
self._mailAddress = mailList
self._subject = 'New error messages were entered in the SystemLoggingDB'
return S_OK()
def checkLFCRegisteredUsers( self, usersData ):
self.log.info( "Checking LFC registered users" )
usersToBeRegistered = {}
result = self.getLFCRegisteredDNs()
if not result[ 'OK' ]:
self.log.error( "Could not get a list of registered DNs from LFC", result[ 'Message' ] )
return result
lfcDNs = result[ 'Value' ]
for user in usersData:
for userDN in usersData[ user ][ 'DN' ]:
if userDN not in lfcDNs:
self.log.info( 'DN "%s" need to be registered in LFC for user %s' % ( userDN, user ) )
if user not in usersToBeRegistered:
usersToBeRegistered[ user ] = []
usersToBeRegistered[ user ].append( userDN )
address = self.am_getOption( 'MailTo', '*****@*****.**' )
fromAddress = self.am_getOption( 'mailFrom', '*****@*****.**' )
if usersToBeRegistered:
subject = 'New LFC Users found'
self.log.info( subject, ", ".join( usersToBeRegistered ) )
body = 'Command to add new entries into LFC: \n'
body += 'login to volhcbXX and run : \n'
body += 'source /afs/cern.ch/lhcb/software/releases/LBSCRIPTS/prod/InstallArea/scripts/LbLogin.csh \n\n'
for lfcuser in usersToBeRegistered:
for lfc_dn in usersToBeRegistered[lfcuser]:
print lfc_dn
body += 'add_DN_LFC --userDN="' + lfc_dn.strip() + '" --nickname=' + lfcuser + '\n'
NotificationClient().sendMail( address, 'UsersAndGroupsAgent: %s' % subject, body, fromAddress )
return S_OK()
def notify(self, request, params):
''' Send notification for a given request with its params to the diracAdmin
'''
address = Operations().getValue('ResourceStatus/Notification/DebugGroup/Users')
msg = 'Matching parameters: ' + str(params)
sbj = '[NOTIFICATION] DIRAC ResourceStatusDB: ' + request + ' entry'
NotificationClient().sendMail(address, sbj, msg, address)
def notifyAboutNewSoftware(self):
"""Send an email to the mailing list if a new software version was defined"""
#Only send email when something was actually added
if not self.modifiedCS:
return
subject = '%s %s added to DIRAC CS' % (self.appName, self.appVersion)
msg = 'New application %s %s declared into Configuration service\n %s' % (
self.appName, self.appVersion, self.comment)
from DIRAC.Core.Security.ProxyInfo import getProxyInfo
from DIRAC.ConfigurationSystem.Client.Helpers.Registry import getUserOption
from DIRAC.FrameworkSystem.Client.NotificationClient import NotificationClient
notifyClient = NotificationClient()
gLogger.notice('Sending mail for software installation to %s' %
(self.mailadress))
res = getProxyInfo()
if not res['OK']:
sender = '*****@*****.**'
else:
if 'username' in res['Value']:
sender = getUserOption(res['Value']['username'], 'Email')
else:
sender = '*****@*****.**'
gLogger.info('*' * 80) # surround email with stars
res = notifyClient.sendMail(self.mailadress,
subject,
msg,
sender,
localAttempt=False)
gLogger.info('*' * 80)
if not res['OK']:
gLogger.error('The mail could not be sent: %s' % res['Message'])
def sendSMS(self, userName, body, fromAddress=None):
""" Send mail to specified address with body.
"""
if len(body) > 160:
return S_ERROR('Exceeded maximum SMS length of 160 characters')
notification = NotificationClient()
return notification.sendSMS(userName, body, fromAddress)
def sendNotification(self, transID, transInfoDict):
"""Send notification email if something was modified for a transformation.
:param int transID: ID of given transformation
:param transInfoDict:
"""
if not self.addressTo or not self.addressFrom or not self.notesToSend:
return
if not self.__notOnlyKeepers(transInfoDict['Type']):
# purge notes
self.notesToSend = ""
return
# remove from the jobCache because something happened
self.jobCache.pop(int(transID), None)
# send the email to recipients
for address in self.addressTo:
result = NotificationClient().sendMail(address,
"%s: %s" %
(self.subject, transID),
self.notesToSend,
self.addressFrom,
localAttempt=False)
if not result['OK']:
self.log.error('Cannot send notification mail',
result['Message'])
# purge notes
self.notesToSend = ""
def web_sendRequest(self):
typeAuth = str(self.request.arguments["typeauth"][0])
loadValue = self.request.arguments["value"]
addresses = Conf.getCSValue('AdminsEmails')
NotificationClient().sendMail(
addresses,
subject="Request from %s %s" % (loadValue[0], loadValue[1]),
body='Type auth: %s, details: %s' % (typeAuth, loadValue))
def initialize(self):
self.transClient = TransformationClient()
self.bkClient = BookkeepingClient()
self.notifyClient = NotificationClient()
self.operations = Operations()
self.email = self.am_getOption("MailTo", '')
return S_OK()
def execute(self):
self.__adminMsgs = {}
self.csapi.downloadCSData()
for vo in self.__voDict:
self.voChanged = False
voAdminUser = getVOOption(vo, "VOAdmin")
voAdminMail = None
if voAdminUser:
voAdminMail = getUserOption(voAdminUser, "Email")
voAdminGroup = getVOOption(vo, "VOAdminGroup",
getVOOption(vo, "DefaultGroup"))
self.log.info(
'Performing VOMS sync for VO %s with credentials %s@%s' %
(vo, voAdminUser, voAdminGroup))
result = self.__syncCSWithVOMS(vo,
proxyUserName=voAdminUser,
proxyUserGroup=voAdminGroup) #pylint: disable=unexpected-keyword-arg
if not result['OK']:
self.log.error('Failed to perform VOMS to CS synchronization:',
'VO %s: %s' % (vo, result["Message"]))
continue
if self.voChanged:
mailMsg = ""
if self.__adminMsgs['Errors']:
mailMsg += "\nErrors list:\n %s" % "\n ".join(
self.__adminMsgs['Errors'])
if self.__adminMsgs['Info']:
mailMsg += "\nRun result:\n %s" % "\n ".join(
self.__adminMsgs['Info'])
NotificationClient().sendMail(
self.am_getOption('MailTo', voAdminMail),
"VOMS2CSAgent run log", mailMsg,
self.am_getOption('mailFrom', "DIRAC system"))
if self.csapi.csModified:
# We have accumulated all the changes, commit them now
self.log.info("There are changes to the CS ready to be committed")
if self.dryRun:
self.log.info("Dry Run: CS won't be updated")
self.csapi.showDiff()
else:
result = self.csapi.commitChanges()
if not result['OK']:
self.log.error("Could not commit configuration changes",
result['Message'])
return result
self.log.notice("Configuration committed")
else:
self.log.info("No changes to the CS recorded at this cycle")
return S_OK()
def sendMail(self,
address,
subject,
body,
fromAddress=None,
localAttempt=True):
""" Send mail to specified address with body.
"""
notification = NotificationClient()
return notification.sendMail(address, subject, body, fromAddress,
localAttempt)
def __updateCS(self, bdiiChangeSet):
queueVODict = {}
changeSet = set()
for entry in bdiiChangeSet:
section, option, _value, new_value = entry
if option == "VO":
queueVODict.setdefault(section, set())
queueVODict[section] = queueVODict[section].union(
set(new_value.split(',')))
else:
changeSet.add(entry)
for section, VOs in queueVODict.items():
changeSet.add((section, 'VO', '', ','.join(VOs)))
if changeSet:
changeList = list(changeSet)
changeList.sort()
body = '\n'.join(
["%s/%s %s -> %s" % entry for entry in changeList])
if body and self.addressTo and self.addressFrom:
notification = NotificationClient()
result = notification.sendMail(self.addressTo,
self.subject,
body,
self.addressFrom,
localAttempt=False)
if body:
self.log.info(
'The following configuration changes were detected:')
self.log.info(body)
for section, option, value, new_value in changeSet:
if value == 'Unknown' or not value:
self.csAPI.setOption(cfgPath(section, option), new_value)
else:
self.csAPI.modifyValue(cfgPath(section, option), new_value)
if self.dryRun:
self.log.info("Dry Run: CS won't be updated")
self.csAPI.showDiff()
else:
result = self.csAPI.commit()
if not result['OK']:
self.log.error("Error while committing to CS",
result['Message'])
else:
self.log.info("Successfully committed %d changes to CS" %
len(changeList))
return result
else:
self.log.info("No changes found")
return S_OK()
def execute( self ):
result = self.__syncCSWithVOMS()
mailMsg = ""
if self.__adminMsgs[ 'Errors' ]:
mailMsg += "\nErrors list:\n %s" % "\n ".join( self.__adminMsgs[ 'Errors' ] )
if self.__adminMsgs[ 'Info' ]:
mailMsg += "\nRun result:\n %s" % "\n ".join( self.__adminMsgs[ 'Info' ] )
NotificationClient().sendMail( self.am_getOption( 'MailTo', '*****@*****.**' ),
"UsersAndGroupsAgent run log", mailMsg,
self.am_getOption( 'mailFrom', '*****@*****.**' ) )
return result
def __init__(self, bkClient=None, dm=None):
"""Module initialization.
"""
self.log = gLogger.getSubLogger('AnalyseXMLSummary')
super(AnalyseXMLSummary, self).__init__(self.log, bkClientIn=bkClient, dm=dm)
self.version = __RCSID__
self.nc = NotificationClient()
self.XMLSummary = ''
self.XMLSummary_o = None
def __sendErrorMail( self, rb, name, command, result, proxy ):
"""
In case or error with RB/WM:
- check if RB/WMS still in use
- remove RB/WMS from current list
- check if RB/WMS not in cache
- add RB/WMS to cache
- send Error mail
"""
if rb in self.resourceBrokers:
try:
self.resourceBrokers.remove( rb )
self.log.info( 'Removed RB from list', rb )
except:
pass
if not self.__failingWMSCache.exists( rb ):
self.__failingWMSCache.add( rb, self.errorClearTime ) # disable for 30 minutes
mailAddress = self.errorMailAddress
msg = ''
if not result['OK']:
subject = "%s: timeout executing %s" % ( rb, name )
msg += '\n%s' % result['Message']
elif result['Value'][0] != 0:
if re.search( 'the server is temporarily drained', ' '.join( result['Value'][1:3] ) ):
return
if re.search( 'System load is too high:', ' '.join( result['Value'][1:3] ) ):
return
subject = "%s: error executing %s" % ( rb, name )
else:
return
msg += ' '.join( command )
msg += '\nreturns: %s\n' % str( result['Value'][0] ) + '\n'.join( result['Value'][1:3] )
msg += '\nUsing Proxy:\n' + getProxyInfoAsString( proxy )['Value']
#msg += '\nUsing Proxy:\n' + gProxyManager.
ticketTime = self.errorClearTime + self.errorTicketTime
if self.__ticketsWMSCache.exists( rb ):
mailAddress = self.alarmMailAddress
# the RB was already detected failing a short time ago
msg = 'Submit GGUS Ticket for this error if not already opened\n' + \
'It has been failing at least for %s hours\n' % ( ticketTime / 60 / 60 ) + msg
else:
self.__ticketsWMSCache.add( rb, ticketTime )
if mailAddress:
result = NotificationClient().sendMail( mailAddress, subject, msg, fromAddress = self.mailFromAddress )
if not result[ 'OK' ]:
self.log.error( "Mail could not be sent" )
return
def __lookForNewSEs(self):
""" Look up BDII for SEs not yet present in the DIRAC CS
"""
bannedSEs = self.am_getOption('BannedSEs', [])
result = getSEsFromCS()
if not result['OK']:
return result
knownSEs = set(result['Value'])
knownSEs = knownSEs.union(set(bannedSEs))
for vo in self.voName:
result = self.__getBdiiSEInfo(vo)
if not result['OK']:
continue
bdiiInfo = result['Value']
result = getGridSRMs(vo, bdiiInfo=bdiiInfo, srmBlackList=knownSEs)
if not result['OK']:
continue
siteDict = result['Value']
body = ''
for site in siteDict:
newSEs = set(siteDict[site].keys()) # pylint: disable=no-member
if not newSEs:
continue
for se in newSEs:
body += '\n New SE %s available at site %s:\n' % (se, site)
backend = siteDict[site][se]['SE'].get(
'GlueSEImplementationName', 'Unknown')
size = siteDict[site][se]['SE'].get(
'GlueSESizeTotal', 'Unknown')
body += ' Backend %s, Size %s' % (backend, size)
if body:
body = "\nWe are glad to inform You about new SE(s) possibly suitable for %s:\n" % vo + body
body += "\n\nTo suppress information about an SE add its name to BannedSEs list.\n"
body += "Add new SEs for vo %s with the command:\n" % vo
body += "dirac-admin-add-resources --vo %s --se\n" % vo
self.log.info(body)
if self.addressTo and self.addressFrom:
notification = NotificationClient()
result = notification.sendMail(self.addressTo,
self.subject,
body,
self.addressFrom,
localAttempt=False)
if not result['OK']:
self.log.error(
'Can not send new site notification mail',
result['Message'])
return S_OK()
def main():
Script.parseCommandLine(ignoreErrors=True)
args = Script.getPositionalArgs()
arg = "".join(args)
if not len(arg) > 0:
gLogger.error("Missing argument")
DIRACexit(2)
try:
head, body = arg.split("\\n\\n")
except Exception as x:
head = "To: %s" % arg
body = sys.stdin.read()
try:
tmp, body = body.split("\\n\\n")
head = tmp + "\\n" + head
except Exception as x:
pass
body = "".join(body.strip())
try:
headers = dict((i.strip(), j.strip()) for i, j in
(item.split(':') for item in head.split('\\n')))
except BaseException:
gLogger.error("Failed to convert string: %s to email headers" % head)
DIRACexit(4)
if "To" not in headers:
gLogger.error("Failed to get 'To:' field from headers %s" % head)
DIRACexit(5)
to = headers["To"]
origin = "%s@%s" % (os.getenv("LOGNAME", "dirac"), socket.getfqdn())
if "From" in headers:
origin = headers["From"]
subject = "Sent from %s" % socket.getfqdn()
if "Subject" in headers:
subject = headers["Subject"]
ntc = NotificationClient()
print("sendMail(%s,%s,%s,%s,%s)" % (to, subject, body, origin, False))
result = ntc.sendMail(to, subject, body, origin, localAttempt=False)
if not result["OK"]:
gLogger.error(result["Message"])
DIRACexit(6)
DIRACexit(0)
def initialize(self):
"""
TokenAgent initialization
"""
try:
self.rsDB = ResourceStatusDB()
self.nc = NotificationClient()
return S_OK()
except Exception:
errorStr = "TokenAgent initialization"
gLogger.exception(errorStr)
return S_ERROR(errorStr)
def parseAuthResponse(self, response, session):
""" Fill session by user profile, tokens, comment, OIDC authorize status, etc.
Prepare dict with user parameters, if DN is absent there try to get it.
Create new or modify existend DIRAC user and store the session
:param dict response: authorization response dictionary
:param basestring session: session id
:return: S_OK(dict)/S_ERROR() -- dictionary contain session status, comment and user profile
"""
result = self.updateSession({'Status': 'finishing'}, session=session)
if not result['OK']:
return result
self.log.info("%s session, parse authentication response:" % session,
response)
result = self.__parse(response, session)
if not result['OK']:
self.log.error(session, 'session error: %s' % result['Message'])
self.updateSession(
{
'Status': 'failed',
'Comment': result['Message']
},
session=session)
return result
parseDict, status, comment, mail = result['Value']
if mail:
for addresses in getEmailsForGroup('dirac_admin'):
result = NotificationClient().sendMail(addresses,
mail['subject'],
mail['body'],
localAttempt=False)
if not result['OK']:
self.updateSession(
{
'Status': 'failed',
'Comment': result['Message']
},
session=session)
self.log.error(session,
'session error: %s' % result['Message'])
return result
self.log.info("%s session, mails to admins:", result['Value'])
return S_OK({
'Status': status,
'Comment': comment,
'UserProfile': parseDict
})
def __init__(self, *args, **kwargs):
''' c'tor
'''
AgentModule.__init__(self, *args, **kwargs)
self.notificationClient = NotificationClient()
try:
self.storageUsageDB = StorageUsageDB()
except SystemExit:
self.storageUsageDB = RPCClient('DataManagement/StorageUsage')
self.defaultQuota = gConfig.getValue(
'/Registry/DefaultStorageQuota',
self.defaultQuota) # Default is 1TB
def __init__(self, useMyProxy=False):
DB.__init__(self, 'ProxyDB', 'Framework/ProxyDB')
random.seed()
self.__defaultRequestLifetime = 300 # 5min
self.__defaultTokenLifetime = 86400 * 7 # 1 week
self.__defaultTokenMaxUses = 50
self.__useMyProxy = useMyProxy
self._minSecsToAllowStore = 3600
self.__notifClient = NotificationClient()
retVal = self.__initializeDB()
if not retVal['OK']:
raise Exception("Can't create tables: %s" % retVal['Message'])
self.purgeExpiredProxies(sendNotifications=False)
self.__checkDBVersion()
def rescheduleFailedJob(jobID, message):
try:
import DIRAC
global jobReport
gLogger.warn('Failure during %s' % (message))
#Setting a job parameter does not help since the job will be rescheduled,
#instead set the status with the cause and then another status showing the
#reschedule operation.
if not jobReport:
gLogger.info('Creating a new JobReport Object')
jobReport = JobReport(int(jobID), 'JobWrapperTemplate')
jobReport.setApplicationStatus('Failed %s ' % message, sendFlag=False)
jobReport.setJobStatus('Rescheduled', message, sendFlag=False)
# We must send Job States and Parameters before it gets reschedule
jobReport.sendStoredStatusInfo()
jobReport.sendStoredJobParameters()
gLogger.info(
'Job will be rescheduled after exception during execution of the JobWrapper'
)
jobManager = RPCClient('WorkloadManagement/JobManager')
result = jobManager.rescheduleJob(int(jobID))
if not result['OK']:
gLogger.warn(result)
# Send mail to debug errors
mailAddress = DIRAC.alarmMail
site = DIRAC.siteName()
subject = 'Job rescheduled at %s' % site
ret = systemCall(0, 'hostname')
wn = ret['Value'][1]
msg = 'Job %s rescheduled at %s, wn=%s\n' % (jobID, site, wn)
msg += message
NotificationClient().sendMail(mailAddress,
subject,
msg,
fromAddress="*****@*****.**",
localAttempt=False)
return
except Exception, x:
gLogger.exception('JobWrapperTemplate failed to reschedule Job')
return
def initialize( self ):
''' TokenAgent initialization
Uses the ProductionManager shifterProxy to modify the ResourceStatus DB
'''
self.notifyHours = self.am_getOption( 'notifyHours', self.notifyHours )
self.adminMail = self.am_getOption( 'adminMail', self.adminMail )
self.rsClient = ResourceStatusClient()
self.rmClient = ResourceManagementClient()
self.noClient = NotificationClient()
self.diracAdmin = DiracAdmin()
return S_OK()
def initialize(self):
""" Standard constructor
"""
try:
self.rsDB = ResourceStatusDB()
self.rmDB = ResourceManagementDB()
self.ResourcesToBeChecked = Queue.Queue()
self.ResourceNamesInCheck = []
self.maxNumberOfThreads = self.am_getOption('maxThreadsInPool', 1)
self.threadPool = ThreadPool(self.maxNumberOfThreads,
self.maxNumberOfThreads)
if not self.threadPool:
self.log.error('Can not create Thread Pool')
return S_ERROR('Can not create Thread Pool')
self.setup = getSetup()['Value']
self.VOExtension = getExt()
configModule = __import__(
self.VOExtension +
"DIRAC.ResourceStatusSystem.Policy.Configurations", globals(),
locals(), ['*'])
self.Resources_check_freq = copy.deepcopy(
configModule.Resources_check_freq)
self.nc = NotificationClient()
self.diracAdmin = DiracAdmin()
self.csAPI = CSAPI()
for i in xrange(self.maxNumberOfThreads):
self.threadPool.generateJobAndQueueIt(self._executeCheck,
args=(None, ))
return S_OK()
except Exception:
errorStr = "RSInspectorAgent initialization"
gLogger.exception(errorStr)
return S_ERROR(errorStr)
def initialize(self):
"""
TokenAgent initialization
"""
self.ELEMENTS = ['Site', 'StorageElementRead', 'StorageElementWrite']
self.notifyHours = self.am_getOption('notifyHours', 10)
try:
self.rsDB = ResourceStatusDB()
self.nc = NotificationClient()
self.VOExt = getExt()
return S_OK()
except Exception:
errorStr = "TokenAgent initialization"
gLogger.exception(errorStr)
return S_ERROR(errorStr)
