class TestComponentInstallation( unittest.TestCase ):
"""
Contains methods for testing of separate elements
"""
def setUp( self ):
self.host = 'localhost'
self.notificationPort = 9154
self.rootPwd = ''
self.csClient = CSAPI()
self.monitoringClient = ComponentMonitoringClient()
self.client = SystemAdministratorClientCLI( self.host )
self.csClient.downloadCSData()
result = self.csClient.getCurrentCFG()
if not result[ 'OK' ]:
raise Exception( result[ 'Message' ] )
cfg = result[ 'Value' ]
setup = cfg.getOption( 'DIRAC/Setup', 'JenkinsSetup' )
self.frameworkSetup = cfg.getOption( 'DIRAC/Setups/' + setup + '/Framework' )
self.rootPwd = cfg.getOption( 'Systems/Databases/Password' )
self.diracPwd = self.rootPwd
def tearDown( self ):
pass
class TestComponentInstallation(unittest.TestCase):
"""
Contains methods for testing of separate elements
"""
def setUp(self):
self.host = 'localhost'
self.notificationPort = 9154
self.rootPwd = ''
self.csClient = CSAPI()
self.monitoringClient = ComponentMonitoringClient()
self.client = SystemAdministratorClientCLI(self.host)
self.csClient.downloadCSData()
result = self.csClient.getCurrentCFG()
if not result['OK']:
raise Exception(result['Message'])
cfg = result['Value']
setup = cfg.getOption('DIRAC/Setup', 'JenkinsSetup')
self.frameworkSetup = cfg.getOption('DIRAC/Setups/' + setup +
'/Framework')
self.rootPwd = cfg.getOption('Systems/Databases/Password')
self.diracPwd = self.rootPwd
def tearDown(self):
pass
def updateCS(changeSet):
global vo, dry, ceBdiiDict
changeList = sorted(changeSet)
if dry:
gLogger.notice('The following needed changes are detected:\n')
else:
gLogger.notice('We are about to make the following changes to CS:\n')
for entry in changeList:
gLogger.notice("%s/%s %s -> %s" % entry)
if not dry:
csAPI = CSAPI()
csAPI.initialize()
result = csAPI.downloadCSData()
if not result['OK']:
gLogger.error('Failed to initialize CSAPI object',
result['Message'])
DIRACExit(-1)
for section, option, value, new_value in changeSet:
if value == 'Unknown' or not value:
csAPI.setOption(cfgPath(section, option), new_value)
else:
csAPI.modifyValue(cfgPath(section, option), new_value)
yn = six.moves.input(
'Do you want to commit changes to CS ? [default yes] [yes|no]: ')
if yn == '' or yn.lower().startswith('y'):
result = csAPI.commit()
if not result['OK']:
gLogger.error("Error while commit to CS", result['Message'])
else:
gLogger.notice("Successfully committed %d changes to CS" %
len(changeSet))
def updateCS( changeSet ):
global vo, dry, ceBdiiDict
changeList = list( changeSet )
changeList.sort()
if dry:
gLogger.notice( 'The following needed changes are detected:\n' )
else:
gLogger.notice( 'We are about to make the following changes to CS:\n' )
for entry in changeList:
gLogger.notice( "%s/%s %s -> %s" % entry )
if not dry:
csAPI = CSAPI()
csAPI.initialize()
result = csAPI.downloadCSData()
if not result['OK']:
gLogger.error( 'Failed to initialize CSAPI object', result['Message'] )
DIRACExit( -1 )
for section, option, value, new_value in changeSet:
if value == 'Unknown' or not value:
csAPI.setOption( cfgPath( section, option ), new_value )
else:
csAPI.modifyValue( cfgPath( section, option ), new_value )
yn = raw_input( 'Do you want to commit changes to CS ? [default yes] [yes|no]: ' )
if yn == '' or yn.lower().startswith( 'y' ):
result = csAPI.commit()
if not result['OK']:
gLogger.error( "Error while commit to CS", result['Message'] )
else:
gLogger.notice( "Successfully committed %d changes to CS" % len( changeSet ) )
class TestComponentInstallation(unittest.TestCase):
"""
Contains methods for testing of separate elements
"""
def setUp(self):
self.host = "localhost"
self.notificationPort = 9154
self.rootPwd = ""
self.csClient = CSAPI()
self.monitoringClient = ComponentMonitoringClient()
self.client = SystemAdministratorClientCLI(self.host)
self.csClient.downloadCSData()
result = self.csClient.getCurrentCFG()
if not result["OK"]:
raise Exception(result["Message"])
cfg = result["Value"]
setup = cfg.getOption("DIRAC/Setup", "dirac-JenkinsSetup")
self.frameworkSetup = cfg.getOption("DIRAC/Setups/" + setup +
"/Framework")
self.rootPwd = cfg.getOption("Systems/Databases/Password")
self.diracPwd = self.rootPwd
result = getProxyInfo()
if not result["OK"]:
raise Exception(result["Message"])
chain = result["Value"]["chain"]
result = chain.getCertInChain(-1)
if not result["OK"]:
raise Exception(result["Message"])
result = result["Value"].getSubjectDN()
if not result["OK"]:
raise Exception(result["Message"])
userDN = result["Value"]
result = getUsernameForDN(userDN)
if not result["OK"]:
raise Exception(result["Message"])
self.user = result["Value"]
if not self.user:
self.user = "******"
def tearDown(self):
pass
class TestComponentInstallation(unittest.TestCase):
"""
Contains methods for testing of separate elements
"""
def setUp(self):
self.host = 'localhost'
self.notificationPort = 9154
self.rootPwd = ''
self.csClient = CSAPI()
self.monitoringClient = ComponentMonitoringClient()
self.client = SystemAdministratorClientCLI(self.host)
self.csClient.downloadCSData()
result = self.csClient.getCurrentCFG()
if not result['OK']:
raise Exception(result['Message'])
cfg = result['Value']
setup = cfg.getOption('DIRAC/Setup', 'dirac-JenkinsSetup')
self.frameworkSetup = cfg.getOption('DIRAC/Setups/' + setup + '/Framework')
self.rootPwd = cfg.getOption('Systems/Databases/Password')
self.diracPwd = self.rootPwd
result = getProxyInfo()
if not result['OK']:
raise Exception(result['Message'])
chain = result['Value']['chain']
result = chain.getCertInChain(-1)
if not result['OK']:
raise Exception(result['Message'])
result = result['Value'].getSubjectDN()
if not result['OK']:
raise Exception(result['Message'])
userDN = result['Value']
result = getUsernameForDN(userDN)
if not result['OK']:
raise Exception(result['Message'])
self.user = result['Value']
if not self.user:
self.user = '******'
def tearDown(self):
pass
class TestComponentInstallation(unittest.TestCase):
"""
Contains methods for testing of separate elements
"""
def setUp(self):
self.host = 'localhost'
self.notificationPort = 9154
self.rootPwd = ''
self.csClient = CSAPI()
self.monitoringClient = ComponentMonitoringClient()
self.client = SystemAdministratorClientCLI(self.host)
self.csClient.downloadCSData()
result = self.csClient.getCurrentCFG()
if not result['OK']:
raise Exception(result['Message'])
cfg = result['Value']
setup = cfg.getOption('DIRAC/Setup', 'dirac-JenkinsSetup')
self.frameworkSetup = cfg.getOption('DIRAC/Setups/' + setup + '/Framework')
self.rootPwd = cfg.getOption('Systems/Databases/Password')
self.diracPwd = self.rootPwd
result = getProxyInfo()
if not result['OK']:
raise Exception(result['Message'])
chain = result['Value']['chain']
result = chain.getCertInChain(-1)
if not result['OK']:
raise Exception(result['Message'])
result = result['Value'].getSubjectDN()
if not result['OK']:
raise Exception(result['Message'])
userDN = result['Value']
result = getUsernameForDN(userDN)
if not result['OK']:
raise Exception(result['Message'])
self.user = result['Value']
if not self.user:
self.user = '******'
def tearDown(self):
pass
class Bdii2CSAgent( AgentModule ):
addressTo = ''
addressFrom = ''
voName = ''
subject = "CE2CSAgent"
alternativeBDIIs = []
def initialize( self ):
self.addressTo = self.am_getOption( 'MailTo', self.addressTo )
self.addressFrom = self.am_getOption( 'MailFrom', self.addressFrom )
# Create a list of alternative bdii urls
self.alternativeBDIIs = self.am_getOption( 'AlternativeBDIIs', [] )
# Check if the bdii url is appended by a port number, if not append the default 2170
for index, url in enumerate( self.alternativeBDIIs ):
if not url.split( ':' )[-1].isdigit():
self.alternativeBDIIs[index] += ':2170'
if self.addressTo and self.addressFrom:
self.log.info( "MailTo", self.addressTo )
self.log.info( "MailFrom", self.addressFrom )
if self.alternativeBDIIs :
self.log.info( "AlternativeBDII URLs:", self.alternativeBDIIs )
self.subject = "CE2CSAgent"
self.processCEs = self.am_getOption( 'ProcessCEs', True )
self.processSEs = self.am_getOption( 'ProcessSEs', False )
self.voName = self.am_getOption( 'VirtualOrganization', [] )
if not self.voName:
self.voName = self.am_getOption( 'VO', [] )
if not self.voName or ( len( self.voName ) == 1 and self.voName[0].lower() == 'all' ):
# Get all VOs defined in the configuration
self.voName = []
result = getVOs()
if result['OK']:
vos = result['Value']
for vo in vos:
vomsVO = getVOOption( vo, "VOMSName" )
if vomsVO:
self.voName.append( vomsVO )
if self.voName:
self.log.info( "Agent will manage VO(s) %s" % self.voName )
else:
self.log.fatal( "VirtualOrganization option not defined for agent" )
return S_ERROR()
self.voBdiiCEDict = {}
self.voBdiiSEDict = {}
self.csAPI = CSAPI()
return self.csAPI.initialize()
def execute( self ):
""" General agent execution method
"""
# Get a "fresh" copy of the CS data
result = self.csAPI.downloadCSData()
if not result['OK']:
self.log.warn( "Could not download a fresh copy of the CS data", result[ 'Message' ] )
if self.processCEs:
self.__lookForNewCEs()
self.__updateCEs()
if self.processSEs:
self.__lookForNewSEs()
self.__updateSEs()
return S_OK()
def __lookForNewCEs( self ):
""" Look up BDII for CEs not yet present in the DIRAC CS
"""
bannedCEs = self.am_getOption( 'BannedCEs', [] )
result = getCEsFromCS()
if not result['OK']:
return result
knownCEs = set( result['Value'] )
knownCEs = knownCEs.union( set( bannedCEs ) )
for vo in self.voName:
result = self.__getBdiiCEInfo( vo )
if not result['OK']:
continue
bdiiInfo = result['Value']
result = getGridCEs( vo, bdiiInfo = bdiiInfo, ceBlackList = knownCEs )
if not result['OK']:
self.log.error( 'Failed to get unused CEs', result['Message'] )
siteDict = result['Value']
body = ''
for site in siteDict:
newCEs = set( siteDict[site].keys() )
if not newCEs:
continue
ceString = ''
for ce in newCEs:
queueString = ''
ceInfo = bdiiInfo[site]['CEs'][ce]
ceString = "CE: %s, GOCDB Site Name: %s" % ( ce, site )
systemTuple = siteDict[site][ce]['System']
osString = "%s_%s_%s" % ( systemTuple )
newCEString = "\n%s\n%s\n" % ( ceString, osString )
for queue in ceInfo['Queues']:
queueStatus = ceInfo['Queues'][queue].get( 'GlueCEStateStatus', 'UnknownStatus' )
if 'production' in queueStatus.lower():
ceType = ceInfo['Queues'][queue].get( 'GlueCEImplementationName', '' )
queueString += " %s %s %s\n" % ( queue, queueStatus, ceType )
if queueString:
ceString = newCEString
ceString += "Queues:\n"
ceString += queueString
if ceString:
body += ceString
if body:
body = "\nWe are glad to inform You about new CE(s) possibly suitable for %s:\n" % vo + body
body += "\n\nTo suppress information about CE add its name to BannedCEs list.\n"
body += "Add new Sites/CEs for vo %s with the command:\n" % vo
body += "dirac-admin-add-resources --vo %s --ce\n" % vo
self.log.info( body )
if self.addressTo and self.addressFrom:
notification = NotificationClient()
result = notification.sendMail( self.addressTo, self.subject, body, self.addressFrom, localAttempt = False )
if not result['OK']:
self.log.error( 'Can not send new site notification mail', result['Message'] )
return S_OK()
def __getBdiiCEInfo( self, vo ):
if vo in self.voBdiiCEDict:
return S_OK( self.voBdiiCEDict[vo] )
self.log.info( "Check for available CEs for VO", vo )
result = getBdiiCEInfo( vo )
message = ''
if not result['OK']:
message = result['Message']
for bdii in self.alternativeBDIIs :
result = getBdiiCEInfo( vo, host = bdii )
if result['OK']:
break
if not result['OK']:
if message:
self.log.error( "Error during BDII request", message )
else:
self.log.error( "Error during BDII request", result['Message'] )
else:
self.voBdiiCEDict[vo] = result['Value']
return result
def __getBdiiSEInfo( self, vo ):
if vo in self.voBdiiSEDict:
return S_OK( self.voBdiiSEDict[vo] )
self.log.info( "Check for available SEs for VO", vo )
result = getBdiiSEInfo( vo )
message = ''
if not result['OK']:
message = result['Message']
for bdii in self.alternativeBDIIs :
result = getBdiiSEInfo( vo, host = bdii )
if result['OK']:
break
if not result['OK']:
if message:
self.log.error( "Error during BDII request", message )
else:
self.log.error( "Error during BDII request", result['Message'] )
else:
self.voBdiiSEDict[vo] = result['Value']
return result
def __updateCEs( self ):
""" Update the Site/CE/queue settings in the CS if they were changed in the BDII
"""
bdiiChangeSet = set()
for vo in self.voName:
result = self.__getBdiiCEInfo( vo )
if not result['OK']:
continue
ceBdiiDict = result['Value']
result = getSiteUpdates( vo, bdiiInfo = ceBdiiDict, log = self.log )
if not result['OK']:
continue
bdiiChangeSet = bdiiChangeSet.union( result['Value'] )
# We have collected all the changes, consolidate VO settings
result = self.__updateCS( bdiiChangeSet )
return result
def __updateCS( self, bdiiChangeSet ):
queueVODict = {}
changeSet = set()
for entry in bdiiChangeSet:
section, option , _value, new_value = entry
if option == "VO":
queueVODict.setdefault( section, set() )
queueVODict[section] = queueVODict[section].union( set( new_value.split( ',' ) ) )
else:
changeSet.add( entry )
for section, VOs in queueVODict.items():
changeSet.add( ( section, 'VO', '', ','.join( VOs ) ) )
if changeSet:
changeList = list( changeSet )
changeList.sort()
body = '\n'.join( [ "%s/%s %s -> %s" % entry for entry in changeList ] )
if body and self.addressTo and self.addressFrom:
notification = NotificationClient()
result = notification.sendMail( self.addressTo, self.subject, body, self.addressFrom, localAttempt = False )
if body:
self.log.info( 'The following configuration changes were detected:' )
self.log.info( body )
for section, option, value, new_value in changeSet:
if value == 'Unknown' or not value:
self.csAPI.setOption( cfgPath( section, option ), new_value )
else:
self.csAPI.modifyValue( cfgPath( section, option ), new_value )
result = self.csAPI.commit()
if not result['OK']:
self.log.error( "Error while committing to CS", result['Message'] )
else:
self.log.info( "Successfully committed %d changes to CS" % len( changeList ) )
return result
else:
self.log.info( "No changes found" )
return S_OK()
def __lookForNewSEs( self ):
""" Look up BDII for SEs not yet present in the DIRAC CS
"""
bannedSEs = self.am_getOption( 'BannedSEs', [] )
result = getSEsFromCS()
if not result['OK']:
return result
knownSEs = set( result['Value'] )
knownSEs = knownSEs.union( set( bannedSEs ) )
for vo in self.voName:
result = self.__getBdiiSEInfo( vo )
if not result['OK']:
continue
bdiiInfo = result['Value']
result = getGridSRMs( vo, bdiiInfo = bdiiInfo, srmBlackList = knownSEs )
if not result['OK']:
continue
siteDict = result['Value']
body = ''
for site in siteDict:
newSEs = set( siteDict[site].keys() )
if not newSEs:
continue
for se in newSEs:
body += '\n New SE %s available at site %s:\n' % ( se, site )
backend = siteDict[site][se]['SE'].get( 'GlueSEImplementationName', 'Unknown' )
size = siteDict[site][se]['SE'].get( 'GlueSESizeTotal', 'Unknown' )
body += ' Backend %s, Size %s' % ( backend, size )
if body:
body = "\nWe are glad to inform You about new SE(s) possibly suitable for %s:\n" % vo + body
body += "\n\nTo suppress information about an SE add its name to BannedSEs list.\n"
body += "Add new SEs for vo %s with the command:\n" % vo
body += "dirac-admin-add-resources --vo %s --se\n" % vo
self.log.info( body )
if self.addressTo and self.addressFrom:
notification = NotificationClient()
result = notification.sendMail( self.addressTo, self.subject, body, self.addressFrom, localAttempt = False )
if not result['OK']:
self.log.error( 'Can not send new site notification mail', result['Message'] )
return S_OK()
def __updateSEs( self ):
""" Update the Storage Element settings in the CS if they were changed in the BDII
"""
bdiiChangeSet = set()
for vo in self.voName:
result = self.__getBdiiSEInfo( vo )
if not result['OK']:
continue
seBdiiDict = result['Value']
result = getSRMUpdates( vo, bdiiInfo = seBdiiDict )
if not result['OK']:
continue
bdiiChangeSet = bdiiChangeSet.union( result['Value'] )
# We have collected all the changes, consolidate VO settings
result = self.__updateCS( bdiiChangeSet )
return result
class CE2CSAgent( AgentModule ):
addressTo = ''
addressFrom = ''
voName = ''
subject = "CE2CSAgent"
alternativeBDIIs = []
def initialize( self ):
# TODO: Have no default and if no mail is found then use the diracAdmin group
# and resolve all associated mail addresses.
self.addressTo = self.am_getOption( 'MailTo', self.addressTo )
self.addressFrom = self.am_getOption( 'MailFrom', self.addressFrom )
# create a list of alternative bdii urls
self.alternativeBDIIs = self.am_getOption( 'AlternativeBDIIs', [] )
# check if the bdii url is appended by a port number, if not append the default 2170
for index, url in enumerate( self.alternativeBDIIs ):
if not url.split( ':' )[-1].isdigit():
self.alternativeBDIIs[index] += ':2170'
if self.addressTo and self.addressFrom:
self.log.info( "MailTo", self.addressTo )
self.log.info( "MailFrom", self.addressFrom )
if self.alternativeBDIIs :
self.log.info( "AlternativeBDII URLs:", self.alternativeBDIIs )
self.subject = "CE2CSAgent"
# This sets the Default Proxy to used as that defined under
# /Operations/Shifter/TestManager
# the shifterProxy option in the Configuration can be used to change this default.
self.am_setOption( 'shifterProxy', 'TestManager' )
self.voName = self.am_getOption( 'VirtualOrganization', self.voName )
if not self.voName:
self.voName = getVO()
if not self.voName:
self.log.fatal( "VO option not defined for agent" )
return S_ERROR()
self.csAPI = CSAPI()
return self.csAPI.initialize()
def execute( self ):
self.log.info( "Start Execution" )
result = getProxyInfo()
if not result[ 'OK' ]:
return result
infoDict = result[ 'Value' ]
self.log.info( formatProxyInfoAsString( infoDict ) )
#Get a "fresh" copy of the CS data
result = self.csAPI.downloadCSData()
if not result[ 'OK' ]:
self.log.warn( "Could not download a fresh copy of the CS data", result[ 'Message' ] )
self.__lookForCE()
self.__infoFromCE()
self.log.info( "End Execution" )
return S_OK()
def __checkAlternativeBDIISite( self, fun, *args ):
if self.alternativeBDIIs:
self.log.warn( "Trying to use alternative bdii sites" )
for site in self.alternativeBDIIs :
self.log.info( "Trying to contact alternative bdii ", site )
if len( args ) == 1 :
result = fun( args[0], host = site )
elif len( args ) == 2 :
result = fun( args[0], vo = args[1], host = site )
if not result['OK'] :
self.log.error ( "Problem contacting alternative bddii", result['Message'] )
elif result['OK'] :
return result
self.log.warn( "Also checking alternative BDII sites failed" )
return result
def __lookForCE( self ):
knownces = self.am_getOption( 'BannedCEs', [] )
result = gConfig.getSections( '/Resources/Sites' )
if not result['OK']:
return
grids = result['Value']
for grid in grids:
result = gConfig.getSections( '/Resources/Sites/%s' % grid )
if not result['OK']:
return
sites = result['Value']
for site in sites:
opt = gConfig.getOptionsDict( '/Resources/Sites/%s/%s' % ( grid, site ) )['Value']
ces = List.fromChar( opt.get( 'CE', '' ) )
knownces += ces
response = ldapCEState( '', vo = self.voName )
if not response['OK']:
self.log.error( "Error during BDII request", response['Message'] )
response = self.__checkAlternativeBDIISite( ldapCEState, '', self.voName )
return response
newces = {}
for queue in response['Value']:
try:
queuename = queue['GlueCEUniqueID']
except:
continue
cename = queuename.split( ":" )[0]
if not cename in knownces:
newces[cename] = None
self.log.debug( "newce", cename )
body = ""
possibleNewSites = []
for ce in newces.iterkeys():
response = ldapCluster( ce )
if not response['OK']:
self.log.warn( "Error during BDII request", response['Message'] )
response = self.__checkAlternativeBDIISite( ldapCluster, ce )
continue
clusters = response['Value']
if len( clusters ) != 1:
self.log.warn( "Error in cluster length", " CE %s Length %d" % ( ce, len( clusters ) ) )
if len( clusters ) == 0:
continue
cluster = clusters[0]
fkey = cluster.get( 'GlueForeignKey', [] )
if type( fkey ) == type( '' ):
fkey = [fkey]
nameBDII = None
for entry in fkey:
if entry.count( 'GlueSiteUniqueID' ):
nameBDII = entry.split( '=' )[1]
break
if not nameBDII:
continue
cestring = "CE: %s, GOCDB Name: %s" % ( ce, nameBDII )
self.log.info( cestring )
response = ldapCE( ce )
if not response['OK']:
self.log.warn( "Error during BDII request", response['Message'] )
response = self.__checkAlternativeBDIISite( ldapCE, ce )
continue
ceinfos = response['Value']
if len( ceinfos ):
ceinfo = ceinfos[0]
systemName = ceinfo.get( 'GlueHostOperatingSystemName', 'Unknown' )
systemVersion = ceinfo.get( 'GlueHostOperatingSystemVersion', 'Unknown' )
systemRelease = ceinfo.get( 'GlueHostOperatingSystemRelease', 'Unknown' )
else:
systemName = "Unknown"
systemVersion = "Unknown"
systemRelease = "Unknown"
osstring = "SystemName: %s, SystemVersion: %s, SystemRelease: %s" % ( systemName, systemVersion, systemRelease )
self.log.info( osstring )
response = ldapCEState( ce, vo = self.voName )
if not response['OK']:
self.log.warn( "Error during BDII request", response['Message'] )
response = self.__checkAlternativeBDIISite( ldapCEState, ce, self.voName )
continue
newcestring = "\n\n%s\n%s" % ( cestring, osstring )
usefull = False
cestates = response['Value']
for cestate in cestates:
queuename = cestate.get( 'GlueCEUniqueID', 'UnknownName' )
queuestatus = cestate.get( 'GlueCEStateStatus', 'UnknownStatus' )
queuestring = "%s %s" % ( queuename, queuestatus )
self.log.info( queuestring )
newcestring += "\n%s" % queuestring
if queuestatus.count( 'Production' ):
usefull = True
if usefull:
body += newcestring
possibleNewSites.append( 'dirac-admin-add-site DIRACSiteName %s %s' % ( nameBDII, ce ) )
if body:
body = "We are glad to inform You about new CE(s) possibly suitable for %s:\n" % self.voName + body
body += "\n\nTo suppress information about CE add its name to BannedCEs list."
for possibleNewSite in possibleNewSites:
body = "%s\n%s" % ( body, possibleNewSite )
self.log.info( body )
if self.addressTo and self.addressFrom:
notification = NotificationClient()
result = notification.sendMail( self.addressTo, self.subject, body, self.addressFrom, localAttempt = False )
return S_OK()
def __infoFromCE( self ):
sitesSection = cfgPath( 'Resources', 'Sites' )
result = gConfig.getSections( sitesSection )
if not result['OK']:
return
grids = result['Value']
changed = False
body = ""
for grid in grids:
gridSection = cfgPath( sitesSection, grid )
result = gConfig.getSections( gridSection )
if not result['OK']:
return
sites = result['Value']
for site in sites:
siteSection = cfgPath( gridSection, site )
opt = gConfig.getOptionsDict( siteSection )['Value']
name = opt.get( 'Name', '' )
if name:
coor = opt.get( 'Coordinates', 'Unknown' )
mail = opt.get( 'Mail', 'Unknown' )
result = ldapSite( name )
if not result['OK']:
self.log.warn( "BDII site %s: %s" % ( name, result['Message'] ) )
result = self.__checkAlternativeBDIISite( ldapSite, name )
if result['OK']:
bdiisites = result['Value']
if len( bdiisites ) == 0:
self.log.warn( name, "Error in bdii: leng = 0" )
else:
if not len( bdiisites ) == 1:
self.log.warn( name, "Warning in bdii: leng = %d" % len( bdiisites ) )
bdiisite = bdiisites[0]
try:
longitude = bdiisite['GlueSiteLongitude']
latitude = bdiisite['GlueSiteLatitude']
newcoor = "%s:%s" % ( longitude, latitude )
except:
self.log.warn( "Error in bdii coor" )
newcoor = "Unknown"
try:
newmail = bdiisite['GlueSiteSysAdminContact'].split( ":" )[-1].strip()
except:
self.log.warn( "Error in bdii mail" )
newmail = "Unknown"
self.log.debug( "%s %s %s" % ( name, newcoor, newmail ) )
if newcoor != coor:
self.log.info( "%s" % ( name ), "%s -> %s" % ( coor, newcoor ) )
if coor == 'Unknown':
self.csAPI.setOption( cfgPath( siteSection, 'Coordinates' ), newcoor )
else:
self.csAPI.modifyValue( cfgPath( siteSection, 'Coordinates' ), newcoor )
changed = True
if newmail != mail:
self.log.info( "%s" % ( name ), "%s -> %s" % ( mail, newmail ) )
if mail == 'Unknown':
self.csAPI.setOption( cfgPath( siteSection, 'Mail' ), newmail )
else:
self.csAPI.modifyValue( cfgPath( siteSection, 'Mail' ), newmail )
changed = True
celist = List.fromChar( opt.get( 'CE', '' ) )
if not celist:
self.log.warn( site, 'Empty site list' )
continue
# result = gConfig.getSections( cfgPath( siteSection,'CEs' )
# if not result['OK']:
# self.log.debug( "Section CEs:", result['Message'] )
for ce in celist:
ceSection = cfgPath( siteSection, 'CEs', ce )
result = gConfig.getOptionsDict( ceSection )
if not result['OK']:
self.log.debug( "Section CE", result['Message'] )
wnTmpDir = 'Unknown'
arch = 'Unknown'
os = 'Unknown'
si00 = 'Unknown'
pilot = 'Unknown'
cetype = 'Unknown'
else:
ceopt = result['Value']
wnTmpDir = ceopt.get( 'wnTmpDir', 'Unknown' )
arch = ceopt.get( 'architecture', 'Unknown' )
os = ceopt.get( 'OS', 'Unknown' )
si00 = ceopt.get( 'SI00', 'Unknown' )
pilot = ceopt.get( 'Pilot', 'Unknown' )
cetype = ceopt.get( 'CEType', 'Unknown' )
result = ldapCE( ce )
if not result['OK']:
self.log.warn( 'Error in bdii for %s' % ce, result['Message'] )
result = self.__checkAlternativeBDIISite( ldapCE, ce )
continue
try:
bdiice = result['Value'][0]
except:
self.log.warn( 'Error in bdii for %s' % ce, result )
bdiice = None
if bdiice:
try:
newwnTmpDir = bdiice['GlueSubClusterWNTmpDir']
except:
newwnTmpDir = 'Unknown'
if wnTmpDir != newwnTmpDir and newwnTmpDir != 'Unknown':
section = cfgPath( ceSection, 'wnTmpDir' )
self.log.info( section, " -> ".join( ( wnTmpDir, newwnTmpDir ) ) )
if wnTmpDir == 'Unknown':
self.csAPI.setOption( section, newwnTmpDir )
else:
self.csAPI.modifyValue( section, newwnTmpDir )
changed = True
try:
newarch = bdiice['GlueHostArchitecturePlatformType']
except:
newarch = 'Unknown'
if arch != newarch and newarch != 'Unknown':
section = cfgPath( ceSection, 'architecture' )
self.log.info( section, " -> ".join( ( arch, newarch ) ) )
if arch == 'Unknown':
self.csAPI.setOption( section, newarch )
else:
self.csAPI.modifyValue( section, newarch )
changed = True
try:
newos = '_'.join( ( bdiice['GlueHostOperatingSystemName'],
bdiice['GlueHostOperatingSystemVersion'],
bdiice['GlueHostOperatingSystemRelease'] ) )
except:
newos = 'Unknown'
if os != newos and newos != 'Unknown':
section = cfgPath( ceSection, 'OS' )
self.log.info( section, " -> ".join( ( os, newos ) ) )
if os == 'Unknown':
self.csAPI.setOption( section, newos )
else:
self.csAPI.modifyValue( section, newos )
changed = True
body = body + "OS was changed %s -> %s for %s at %s\n" % ( os, newos, ce, site )
try:
newsi00 = bdiice['GlueHostBenchmarkSI00']
except:
newsi00 = 'Unknown'
if si00 != newsi00 and newsi00 != 'Unknown':
section = cfgPath( ceSection, 'SI00' )
self.log.info( section, " -> ".join( ( si00, newsi00 ) ) )
if si00 == 'Unknown':
self.csAPI.setOption( section, newsi00 )
else:
self.csAPI.modifyValue( section, newsi00 )
changed = True
try:
rte = bdiice['GlueHostApplicationSoftwareRunTimeEnvironment']
if self.voName.lower() == 'lhcb':
if 'VO-lhcb-pilot' in rte:
newpilot = 'True'
else:
newpilot = 'False'
else:
newpilot = 'Unknown'
except:
newpilot = 'Unknown'
if pilot != newpilot and newpilot != 'Unknown':
section = cfgPath( ceSection, 'Pilot' )
self.log.info( section, " -> ".join( ( pilot, newpilot ) ) )
if pilot == 'Unknown':
self.csAPI.setOption( section, newpilot )
else:
self.csAPI.modifyValue( section, newpilot )
changed = True
result = ldapCEState( ce, vo = self.voName ) #getBDIICEVOView
if not result['OK']:
self.log.warn( 'Error in bdii for queue %s' % ce, result['Message'] )
result = self.__checkAlternativeBDIISite( ldapCEState, ce, self.voName )
continue
try:
queues = result['Value']
except:
self.log.warn( 'Error in bdii for queue %s' % ce, result['Massage'] )
continue
newcetype = 'Unknown'
for queue in queues:
try:
queuetype = queue['GlueCEImplementationName']
except:
queuetype = 'Unknown'
if newcetype == 'Unknown':
newcetype = queuetype
else:
if queuetype != newcetype:
self.log.warn( 'Error in bdii for ce %s ' % ce, 'different cetypes %s %s' % ( newcetype, queuetype ) )
if newcetype=='ARC-CE':
newcetype = 'ARC'
if cetype != newcetype and newcetype != 'Unknown':
section = cfgPath( ceSection, 'CEType' )
self.log.info( section, " -> ".join( ( cetype, newcetype ) ) )
if cetype == 'Unknown':
self.csAPI.setOption( section, newcetype )
else:
self.csAPI.modifyValue( section, newcetype )
changed = True
for queue in queues:
try:
queueName = queue['GlueCEUniqueID'].split( '/' )[-1]
except:
self.log.warn( 'error in queuename ', queue )
continue
try:
newmaxCPUTime = queue['GlueCEPolicyMaxCPUTime']
except:
newmaxCPUTime = None
newsi00 = None
try:
caps = queue['GlueCECapability']
if type( caps ) == type( '' ):
caps = [caps]
for cap in caps:
if cap.count( 'CPUScalingReferenceSI00' ):
newsi00 = cap.split( '=' )[-1]
except:
newsi00 = None
queueSection = cfgPath( ceSection, 'Queues', queueName )
result = gConfig.getOptionsDict( queueSection )
if not result['OK']:
self.log.warn( "Section Queues", result['Message'] )
maxCPUTime = 'Unknown'
si00 = 'Unknown'
else:
queueopt = result['Value']
maxCPUTime = queueopt.get( 'maxCPUTime', 'Unknown' )
si00 = queueopt.get( 'SI00', 'Unknown' )
if newmaxCPUTime and ( maxCPUTime != newmaxCPUTime ):
section = cfgPath( queueSection, 'maxCPUTime' )
self.log.info( section, " -> ".join( ( maxCPUTime, newmaxCPUTime ) ) )
if maxCPUTime == 'Unknown':
self.csAPI.setOption( section, newmaxCPUTime )
else:
self.csAPI.modifyValue( section, newmaxCPUTime )
changed = True
if newsi00 and ( si00 != newsi00 ):
section = cfgPath( queueSection, 'SI00' )
self.log.info( section, " -> ".join( ( si00, newsi00 ) ) )
if si00 == 'Unknown':
self.csAPI.setOption( section, newsi00 )
else:
self.csAPI.modifyValue( section, newsi00 )
changed = True
if changed:
self.log.info( body )
if body and self.addressTo and self.addressFrom:
notification = NotificationClient()
result = notification.sendMail( self.addressTo, self.subject, body, self.addressFrom, localAttempt = False )
return self.csAPI.commit()
else:
self.log.info( "No changes found" )
return S_OK()
class CE2CSAgent(AgentModule):
""" !!!Out-dated!!! Moved to Bdii2CSAgent
"""
addressTo = ''
addressFrom = ''
voName = ''
subject = "CE2CSAgent"
alternativeBDIIs = []
csAPI = None
def initialize(self):
# TODO: Have no default and if no mail is found then use the diracAdmin group
# and resolve all associated mail addresses.
self.addressTo = self.am_getOption('MailTo', self.addressTo)
self.addressFrom = self.am_getOption('MailFrom', self.addressFrom)
# Create a list of alternative bdii urls
self.alternativeBDIIs = self.am_getOption('AlternativeBDIIs', [])
# Check if the bdii url is appended by a port number, if not append the default 2170
for index, url in enumerate(self.alternativeBDIIs):
if not url.split(':')[-1].isdigit():
self.alternativeBDIIs[index] += ':2170'
if self.addressTo and self.addressFrom:
self.log.info("MailTo", self.addressTo)
self.log.info("MailFrom", self.addressFrom)
if self.alternativeBDIIs:
self.log.info("AlternativeBDII URLs:", self.alternativeBDIIs)
self.subject = "CE2CSAgent"
# This sets the Default Proxy to used as that defined under
# /Operations/Shifter/TestManager
# the shifterProxy option in the Configuration can be used to change this default.
self.am_setOption('shifterProxy', 'TestManager')
self.voName = self.am_getOption('VirtualOrganization', [])
if not self.voName:
vo = getVO()
if vo:
self.voName = [vo]
if self.voName:
self.log.info("Agent will manage VO(s) %s" % self.voName)
else:
self.log.fatal("VirtualOrganization option not defined for agent")
return S_ERROR()
self.csAPI = CSAPI()
return self.csAPI.initialize()
def execute(self):
self.log.info("Start Execution")
result = getProxyInfo()
if not result['OK']:
return result
infoDict = result['Value']
self.log.info(formatProxyInfoAsString(infoDict))
# Get a "fresh" copy of the CS data
result = self.csAPI.downloadCSData()
if not result['OK']:
self.log.warn("Could not download a fresh copy of the CS data",
result['Message'])
self.__lookForCE()
self.__infoFromCE()
self.log.info("End Execution")
return S_OK()
def __checkAlternativeBDIISite(self, fun, *args):
if self.alternativeBDIIs:
self.log.warn("Trying to use alternative BDII sites")
for site in self.alternativeBDIIs:
self.log.info("Trying to contact alternative BDII", site)
if len(args) == 1:
result = fun(args[0], host=site)
elif len(args) == 2:
result = fun(args[0], vo=args[1], host=site)
if not result['OK']:
self.log.error("Problem contacting alternative BDII",
result['Message'])
elif result['OK']:
return result
self.log.warn("Also checking alternative BDII sites failed")
return result
def __lookForCE(self):
knownCEs = self.am_getOption('BannedCEs', [])
result = gConfig.getSections('/Resources/Sites')
if not result['OK']:
return
grids = result['Value']
for grid in grids:
result = gConfig.getSections('/Resources/Sites/%s' % grid)
if not result['OK']:
return
sites = result['Value']
for site in sites:
opt = gConfig.getOptionsDict('/Resources/Sites/%s/%s' %
(grid, site))['Value']
ces = List.fromChar(opt.get('CE', ''))
knownCEs += ces
response = ''
for vo in self.voName:
self.log.info("Check for available CEs for VO", vo)
response = ldapCEState('', vo)
if not response['OK']:
self.log.error("Error during BDII request",
response['Message'])
response = self.__checkAlternativeBDIISite(ldapCEState, '', vo)
return response
newCEs = {}
for queue in response['Value']:
try:
queueName = queue['GlueCEUniqueID']
except:
continue
ceName = queueName.split(":")[0]
if not ceName in knownCEs:
newCEs[ceName] = None
self.log.debug("New CE", ceName)
body = ""
possibleNewSites = []
for ce in newCEs.iterkeys():
response = ldapCluster(ce)
if not response['OK']:
self.log.warn("Error during BDII request",
response['Message'])
response = self.__checkAlternativeBDIISite(ldapCluster, ce)
continue
clusters = {}
clusters = response['Value']
if len(clusters) != 1:
self.log.warn("Error in cluster length",
" CE %s Length %d" % (ce, len(clusters)))
if len(clusters) == 0:
continue
cluster = clusters[0]
fkey = cluster.get('GlueForeignKey', [])
if type(fkey) == type(''):
fkey = [fkey]
nameBDII = None
for entry in fkey:
if entry.count('GlueSiteUniqueID'):
nameBDII = entry.split('=')[1]
break
if not nameBDII:
continue
ceString = "CE: %s, GOCDB Name: %s" % (ce, nameBDII)
self.log.info(ceString)
response = ldapCE(ce)
if not response['OK']:
self.log.warn("Error during BDII request",
response['Message'])
response = self.__checkAlternativeBDIISite(ldapCE, ce)
continue
ceInfos = response['Value']
if len(ceInfos):
ceInfo = ceInfos[0]
systemName = ceInfo.get('GlueHostOperatingSystemName',
'Unknown')
systemVersion = ceInfo.get(
'GlueHostOperatingSystemVersion', 'Unknown')
systemRelease = ceInfo.get(
'GlueHostOperatingSystemRelease', 'Unknown')
else:
systemName = "Unknown"
systemVersion = "Unknown"
systemRelease = "Unknown"
osString = "SystemName: %s, SystemVersion: %s, SystemRelease: %s" % (
systemName, systemVersion, systemRelease)
self.log.info(osString)
response = ldapCEState(ce, vo)
if not response['OK']:
self.log.warn("Error during BDII request",
response['Message'])
response = self.__checkAlternativeBDIISite(
ldapCEState, ce, vo)
continue
newCEString = "\n\n%s\n%s" % (ceString, osString)
usefull = False
ceStates = response['Value']
for ceState in ceStates:
queueName = ceState.get('GlueCEUniqueID', 'UnknownName')
queueStatus = ceState.get('GlueCEStateStatus',
'UnknownStatus')
queueString = "%s %s" % (queueName, queueStatus)
self.log.info(queueString)
newCEString += "\n%s" % queueString
if queueStatus.count('Production'):
usefull = True
if usefull:
body += newCEString
possibleNewSites.append(
'dirac-admin-add-site DIRACSiteName %s %s' %
(nameBDII, ce))
if body:
body = "We are glad to inform You about new CE(s) possibly suitable for %s:\n" % vo + body
body += "\n\nTo suppress information about CE add its name to BannedCEs list."
for possibleNewSite in possibleNewSites:
body = "%s\n%s" % (body, possibleNewSite)
self.log.info(body)
if self.addressTo and self.addressFrom:
notification = NotificationClient()
result = notification.sendMail(self.addressTo,
self.subject,
body,
self.addressFrom,
localAttempt=False)
return S_OK()
def __infoFromCE(self):
sitesSection = cfgPath('Resources', 'Sites')
result = gConfig.getSections(sitesSection)
if not result['OK']:
return
grids = result['Value']
changed = False
body = ""
for grid in grids:
gridSection = cfgPath(sitesSection, grid)
result = gConfig.getSections(gridSection)
if not result['OK']:
return
sites = result['Value']
for site in sites:
siteSection = cfgPath(gridSection, site)
opt = gConfig.getOptionsDict(siteSection)['Value']
name = opt.get('Name', '')
if name:
coor = opt.get('Coordinates', 'Unknown')
mail = opt.get('Mail', 'Unknown')
result = ldapSite(name)
if not result['OK']:
self.log.warn("BDII site %s: %s" %
(name, result['Message']))
result = self.__checkAlternativeBDIISite(
ldapSite, name)
if result['OK']:
bdiiSites = result['Value']
if len(bdiiSites) == 0:
self.log.warn(name, "Error in BDII: leng = 0")
else:
if not len(bdiiSites) == 1:
self.log.warn(
name, "Warning in BDII: leng = %d" %
len(bdiiSites))
bdiiSite = bdiiSites[0]
try:
longitude = bdiiSite['GlueSiteLongitude']
latitude = bdiiSite['GlueSiteLatitude']
newcoor = "%s:%s" % (longitude, latitude)
except:
self.log.warn("Error in BDII coordinates")
newcoor = "Unknown"
try:
newmail = bdiiSite[
'GlueSiteSysAdminContact'].split(
":")[-1].strip()
except:
self.log.warn("Error in BDII mail")
newmail = "Unknown"
self.log.debug("%s %s %s" %
(name, newcoor, newmail))
if newcoor != coor:
self.log.info("%s" % (name),
"%s -> %s" % (coor, newcoor))
if coor == 'Unknown':
self.csAPI.setOption(
cfgPath(siteSection, 'Coordinates'),
newcoor)
else:
self.csAPI.modifyValue(
cfgPath(siteSection, 'Coordinates'),
newcoor)
changed = True
if newmail != mail:
self.log.info("%s" % (name),
"%s -> %s" % (mail, newmail))
if mail == 'Unknown':
self.csAPI.setOption(
cfgPath(siteSection, 'Mail'), newmail)
else:
self.csAPI.modifyValue(
cfgPath(siteSection, 'Mail'), newmail)
changed = True
ceList = List.fromChar(opt.get('CE', ''))
if not ceList:
self.log.warn(site, 'Empty site list')
continue
# result = gConfig.getSections( cfgPath( siteSection,'CEs' )
# if not result['OK']:
# self.log.debug( "Section CEs:", result['Message'] )
for ce in ceList:
ceSection = cfgPath(siteSection, 'CEs', ce)
result = gConfig.getOptionsDict(ceSection)
if not result['OK']:
self.log.debug("Section CE", result['Message'])
wnTmpDir = 'Unknown'
arch = 'Unknown'
os = 'Unknown'
si00 = 'Unknown'
pilot = 'Unknown'
ceType = 'Unknown'
else:
ceopt = result['Value']
wnTmpDir = ceopt.get('wnTmpDir', 'Unknown')
arch = ceopt.get('architecture', 'Unknown')
os = ceopt.get('OS', 'Unknown')
si00 = ceopt.get('SI00', 'Unknown')
pilot = ceopt.get('Pilot', 'Unknown')
ceType = ceopt.get('CEType', 'Unknown')
result = ldapCE(ce)
if not result['OK']:
self.log.warn('Error in BDII for %s' % ce,
result['Message'])
result = self.__checkAlternativeBDIISite(ldapCE, ce)
continue
try:
bdiiCE = result['Value'][0]
except:
self.log.warn('Error in BDII for %s' % ce, result)
bdiiCE = None
if bdiiCE:
try:
newWNTmpDir = bdiiCE['GlueSubClusterWNTmpDir']
except:
newWNTmpDir = 'Unknown'
if wnTmpDir != newWNTmpDir and newWNTmpDir != 'Unknown':
section = cfgPath(ceSection, 'wnTmpDir')
self.log.info(section, " -> ".join(
(wnTmpDir, newWNTmpDir)))
if wnTmpDir == 'Unknown':
self.csAPI.setOption(section, newWNTmpDir)
else:
self.csAPI.modifyValue(section, newWNTmpDir)
changed = True
try:
newArch = bdiiCE[
'GlueHostArchitecturePlatformType']
except:
newArch = 'Unknown'
if arch != newArch and newArch != 'Unknown':
section = cfgPath(ceSection, 'architecture')
self.log.info(section, " -> ".join(
(arch, newArch)))
if arch == 'Unknown':
self.csAPI.setOption(section, newArch)
else:
self.csAPI.modifyValue(section, newArch)
changed = True
try:
newOS = '_'.join(
(bdiiCE['GlueHostOperatingSystemName'],
bdiiCE['GlueHostOperatingSystemVersion'],
bdiiCE['GlueHostOperatingSystemRelease']))
except:
newOS = 'Unknown'
if os != newOS and newOS != 'Unknown':
section = cfgPath(ceSection, 'OS')
self.log.info(section, " -> ".join((os, newOS)))
if os == 'Unknown':
self.csAPI.setOption(section, newOS)
else:
self.csAPI.modifyValue(section, newOS)
changed = True
body = body + "OS was changed %s -> %s for %s at %s\n" % (
os, newOS, ce, site)
try:
newSI00 = bdiiCE['GlueHostBenchmarkSI00']
except:
newSI00 = 'Unknown'
if si00 != newSI00 and newSI00 != 'Unknown':
section = cfgPath(ceSection, 'SI00')
self.log.info(section, " -> ".join(
(si00, newSI00)))
if si00 == 'Unknown':
self.csAPI.setOption(section, newSI00)
else:
self.csAPI.modifyValue(section, newSI00)
changed = True
try:
rte = bdiiCE[
'GlueHostApplicationSoftwareRunTimeEnvironment']
for vo in self.voName:
if vo.lower() == 'lhcb':
if 'VO-lhcb-pilot' in rte:
newPilot = 'True'
else:
newPilot = 'False'
else:
newPilot = 'Unknown'
except:
newPilot = 'Unknown'
if pilot != newPilot and newPilot != 'Unknown':
section = cfgPath(ceSection, 'Pilot')
self.log.info(section, " -> ".join(
(pilot, newPilot)))
if pilot == 'Unknown':
self.csAPI.setOption(section, newPilot)
else:
self.csAPI.modifyValue(section, newPilot)
changed = True
newVO = ''
for vo in self.voName:
result = ldapCEState(ce, vo) #getBDIICEVOView
if not result['OK']:
self.log.warn('Error in BDII for queue %s' % ce,
result['Message'])
result = self.__checkAlternativeBDIISite(
ldapCEState, ce, vo)
continue
try:
queues = result['Value']
except:
self.log.warn('Error in BDII for queue %s' % ce,
result['Massage'])
continue
newCEType = 'Unknown'
for queue in queues:
try:
queueType = queue['GlueCEImplementationName']
except:
queueType = 'Unknown'
if newCEType == 'Unknown':
newCEType = queueType
else:
if queueType != newCEType:
self.log.warn(
'Error in BDII for CE %s ' % ce,
'different CE types %s %s' %
(newCEType, queueType))
if newCEType == 'ARC-CE':
newCEType = 'ARC'
if ceType != newCEType and newCEType != 'Unknown':
section = cfgPath(ceSection, 'CEType')
self.log.info(section, " -> ".join(
(ceType, newCEType)))
if ceType == 'Unknown':
self.csAPI.setOption(section, newCEType)
else:
self.csAPI.modifyValue(section, newCEType)
changed = True
for queue in queues:
try:
queueName = queue['GlueCEUniqueID'].split(
'/')[-1]
except:
self.log.warn('Error in queueName ', queue)
continue
try:
newMaxCPUTime = queue['GlueCEPolicyMaxCPUTime']
except:
newMaxCPUTime = None
newSI00 = None
try:
caps = queue['GlueCECapability']
if type(caps) == type(''):
caps = [caps]
for cap in caps:
if cap.count('CPUScalingReferenceSI00'):
newSI00 = cap.split('=')[-1]
except:
newSI00 = None
queueSection = cfgPath(ceSection, 'Queues',
queueName)
result = gConfig.getOptionsDict(queueSection)
if not result['OK']:
self.log.warn("Section Queues",
result['Message'])
maxCPUTime = 'Unknown'
si00 = 'Unknown'
allowedVOs = ['']
else:
queueOpt = result['Value']
maxCPUTime = queueOpt.get(
'maxCPUTime', 'Unknown')
si00 = queueOpt.get('SI00', 'Unknown')
if newVO == '': # Remember previous iteration, if none - read from conf
allowedVOs = queueOpt.get('VO',
'').split(",")
else: # Else use newVO, as it can contain changes, which aren't in conf yet
allowedVOs = newVO.split(",")
if newMaxCPUTime and (maxCPUTime != newMaxCPUTime):
section = cfgPath(queueSection, 'maxCPUTime')
self.log.info(
section, " -> ".join(
(maxCPUTime, newMaxCPUTime)))
if maxCPUTime == 'Unknown':
self.csAPI.setOption(
section, newMaxCPUTime)
else:
self.csAPI.modifyValue(
section, newMaxCPUTime)
changed = True
if newSI00 and (si00 != newSI00):
section = cfgPath(queueSection, 'SI00')
self.log.info(section, " -> ".join(
(si00, newSI00)))
if si00 == 'Unknown':
self.csAPI.setOption(section, newSI00)
else:
self.csAPI.modifyValue(section, newSI00)
changed = True
modifyVO = True # Flag saying if we need VO option to change
newVO = ''
if allowedVOs != ['']:
for allowedVO in allowedVOs:
allowedVO = allowedVO.strip(
) # Get rid of spaces
newVO += allowedVO
if allowedVO == vo: # Current VO has been already in list
newVO = ''
modifyVO = False # Don't change anything
break # Skip next 'if', proceed to next VO
newVO += ', '
if modifyVO:
section = cfgPath(queueSection, 'VO')
newVO += vo
self.log.info(
section, " -> ".join(
('%s' % allowedVOs, newVO)))
if allowedVOs == ['']:
self.csAPI.setOption(section, newVO)
else:
self.csAPI.modifyValue(section, newVO)
changed = True
if changed:
self.log.info(body)
if body and self.addressTo and self.addressFrom:
notification = NotificationClient()
result = notification.sendMail(self.addressTo,
self.subject,
body,
self.addressFrom,
localAttempt=False)
return self.csAPI.commit()
else:
self.log.info("No changes found")
return S_OK()
class StorageMonitorAgent( AgentModule ):
addressTo = ''
addressFrom = ''
voName = ''
bdii = 'lcg-bdii.cern.ch:2170'
productionSEs = []
subject = "StorageMonitorAgent"
def initialize( self ):
self.addressTo = self.am_getOption( 'MailTo', self.addressTo )
self.addressFrom = self.am_getOption( 'MailFrom', self.addressFrom )
if self.addressTo and self.addressFrom:
self.log.info( "MailTo", self.addressTo )
self.log.info( "MailFrom", self.addressFrom )
self.productionSEs = self.am_getOption( 'ProductionSEs', self.productionSEs)
if self.productionSEs:
self.log.info( "ProductionSEs", self.productionSEs )
else:
self.log.fatal( "ProductionSEs option not defined for agent" )
return S_ERROR()
self.voName = self.am_getOption( 'VirtualOrganization', self.voName )
if self.voName:
self.log.info( "Agent will manage VO", self.voName )
else:
self.log.fatal( "VirtualOrganization option not defined for agent" )
return S_ERROR()
self.bdii = self.am_getOption( 'Bdii', self.bdii )
if self.bdii:
self.log.info( "Bdii", self.bdii )
self.csAPI = CSAPI()
return self.csAPI.initialize()
def execute( self ):
""" General agent execution method
"""
# Get a "fresh" copy of the CS data
result = self.csAPI.downloadCSData()
if not result['OK']:
self.log.warn( "Could not download a fresh copy of the CS data", result[ 'Message' ] )
# Execute command to retrieve storage usage information
cmdTuple = ['lcg-infosites', '--vo', self.voName , 'se']
self.log.info( "Executing %s" % cmdTuple )
ret = systemCall( 0, cmdTuple, env = {'LCG_GFAL_INFOSYS':self.bdii} )
if not ret['OK']:
return ret
elif not ret['Value'][1] != '':
self.log.error( "Error while executing %s" % cmdTuple )
return S_ERROR()
# initialize sedict
sedict = {}
for SE in self.productionSEs:
sedict[SE] = []
fields = ['SE', 'Available(TB)', 'Used(TB)', 'Total(TB)', 'Used(%)' ]
records = []
fullSEList = []
for se in ret['Value'][1].split( '\n' ):
if len( se.split() ) == 4:
spacedict = {}
SE = se.split()[3]
if SE in self.productionSEs and se.split()[0] != 'n.a' and se.split()[1] != 'n.a':
# ## convert into TB
available = float( se.split()[0] ) / 1e9
used = float( se.split()[1] ) / 1e9
spacedict['Available'] = available
spacedict['Used'] = used
spacedict['Total'] = available + used
sedict[SE].append(spacedict)
for SE in self.productionSEs:
for spacedict in sedict[SE]:
available = '%.1f' % spacedict['Available']
used = '%.1f' % spacedict['Used']
total = '%.1f' % spacedict['Total']
fraction_used = spacedict['Used'] / spacedict['Total'] * 100
if fraction_used > 90.:
fullSEList.append(SE)
self.log.warn( "%s full at %.1f%%" % (SE, fraction_used) )
fraction_used = '%.1f' % fraction_used
records.append( [SE, available, used, total, fraction_used] )
body = printTable( fields, records, printOut = False )
if len(fullSEList) > 0:
self.subject = 'CRITICAL storage usage beyond 90%%: %s' % ( ', '.join( fullSEList ) )
if self.addressTo and self.addressFrom:
notification = NotificationClient()
result = notification.sendMail( self.addressTo, self.subject, body, self.addressFrom, localAttempt = False )
if not result['OK']:
self.log.error( 'Can not send notification mail', result['Message'] )
return S_OK()
def __syncCSWithVOMS( self ):
self.__adminMsgs = { 'Errors' : [], 'Info' : [] }
#Get DIRAC VOMS Mapping
self.log.info( "Getting DIRAC VOMS mapping" )
mappingSection = '/Registry/VOMS/Mapping'
ret = gConfig.getOptionsDict( mappingSection )
if not ret['OK']:
self.log.fatal( 'No VOMS to DIRAC Group Mapping Available' )
return ret
vomsMapping = ret['Value']
self.log.info( "There are %s registered voms mappings in DIRAC" % len( vomsMapping ) )
#Get VOMS VO name
self.log.info( "Getting VOMS VO name" )
result = self.vomsSrv.admGetVOName()
if not ret['OK']:
self.log.fatal( 'Could not retrieve VOMS VO name' )
voNameInVOMS = result[ 'Value' ]
self.log.info( "VOMS VO Name is %s" % voNameInVOMS )
#Get VOMS roles
self.log.info( "Getting the list of registered roles in VOMS" )
result = self.vomsSrv.admListRoles()
if not ret['OK']:
self.log.fatal( 'Could not retrieve registered roles in VOMS' )
rolesInVOMS = result[ 'Value' ]
self.log.info( "There are %s registered roles in VOMS" % len( rolesInVOMS ) )
print rolesInVOMS
rolesInVOMS.append( '' )
#Map VOMS roles
vomsRoles = {}
for role in rolesInVOMS:
if role:
role = "%s/%s" % ( voNameInVOMS, role )
else:
role = voNameInVOMS
groupsForRole = []
for group in vomsMapping:
if vomsMapping[ group ] == role:
groupsForRole.append( group )
if groupsForRole:
vomsRoles[ role ] = { 'Groups' : groupsForRole, 'Users' : [] }
self.log.info( "DIRAC valid VOMS roles are:\n\t", "\n\t ".join( vomsRoles.keys() ) )
#Get DIRAC users
self.log.info( "Getting the list of registered users in DIRAC" )
csapi = CSAPI()
ret = csapi.listUsers()
if not ret['OK']:
self.log.fatal( 'Could not retrieve current list of Users' )
return ret
currentUsers = ret['Value']
ret = csapi.describeUsers( currentUsers )
if not ret['OK']:
self.log.fatal( 'Could not retrieve current User description' )
return ret
currentUsers = ret['Value']
self.__adminMsgs[ 'Info' ].append( "There are %s registered users in DIRAC" % len( currentUsers ) )
self.log.info( "There are %s registered users in DIRAC" % len( currentUsers ) )
#Get VOMS user entries
self.log.info( "Getting the list of registered user entries in VOMS" )
result = self.vomsSrv.admListMembers()
if not ret['OK']:
self.log.fatal( 'Could not retrieve registered user entries in VOMS' )
usersInVOMS = result[ 'Value' ]
self.__adminMsgs[ 'Info' ].append( "There are %s registered user entries in VOMS" % len( usersInVOMS ) )
self.log.info( "There are %s registered user entries in VOMS" % len( usersInVOMS ) )
#Consolidate users by nickname
usersData = {}
newUserNames = []
knownUserNames = []
obsoleteUserNames = []
self.log.info( "Retrieving usernames..." )
usersInVOMS.sort()
for iUPos in range( len( usersInVOMS ) ):
userName = ''
user = usersInVOMS[ iUPos ]
for oldUser in currentUsers:
if user[ 'DN' ].strip() in List.fromChar( currentUsers[oldUser][ 'DN' ] ):
userName = oldUser
if not userName:
result = self.vomsSrv.attGetUserNickname( user[ 'DN' ], user[ 'CA' ] )
if result[ 'OK' ]:
userName = result[ 'Value' ]
else:
self.__adminMsgs[ 'Errors' ].append( "Could not retrieve nickname for DN %s" % user[ 'DN' ] )
self.log.error( "Could not get nickname for DN", user[ 'DN' ] )
userName = user[ 'mail' ][:user[ 'mail' ].find( '@' )]
if not userName:
self.log.error( "Empty nickname for DN", user[ 'DN' ] )
self.__adminMsgs[ 'Errors' ].append( "Empty nickname for DN %s" % user[ 'DN' ] )
continue
self.log.info( " (%02d%%) Found username %s : %s " % ( ( iUPos * 100 / len( usersInVOMS ) ), userName, user[ 'DN' ] ) )
if userName not in usersData:
usersData[ userName ] = { 'DN': [], 'CA': [], 'Email': [], 'Groups' : ['user'] }
for key in ( 'DN', 'CA', 'mail' ):
value = user[ key ]
if value:
if key == "mail":
List.appendUnique( usersData[ userName ][ 'Email' ], value )
else:
usersData[ userName ][ key ].append( value.strip() )
if userName not in currentUsers:
List.appendUnique( newUserNames, userName )
else:
List.appendUnique( knownUserNames, userName )
self.log.info( "Finished retrieving usernames" )
if newUserNames:
self.log.info( "There are %s new users" % len( newUserNames ) )
else:
self.log.info( "There are no new users" )
#Get the list of users for each group
result = csapi.listGroups()
if not result[ 'OK' ]:
self.log.error( "Could not get the list of groups in DIRAC", result[ 'Message' ] )
return result
staticGroups = result[ 'Value' ]
vomsGroups = []
self.log.info( "Mapping users in VOMS to groups" )
for vomsRole in vomsRoles:
self.log.info( " Getting users for role %s" % vomsRole )
groupsForRole = vomsRoles[ vomsRole ][ 'Groups' ]
vomsMap = vomsRole.split( "Role=" )
for g in groupsForRole:
if g in staticGroups:
staticGroups.pop( staticGroups.index( g ) )
else:
vomsGroups.append( g )
if len( vomsMap ) == 1:
# no Role
users = usersInVOMS
else:
vomsGroup = "Role=".join( vomsMap[:-1] )
if vomsGroup[-1] == "/":
vomsGroup = vomsGroup[:-1]
vomsRole = "Role=%s" % vomsMap[-1]
result = self.vomsSrv.admListUsersWithRole( vomsGroup, vomsRole )
if not result[ 'OK' ]:
errorMsg = "Could not get list of users for VOMS %s" % ( vomsMapping[ group ] )
self.__adminMsgs[ 'Errors' ].append( errorMsg )
self.log.error( errorMsg, result[ 'Message' ] )
return result
users = result['Value']
numUsersInGroup = 0
for vomsUser in users:
for userName in usersData:
if vomsUser[ 'DN' ] in usersData[ userName ][ 'DN' ]:
numUsersInGroup += 1
usersData[ userName ][ 'Groups' ].extend( groupsForRole )
infoMsg = "There are %s users in group(s) %s for VOMS Role %s" % ( numUsersInGroup, ",".join( groupsForRole ), vomsRole )
self.__adminMsgs[ 'Info' ].append( infoMsg )
self.log.info( " %s" % infoMsg )
self.log.info( "Checking static groups" )
staticUsers = []
for group in staticGroups:
self.log.info( " Checking static group %s" % group )
numUsersInGroup = 0
result = csapi.listUsers( group )
if not result[ 'OK' ]:
self.log.error( "Could not get the list of users in DIRAC group %s" % group , result[ 'Message' ] )
return result
for userName in result[ 'Value' ]:
if userName in usersData:
numUsersInGroup += 1
usersData[ userName ][ 'Groups' ].append( group )
else:
if group not in vomsGroups and userName not in staticUsers:
staticUsers.append( userName )
infoMsg = "There are %s users in group %s" % ( numUsersInGroup, group )
self.__adminMsgs[ 'Info' ].append( infoMsg )
self.log.info( " %s" % infoMsg )
if staticUsers:
infoMsg = "There are %s static users: %s" % ( len( staticUsers ) , ', '.join( staticUsers ) )
self.__adminMsgs[ 'Info' ].append( infoMsg )
self.log.info( "%s" % infoMsg )
for user in currentUsers:
if user not in usersData and user not in staticUsers:
self.log.info( 'User %s is no longer valid' % user )
obsoleteUserNames.append( user )
#Do the CS Sync
self.log.info( "Updating CS..." )
ret = csapi.downloadCSData()
if not ret['OK']:
self.log.fatal( 'Can not update from CS', ret['Message'] )
return ret
usersWithMoreThanOneDN = {}
for user in usersData:
csUserData = dict( usersData[ user ] )
if len( csUserData[ 'DN' ] ) > 1:
usersWithMoreThanOneDN[ user ] = csUserData[ 'DN' ]
result = csapi.describeUsers( [ user ] )
if result[ 'OK' ]:
if result[ 'Value' ]:
prevUser = result[ 'Value' ][ user ]
prevDNs = List.fromChar( prevUser[ 'DN' ] )
newDNs = csUserData[ 'DN' ]
for DN in newDNs:
if DN not in prevDNs:
self.__adminMsgs[ 'Info' ].append( "User %s has new DN %s" % ( user, DN ) )
for DN in prevDNs:
if DN not in newDNs:
self.__adminMsgs[ 'Info' ].append( "User %s has lost a DN %s" % ( user, DN ) )
else:
newDNs = csUserData[ 'DN' ]
for DN in newDNs:
self.__adminMsgs[ 'Info' ].append( "New user %s has new DN %s" % ( user, DN ) )
for k in ( 'DN', 'CA', 'Email' ):
csUserData[ k ] = ", ".join( csUserData[ k ] )
result = csapi.modifyUser( user, csUserData, createIfNonExistant = True )
if not result[ 'OK' ]:
self.__adminMsgs[ 'Error' ].append( "Cannot modify user %s: %s" % ( user, result[ 'Message' ] ) )
self.log.error( "Cannot modify user", user )
if usersWithMoreThanOneDN:
self.__adminMsgs[ 'Info' ].append( "\nUsers with more than one DN:" )
for uwmtod in sorted( usersWithMoreThanOneDN ):
self.__adminMsgs[ 'Info' ].append( " %s" % uwmtod )
self.__adminMsgs[ 'Info' ].append( " + DN list:" )
for DN in usersWithMoreThanOneDN[uwmtod]:
self.__adminMsgs[ 'Info' ].append( " - %s" % DN )
if obsoleteUserNames:
self.__adminMsgs[ 'Info' ].append( "\nObsolete users:" )
address = self.am_getOption( 'MailTo', '*****@*****.**' )
fromAddress = self.am_getOption( 'mailFrom', '*****@*****.**' )
subject = 'Obsolete LFC Users found'
body = 'Delete entries into LFC: \n'
for obsoleteUser in obsoleteUserNames:
self.log.info( subject, ", ".join( obsoleteUserNames ) )
body += 'for ' + obsoleteUser + '\n'
self.__adminMsgs[ 'Info' ].append( " %s" % obsoleteUser )
self.log.info( "Deleting %s users" % len( obsoleteUserNames ) )
NotificationClient().sendMail( address, 'UsersAndGroupsAgent: %s' % subject, body, fromAddress )
csapi.deleteUsers( obsoleteUserNames )
if newUserNames:
self.__adminMsgs[ 'Info' ].append( "\nNew users:" )
for newUser in newUserNames:
self.__adminMsgs[ 'Info' ].append( " %s" % newUser )
self.__adminMsgs[ 'Info' ].append( " + DN list:" )
for DN in usersData[newUser][ 'DN' ]:
self.__adminMsgs[ 'Info' ].append( " - %s" % DN )
self.__adminMsgs[ 'Info' ].append( " + EMail: %s" % usersData[newUser][ 'Email' ] )
result = csapi.commitChanges()
if not result[ 'OK' ]:
self.log.error( "Could not commit configuration changes", result[ 'Message' ] )
return result
self.log.info( "Configuration committed" )
#LFC Check
if self.am_getOption( "LFCCheckEnabled", True ):
result = self.checkLFCRegisteredUsers( usersData )
if not result[ 'OK' ]:
return result
return S_OK()
class Bdii2CSAgent(AgentModule):
def __init__(self, *args, **kwargs):
""" Defines default parameters
"""
super(Bdii2CSAgent, self).__init__(*args, **kwargs)
self.addressTo = ''
self.addressFrom = ''
self.voName = []
self.subject = self.am_getModuleParam('fullName')
self.alternativeBDIIs = []
self.voBdiiCEDict = {}
self.voBdiiSEDict = {}
self.host = 'lcg-bdii.cern.ch:2170'
self.glue2URLs = []
self.glue2Only = True
self.csAPI = None
# What to get
self.processCEs = True
self.selectedSites = []
# Update the CS or not?
self.dryRun = False
def initialize(self):
""" Gets run paramaters from the configuration
"""
self.addressTo = self.am_getOption('MailTo', self.addressTo)
self.addressFrom = self.am_getOption('MailFrom', self.addressFrom)
# Create a list of alternative bdii urls
self.alternativeBDIIs = self.am_getOption('AlternativeBDIIs', self.alternativeBDIIs)
self.host = self.am_getOption('Host', self.host)
self.glue2URLs = self.am_getOption('GLUE2URLs', self.glue2URLs)
self.glue2Only = self.am_getOption('GLUE2Only', self.glue2Only)
# Check if the bdii url is appended by a port number, if not append the default 2170
for index, url in enumerate(self.alternativeBDIIs):
if not url.split(':')[-1].isdigit():
self.alternativeBDIIs[index] += ':2170'
if self.addressTo and self.addressFrom:
self.log.info("MailTo", self.addressTo)
self.log.info("MailFrom", self.addressFrom)
if self.alternativeBDIIs:
self.log.info("AlternativeBDII URLs:", self.alternativeBDIIs)
self.processCEs = self.am_getOption('ProcessCEs', self.processCEs)
self.selectedSites = self.am_getOption('SelectedSites', [])
self.dryRun = self.am_getOption('DryRun', self.dryRun)
self.voName = self.am_getOption('VirtualOrganization', self.voName)
if not self.voName:
self.voName = self.am_getOption('VO', [])
if not self.voName or (len(self.voName) == 1 and self.voName[0].lower() == 'all'):
# Get all VOs defined in the configuration
self.voName = []
result = getVOs()
if result['OK']:
vos = result['Value']
for vo in vos:
vomsVO = getVOOption(vo, "VOMSName")
if vomsVO:
self.voName.append(vomsVO)
if self.voName:
self.log.info("Agent will manage VO(s) %s" % self.voName)
else:
self.log.fatal("VirtualOrganization option not defined for agent")
return S_ERROR()
self.csAPI = CSAPI()
return self.csAPI.initialize()
def execute(self):
""" General agent execution method
"""
self.voBdiiCEDict = {}
# Get a "fresh" copy of the CS data
result = self.csAPI.downloadCSData()
if not result['OK']:
self.log.warn("Could not download a fresh copy of the CS data", result['Message'])
# Refresh the configuration from the master server
gConfig.forceRefresh(fromMaster=True)
if self.processCEs:
self.__lookForNewCEs()
self.__updateCEs()
return S_OK()
def __lookForNewCEs(self):
""" Look up BDII for CEs not yet present in the DIRAC CS
"""
bannedCEs = self.am_getOption('BannedCEs', [])
for vo in self.voName:
# get the known CEs for a given VO, so we can know the unknowns, or no longer supported,
# for a VO
res = getQueues(community=vo)
if not res['OK']:
return res
knownCEs = set()
for _site, ces in res['Value'].items():
knownCEs.update(ces)
knownCEs.update(bannedCEs)
result = self.__getBdiiCEInfo(vo)
if not result['OK']:
continue
bdiiInfo = result['Value']
result = getGridCEs(vo, bdiiInfo=bdiiInfo, ceBlackList=knownCEs)
if not result['OK']:
self.log.error('Failed to get unused CEs', result['Message'])
continue # next VO
siteDict = result['Value']
unknownCEs = set(result['UnknownCEs']) - set(bannedCEs)
body = ''
for site in siteDict:
newCEs = set(siteDict[site]) # pylint: disable=no-member
if not newCEs:
continue
ceString = ''
for ce in newCEs:
queueString = ''
ceInfo = bdiiInfo[site]['CEs'][ce]
newCEString = "CE: %s, GOCDB Site Name: %s" % (ce, site)
systemTuple = siteDict[site][ce]['System']
osString = "%s_%s_%s" % (systemTuple)
newCEString = "\n%s\n%s\n" % (newCEString, osString)
for queue in ceInfo['Queues']:
queueStatus = ceInfo['Queues'][queue].get('GlueCEStateStatus', 'UnknownStatus')
if 'production' in queueStatus.lower():
ceType = ceInfo['Queues'][queue].get('GlueCEImplementationName', '')
queueString += " %s %s %s\n" % (queue, queueStatus, ceType)
if queueString:
ceString += newCEString
ceString += "Queues:\n"
ceString += queueString
if ceString:
body += ceString
if siteDict:
body = "\nWe are glad to inform You about new CE(s) possibly suitable for %s:\n" % vo + body
body += "\n\nTo suppress information about CE add its name to BannedCEs list.\n"
body += "Add new Sites/CEs for vo %s with the command:\n" % vo
body += "dirac-admin-add-resources --vo %s --ce\n" % vo
if unknownCEs:
body += '\n\n'
body += 'There is no (longer) information about the following CEs for the %s VO.\n' % vo
body += '\n'.join(sorted(unknownCEs))
body += '\n\n'
if body:
self.log.info(body)
if self.addressTo and self.addressFrom:
notification = NotificationClient()
result = notification.sendMail(self.addressTo, self.subject, body, self.addressFrom,
localAttempt=False, avoidSpam=True)
if not result['OK']:
self.log.error('Can not send new site notification mail', result['Message'])
return S_OK()
def __getBdiiCEInfo(self, vo):
if vo in self.voBdiiCEDict:
return S_OK(self.voBdiiCEDict[vo])
self.log.info("Check for available CEs for VO", vo)
totalResult = S_OK({})
message = ''
mainResult = getBdiiCEInfo(vo, host=self.host, glue2=self.glue2Only)
if not mainResult['OK']:
self.log.error("Failed getting information from default bdii", mainResult['Message'])
message = mainResult['Message']
for bdii in reversed(self.alternativeBDIIs):
resultAlt = getBdiiCEInfo(vo, host=bdii, glue2=self.glue2Only)
if resultAlt['OK']:
totalResult['Value'].update(resultAlt['Value'])
else:
self.log.error("Failed getting information from %s " % bdii, resultAlt['Message'])
message = (message + "\n" + resultAlt['Message']).strip()
for glue2URL in self.glue2URLs:
if self.glue2Only:
break
resultGlue2 = getBdiiCEInfo(vo, host=glue2URL, glue2=True)
if resultGlue2['OK']:
totalResult['Value'].update(resultGlue2['Value'])
else:
self.log.error("Failed getting GLUE2 information for", "%s, %s: %s" %
(glue2URL, vo, resultGlue2['Message']))
message = (message + "\n" + resultGlue2['Message']).strip()
if mainResult['OK']:
totalResult['Value'].update(mainResult['Value'])
if not totalResult['Value'] and message: # Dict is empty and we have an error message
self.log.error("Error during BDII request", message)
totalResult = S_ERROR(message)
else:
self.voBdiiCEDict[vo] = totalResult['Value']
self.__purgeSites(totalResult['Value'])
return totalResult
def __updateCEs(self):
""" Update the Site/CE/queue settings in the CS if they were changed in the BDII
"""
bdiiChangeSet = set()
for vo in self.voName:
result = self.__getBdiiCEInfo(vo)
if not result['OK']:
continue
ceBdiiDict = result['Value']
result = getSiteUpdates(vo, bdiiInfo=ceBdiiDict, log=self.log)
if not result['OK']:
continue
bdiiChangeSet = bdiiChangeSet.union(result['Value'])
# We have collected all the changes, consolidate VO settings
result = self.__updateCS(bdiiChangeSet)
return result
def __purgeSites(self, ceBdiiDict):
"""Remove all sites that are not in self.selectedSites.
Modifies the ceBdiiDict!
"""
if not self.selectedSites:
return
for site in list(ceBdiiDict):
ces = list(ceBdiiDict[site]['CEs'])
if not ces:
self.log.error("No CE information for site:", site)
continue
siteInCS = 'Not_In_CS'
for ce in ces:
res = getCESiteMapping(ce)
if not res['OK']:
self.log.error("Failed to get DIRAC site name for ce", "%s: %s" % (ce, res['Message']))
continue
# if the ce is not in the CS the returned value will be empty
if ce in res['Value']:
siteInCS = res['Value'][ce]
break
self.log.debug("Checking site %s (%s), aka %s" % (site, ces, siteInCS))
if siteInCS in self.selectedSites:
continue
self.log.info("Dropping site %s, aka %s" % (site, siteInCS))
ceBdiiDict.pop(site)
return
def __updateCS(self, bdiiChangeSet):
queueVODict = {}
changeSet = set()
for entry in bdiiChangeSet:
section, option, _value, new_value = entry
if option == "VO":
queueVODict.setdefault(section, set())
queueVODict[section] = queueVODict[section].union(set(new_value.split(',')))
else:
changeSet.add(entry)
for section, VOs in queueVODict.items(): # can be an iterator
changeSet.add((section, 'VO', '', ','.join(VOs)))
if changeSet:
changeList = sorted(changeSet)
body = '\n'.join(["%s/%s %s -> %s" % entry for entry in changeList])
if body and self.addressTo and self.addressFrom:
notification = NotificationClient()
result = notification.sendMail(self.addressTo, self.subject, body, self.addressFrom, localAttempt=False,
avoidSpam=True)
if body:
self.log.info('The following configuration changes were detected:')
self.log.info(body)
for section, option, value, new_value in changeSet:
if value == 'Unknown' or not value:
self.csAPI.setOption(cfgPath(section, option), new_value)
else:
self.csAPI.modifyValue(cfgPath(section, option), new_value)
if self.dryRun:
self.log.info("Dry Run: CS won't be updated")
self.csAPI.showDiff()
else:
result = self.csAPI.commit()
if not result['OK']:
self.log.error("Error while committing to CS", result['Message'])
else:
self.log.info("Successfully committed %d changes to CS" % len(changeList))
return result
else:
self.log.info("No changes found")
return S_OK()
class VOMS2CSAgent(AgentModule):
def __init__(self, *args, **kwargs):
""" Defines default parameters
"""
super(VOMS2CSAgent, self).__init__(*args, **kwargs)
self.__voDict = {}
self.__adminMsgs = {}
self.csapi = None
self.voChanged = False
self.dryRun = False
self.autoAddUsers = False
self.autoModifyUsers = False
self.autoDeleteUsers = False
self.detailedReport = True
self.makeFCEntry = False
def initialize(self):
""" Initialize the default parameters
"""
voNames = self.am_getOption('VO', [])
if not voNames[0].lower() == "none":
if voNames[0].lower() == "any":
voNames = []
result = getVOMSVOs(voNames)
if not result['OK']:
return result
self.__voDict = result['Value']
self.log.notice("VOs: %s" % self.__voDict.keys())
self.csapi = CSAPI()
self.dryRun = self.am_getOption('DryRun', self.dryRun)
self.autoAddUsers = self.am_getOption('AutoAddUsers',
self.autoAddUsers)
self.autoModifyUsers = self.am_getOption('AutoModifyUsers',
self.autoModifyUsers)
self.autoDeleteUsers = self.am_getOption('AutoDeleteUsers',
self.autoDeleteUsers)
self.detailedReport = self.am_getOption('DetailedReport',
self.detailedReport)
self.makeFCEntry = self.am_getOption('MakeHomeDirectory',
self.makeFCEntry)
return S_OK()
def execute(self):
self.__adminMsgs = {}
self.csapi.downloadCSData()
for vo in self.__voDict:
self.voChanged = False
voAdminUser = getVOOption(vo, "VOAdmin")
voAdminMail = None
if voAdminUser:
voAdminMail = getUserOption(voAdminUser, "Email")
voAdminGroup = getVOOption(vo, "VOAdminGroup",
getVOOption(vo, "DefaultGroup"))
self.log.info(
'Performing VOMS sync for VO %s with credentials %s@%s' %
(vo, voAdminUser, voAdminGroup))
result = self.__syncCSWithVOMS(vo,
proxyUserName=voAdminUser,
proxyUserGroup=voAdminGroup) #pylint: disable=unexpected-keyword-arg
if not result['OK']:
self.log.error('Failed to perform VOMS to CS synchronization:',
'VO %s: %s' % (vo, result["Message"]))
continue
resultDict = result['Value']
newUsers = resultDict.get("NewUsers", [])
modUsers = resultDict.get("ModifiedUsers", [])
delUsers = resultDict.get("DeletedUsers", [])
self.log.info( "Run results: new users %d, modified users %d, deleted users %d" % \
( len( newUsers ), len( modUsers ), len( delUsers ) ) )
if self.csapi.csModified:
# We have accumulated all the changes, commit them now
self.log.info(
"There are changes to the CS for vo %s ready to be committed"
% vo)
if self.dryRun:
self.log.info("Dry Run: CS won't be updated")
self.csapi.showDiff()
else:
result = self.csapi.commitChanges()
if not result['OK']:
self.log.error(
"Could not commit configuration changes",
result['Message'])
return result
self.log.notice("Configuration committed for VO %s" % vo)
else:
self.log.info(
"No changes to the CS for VO %s recorded at this cycle" %
vo)
# Add user home directory in the file catalog
if self.makeFCEntry and newUsers:
self.log.info("Creating home directories for users %s" %
str(newUsers))
result = self.__addHomeDirectory(vo,
newUsers,
proxyUserName=voAdminUser,
proxyUserGroup=voAdminGroup) #pylint: disable=unexpected-keyword-arg
if not result['OK']:
self.log.error('Failed to create user home directories:',
'VO %s: %s' % (vo, result["Message"]))
else:
for user in result['Value']['Failed']:
self.log.error( "Failed to create home directory", "user: %s, operation: %s" % \
( user, result['Value']['Failed'][user] ) )
self.__adminMsgs[ 'Errors' ].append( "Failed to create home directory for user %s: operation %s" % \
( user, result['Value']['Failed'][user] ) )
for user in result['Value']['Successful']:
self.__adminMsgs['Info'].append(
"Created home directory for user %s" % user)
if self.voChanged or self.detailedReport:
mailMsg = ""
if self.__adminMsgs['Errors']:
mailMsg += "\nErrors list:\n %s" % "\n ".join(
self.__adminMsgs['Errors'])
if self.__adminMsgs['Info']:
mailMsg += "\nRun result:\n %s" % "\n ".join(
self.__adminMsgs['Info'])
if self.detailedReport:
result = self.getVOUserReport(vo)
if result['OK']:
mailMsg += '\n\n'
mailMsg += result['Value']
else:
mailMsg += 'Failed to produce a detailed user report'
mailMsg += result['Message']
NotificationClient().sendMail(
self.am_getOption('MailTo', voAdminMail),
"VOMS2CSAgent run log", mailMsg,
self.am_getOption(
'MailFrom',
self.am_getOption('mailFrom', "DIRAC system")))
return S_OK()
def getVOUserData(self, vo, refreshFlag=False):
""" Get a report for users of a given VO
:param str vo: VO name
:return: S_OK/S_ERROR, Value = user description dictionary
"""
if refreshFlag:
gConfig.forceRefresh()
# Get DIRAC users
diracUsers = getUsersInVO(vo)
if not diracUsers:
return S_ERROR("No VO users found for %s" % vo)
if refreshFlag:
result = self.csapi.downloadCSData()
if not result['OK']:
return result
result = self.csapi.describeUsers(diracUsers)
if not result['OK']:
self.log.error('Could not retrieve CS User description')
return result
def getVOUserReport(self, vo):
"""
:param str vo: VO name
:return: report string
"""
result = self.getVOUserData(vo, refreshFlag=True)
if not result['OK']:
return result
userDict = result['Value']
# Get DIRAC group vs VOMS Role Mappings
result = getVOMSRoleGroupMapping(vo)
if not result['OK']:
return result
diracVOMSMapping = result['Value']['DIRACVOMS']
records = []
groupDict = defaultdict(int)
multiDNUsers = {}
suspendedUsers = []
for user in userDict:
for group in userDict[user]['Groups']:
groupDict[group] += 1
dnList = fromChar(userDict[user]['DN'])
if len(dnList) > 1:
multiDNUsers[user] = dnList
if userDict[user].get('Status', 'Active') == 'Suspended':
suspendedUsers.append(user)
for group in diracVOMSMapping:
records.append(
(group, str(groupDict[group]), diracVOMSMapping.get(group,
'')))
fields = ['Group', 'Number of users', 'VOMS Role']
output = printTable(fields,
records,
sortField='Group',
printOut=False,
numbering=False)
if multiDNUsers:
output += '\nUsers with multiple DNs:\n'
for user in multiDNUsers:
output += ' %s:\n' % user
for dn in multiDNUsers[user]:
output += ' %s\n' % dn
if suspendedUsers:
output += '\n%d suspended users:\n' % len(suspendedUsers)
output += ' %s' % ','.join(suspendedUsers)
return S_OK(output)
@executeWithUserProxy
def __syncCSWithVOMS(self, vo):
self.__adminMsgs = {'Errors': [], 'Info': []}
resultDict = defaultdict(list)
# Get DIRAC group vs VOMS Role Mappings
result = getVOMSRoleGroupMapping(vo)
if not result['OK']:
return result
vomsDIRACMapping = result['Value']['VOMSDIRAC']
diracVOMSMapping = result['Value']['DIRACVOMS']
noVOMSGroups = result['Value']['NoVOMS']
noSyncVOMSGroups = result['Value']['NoSyncVOMS']
vomsSrv = VOMSService(vo)
# Get VOMS VO name
result = vomsSrv.admGetVOName()
if not result['OK']:
self.log.error('Could not retrieve VOMS VO name', "for %s" % vo)
return result
vomsVOName = result['Value'].lstrip('/')
self.log.verbose("VOMS VO Name for %s is %s" % (vo, vomsVOName))
# Get VOMS users
result = vomsSrv.getUsers()
if not result['OK']:
self.log.error('Could not retrieve user information from VOMS',
result['Message'])
return result
vomsUserDict = result['Value']
message = "There are %s user entries in VOMS for VO %s" % (
len(vomsUserDict), vomsVOName)
self.__adminMsgs['Info'].append(message)
self.log.info(message)
# Get DIRAC users
result = self.getVOUserData(vo)
if not result['OK']:
return result
diracUserDict = result['Value']
self.__adminMsgs['Info'].append(
"There are %s registered users in DIRAC for VO %s" %
(len(diracUserDict), vo))
self.log.info("There are %s registered users in DIRAC VO %s" %
(len(diracUserDict), vo))
# Find new and obsoleted user DNs
existingDNs = []
obsoletedDNs = []
newDNs = []
for user in diracUserDict:
dn = diracUserDict[user]['DN']
# We can have users with more than one DN registered
dnList = fromChar(dn)
existingDNs.extend(dnList)
for dn in dnList:
if dn not in vomsUserDict:
obsoletedDNs.append(dn)
for dn in vomsUserDict:
if dn not in existingDNs:
newDNs.append(dn)
allDiracUsers = getAllUsers()
nonVOUserDict = {}
nonVOUsers = list(set(allDiracUsers) - set(diracUserDict.keys()))
if nonVOUsers:
result = self.csapi.describeUsers(nonVOUsers)
if not result['OK']:
self.log.error('Could not retrieve CS User description')
return result
nonVOUserDict = result['Value']
# Process users
defaultVOGroup = getVOOption(vo, "DefaultGroup", "%s_user" % vo)
newAddedUserDict = {}
for dn in vomsUserDict:
nickName = ''
newDNForExistingUser = ''
diracName = ''
if dn in existingDNs:
for user in diracUserDict:
if dn == diracUserDict[user]['DN']:
diracName = user
if dn in newDNs:
# Find if the DN is already registered in the DIRAC CS
for user in nonVOUserDict:
if dn == nonVOUserDict[user]['DN']:
diracName = user
# Check the nickName in the same VO to see if the user is already registered
# with another DN
result = vomsSrv.attGetUserNickname(dn, vomsUserDict[dn]['CA'])
if result['OK']:
nickName = result['Value']
if nickName in diracUserDict or nickName in newAddedUserDict:
diracName = nickName
# This is a flag for adding the new DN to an already existing user
newDNForExistingUser = dn
# We have a real new user
if not diracName:
if nickName:
newDiracName = nickName
else:
newDiracName = getUserName(dn,
vomsUserDict[dn]['mail'],
vo)
# If the chosen user name exists already, append a distinguishing suffix
ind = 1
trialName = newDiracName
while newDiracName in allDiracUsers:
# We have a user with the same name but with a different DN
newDiracName = "%s_%d" % (trialName, ind)
ind += 1
# We now have everything to add the new user
userDict = {
"DN": dn,
"CA": vomsUserDict[dn]['CA'],
"Email": vomsUserDict[dn]['mail']
}
groupsWithRole = []
for role in vomsUserDict[dn]['Roles']:
fullRole = "/%s/%s" % (vomsVOName, role)
groupList = vomsDIRACMapping.get(fullRole, [])
for group in groupList:
if group not in noSyncVOMSGroups:
groupsWithRole.append(group)
userDict['Groups'] = list(
set(groupsWithRole + [defaultVOGroup]))
message = "\n Added new user %s:\n" % newDiracName
for key in userDict:
message += " %s: %s\n" % (key, str(userDict[key]))
self.__adminMsgs['Info'].append(message)
self.voChanged = True
if self.autoAddUsers:
self.log.info("Adding new user %s: %s" %
(newDiracName, str(userDict)))
result = self.csapi.modifyUser(
newDiracName, userDict, createIfNonExistant=True)
if not result['OK']:
self.log.warn('Failed adding new user %s' %
newDiracName)
resultDict['NewUsers'].append(newDiracName)
newAddedUserDict[newDiracName] = userDict
continue
# We have an already existing user
modified = False
userDict = {
"DN": dn,
"CA": vomsUserDict[dn]['CA'],
"Email": vomsUserDict[dn]['mail']
}
if newDNForExistingUser:
userDict['DN'] = ','.join([dn, diracUserDict[diracName]['DN']])
modified = True
existingGroups = diracUserDict.get(diracName, {}).get('Groups', [])
nonVOGroups = list(
set(existingGroups) - set(diracVOMSMapping.keys()))
groupsWithRole = []
for role in vomsUserDict[dn]['Roles']:
fullRole = "/%s/%s" % (vomsVOName, role)
groupList = vomsDIRACMapping.get(fullRole, [])
for group in groupList:
if group not in noSyncVOMSGroups:
groupsWithRole.append(group)
keepGroups = nonVOGroups + groupsWithRole + [defaultVOGroup]
for group in existingGroups:
if group in nonVOGroups:
continue
role = diracVOMSMapping.get(group, '')
# Among already existing groups for the user keep those without a special VOMS Role
# because this membership is done by hand in the CS
if not "Role" in role:
keepGroups.append(group)
# Keep existing groups with no VOMS attribute if any
if group in noVOMSGroups:
keepGroups.append(group)
# Keep groups for which syncronization with VOMS is forbidden
if group in noSyncVOMSGroups:
keepGroups.append(group)
userDict['Groups'] = list(set(keepGroups))
# Merge together groups for the same user but different DNs
if diracName in newAddedUserDict:
otherGroups = newAddedUserDict[diracName].get('Groups', [])
userDict['Groups'] = list(set(keepGroups + otherGroups))
modified = True
# Check if something changed before asking CSAPI to modify
if diracName in diracUserDict:
message = "\n Modified user %s:\n" % diracName
modMsg = ''
for key in userDict:
if key == "Groups":
addedGroups = set(userDict[key]) - set(
diracUserDict.get(diracName, {}).get(key, []))
removedGroups = set(
diracUserDict.get(diracName, {}).get(
key, [])) - set(userDict[key])
if addedGroups:
modMsg += " Added to group(s) %s\n" % ','.join(
addedGroups)
if removedGroups:
modMsg += " Removed from group(s) %s\n" % ','.join(
removedGroups)
else:
oldValue = str(
diracUserDict.get(diracName, {}).get(key, ''))
if str(userDict[key]) != oldValue:
modMsg += " %s: %s -> %s\n" % (
key, oldValue, str(userDict[key]))
if modMsg:
self.__adminMsgs['Info'].append(message + modMsg)
modified = True
if self.autoModifyUsers and modified:
result = self.csapi.modifyUser(diracName, userDict)
if result['OK'] and result['Value']:
self.log.info("Modified user %s: %s" %
(diracName, str(userDict)))
self.voChanged = True
resultDict['ModifiedUsers'].append(diracName)
# Check if there are potentially obsoleted users
oldUsers = set()
for user in diracUserDict:
dnSet = set(fromChar(diracUserDict[user]['DN']))
if not dnSet.intersection(set(
vomsUserDict.keys())) and user not in nonVOUserDict:
for group in diracUserDict[user]['Groups']:
if group not in noVOMSGroups:
oldUsers.add(user)
if oldUsers:
self.voChanged = True
if self.autoDeleteUsers:
self.log.info('The following users will be deleted: %s' %
str(oldUsers))
result = self.csapi.deleteUsers(oldUsers)
if result['OK']:
self.__adminMsgs['Info'].append(
'The following users are deleted from CS:\n %s\n' %
str(oldUsers))
resultDict['DeletedUsers'] = oldUsers
else:
self.__adminMsgs['Errors'].append(
'Error in deleting users from CS:\n %s' %
str(oldUsers))
self.log.error('Error while user deletion from CS', result)
else:
self.__adminMsgs['Info'].append(
'The following users to be checked for deletion:\n %s' %
str(oldUsers))
self.log.info(
'The following users to be checked for deletion: %s' %
str(oldUsers))
return S_OK(resultDict)
@executeWithUserProxy
def __addHomeDirectory(self, vo, newUsers):
fc = FileCatalog(vo=vo)
defaultVOGroup = getVOOption(vo, "DefaultGroup", "%s_user" % vo)
failed = {}
successful = {}
for user in newUsers:
result = fc.addUser(user)
if not result['OK']:
failed[user] = "addUser"
continue
dirName = '/%s/user/%s/%s' % (vo, user[0], user)
result = fc.createDirectory(dirName)
if not result['OK']:
failed[user] = "createDirectory"
continue
result = fc.changePathOwner({dirName: user}, recursive=True)
if not result['OK']:
failed[user] = "changePathOwner"
continue
result = fc.changePathGroup({dirName: defaultVOGroup},
recursive=True)
if not result['OK']:
failed[user] = "changePathGroup"
continue
successful[user] = True
return S_OK({"Successful": successful, "Failed": failed})
class CE2CSAgent(AgentModule):
addressTo = ""
addressFrom = ""
voName = ""
subject = "CE2CSAgent"
alternativeBDIIs = []
def initialize(self):
# TODO: Have no default and if no mail is found then use the diracAdmin group
# and resolve all associated mail addresses.
self.addressTo = self.am_getOption("MailTo", self.addressTo)
self.addressFrom = self.am_getOption("MailFrom", self.addressFrom)
# Create a list of alternative bdii urls
self.alternativeBDIIs = self.am_getOption("AlternativeBDIIs", [])
# Check if the bdii url is appended by a port number, if not append the default 2170
for index, url in enumerate(self.alternativeBDIIs):
if not url.split(":")[-1].isdigit():
self.alternativeBDIIs[index] += ":2170"
if self.addressTo and self.addressFrom:
self.log.info("MailTo", self.addressTo)
self.log.info("MailFrom", self.addressFrom)
if self.alternativeBDIIs:
self.log.info("AlternativeBDII URLs:", self.alternativeBDIIs)
self.subject = "CE2CSAgent"
# This sets the Default Proxy to used as that defined under
# /Operations/Shifter/TestManager
# the shifterProxy option in the Configuration can be used to change this default.
self.am_setOption("shifterProxy", "TestManager")
self.voName = self.am_getOption("VirtualOrganization", [])
if not self.voName:
vo = getVO()
if vo:
self.voName = [vo]
if self.voName:
self.log.info("Agent will manage VO(s) %s" % self.voName)
else:
self.log.fatal("VirtualOrganization option not defined for agent")
return S_ERROR()
self.csAPI = CSAPI()
return self.csAPI.initialize()
def execute(self):
self.log.info("Start Execution")
result = getProxyInfo()
if not result["OK"]:
return result
infoDict = result["Value"]
self.log.info(formatProxyInfoAsString(infoDict))
# Get a "fresh" copy of the CS data
result = self.csAPI.downloadCSData()
if not result["OK"]:
self.log.warn("Could not download a fresh copy of the CS data", result["Message"])
self.__lookForCE()
self.__infoFromCE()
self.log.info("End Execution")
return S_OK()
def __checkAlternativeBDIISite(self, fun, *args):
if self.alternativeBDIIs:
self.log.warn("Trying to use alternative BDII sites")
for site in self.alternativeBDIIs:
self.log.info("Trying to contact alternative BDII", site)
if len(args) == 1:
result = fun(args[0], host=site)
elif len(args) == 2:
result = fun(args[0], vo=args[1], host=site)
if not result["OK"]:
self.log.error("Problem contacting alternative BDII", result["Message"])
elif result["OK"]:
return result
self.log.warn("Also checking alternative BDII sites failed")
return result
def __lookForCE(self):
knownCEs = self.am_getOption("BannedCEs", [])
result = gConfig.getSections("/Resources/Sites")
if not result["OK"]:
return
grids = result["Value"]
for grid in grids:
result = gConfig.getSections("/Resources/Sites/%s" % grid)
if not result["OK"]:
return
sites = result["Value"]
for site in sites:
opt = gConfig.getOptionsDict("/Resources/Sites/%s/%s" % (grid, site))["Value"]
ces = List.fromChar(opt.get("CE", ""))
knownCEs += ces
response = ""
for vo in self.voName:
self.log.info("Check for available CEs for VO", vo)
response = ldapCEState("", vo)
if not response["OK"]:
self.log.error("Error during BDII request", response["Message"])
response = self.__checkAlternativeBDIISite(ldapCEState, "", vo)
return response
newCEs = {}
for queue in response["Value"]:
try:
queueName = queue["GlueCEUniqueID"]
except:
continue
ceName = queueName.split(":")[0]
if not ceName in knownCEs:
newCEs[ceName] = None
self.log.debug("New CE", ceName)
body = ""
possibleNewSites = []
for ce in newCEs.iterkeys():
response = ldapCluster(ce)
if not response["OK"]:
self.log.warn("Error during BDII request", response["Message"])
response = self.__checkAlternativeBDIISite(ldapCluster, ce)
continue
clusters = response["Value"]
if len(clusters) != 1:
self.log.warn("Error in cluster length", " CE %s Length %d" % (ce, len(clusters)))
if len(clusters) == 0:
continue
cluster = clusters[0]
fkey = cluster.get("GlueForeignKey", [])
if type(fkey) == type(""):
fkey = [fkey]
nameBDII = None
for entry in fkey:
if entry.count("GlueSiteUniqueID"):
nameBDII = entry.split("=")[1]
break
if not nameBDII:
continue
ceString = "CE: %s, GOCDB Name: %s" % (ce, nameBDII)
self.log.info(ceString)
response = ldapCE(ce)
if not response["OK"]:
self.log.warn("Error during BDII request", response["Message"])
response = self.__checkAlternativeBDIISite(ldapCE, ce)
continue
ceInfos = response["Value"]
if len(ceInfos):
ceInfo = ceInfos[0]
systemName = ceInfo.get("GlueHostOperatingSystemName", "Unknown")
systemVersion = ceInfo.get("GlueHostOperatingSystemVersion", "Unknown")
systemRelease = ceInfo.get("GlueHostOperatingSystemRelease", "Unknown")
else:
systemName = "Unknown"
systemVersion = "Unknown"
systemRelease = "Unknown"
osString = "SystemName: %s, SystemVersion: %s, SystemRelease: %s" % (
systemName,
systemVersion,
systemRelease,
)
self.log.info(osString)
response = ldapCEState(ce, vo)
if not response["OK"]:
self.log.warn("Error during BDII request", response["Message"])
response = self.__checkAlternativeBDIISite(ldapCEState, ce, vo)
continue
newCEString = "\n\n%s\n%s" % (ceString, osString)
usefull = False
ceStates = response["Value"]
for ceState in ceStates:
queueName = ceState.get("GlueCEUniqueID", "UnknownName")
queueStatus = ceState.get("GlueCEStateStatus", "UnknownStatus")
queueString = "%s %s" % (queueName, queueStatus)
self.log.info(queueString)
newCEString += "\n%s" % queueString
if queueStatus.count("Production"):
usefull = True
if usefull:
body += newCEString
possibleNewSites.append("dirac-admin-add-site DIRACSiteName %s %s" % (nameBDII, ce))
if body:
body = "We are glad to inform You about new CE(s) possibly suitable for %s:\n" % vo + body
body += "\n\nTo suppress information about CE add its name to BannedCEs list."
for possibleNewSite in possibleNewSites:
body = "%s\n%s" % (body, possibleNewSite)
self.log.info(body)
if self.addressTo and self.addressFrom:
notification = NotificationClient()
result = notification.sendMail(
self.addressTo, self.subject, body, self.addressFrom, localAttempt=False
)
return S_OK()
def __infoFromCE(self):
sitesSection = cfgPath("Resources", "Sites")
result = gConfig.getSections(sitesSection)
if not result["OK"]:
return
grids = result["Value"]
changed = False
body = ""
for grid in grids:
gridSection = cfgPath(sitesSection, grid)
result = gConfig.getSections(gridSection)
if not result["OK"]:
return
sites = result["Value"]
for site in sites:
siteSection = cfgPath(gridSection, site)
opt = gConfig.getOptionsDict(siteSection)["Value"]
name = opt.get("Name", "")
if name:
coor = opt.get("Coordinates", "Unknown")
mail = opt.get("Mail", "Unknown")
result = ldapSite(name)
if not result["OK"]:
self.log.warn("BDII site %s: %s" % (name, result["Message"]))
result = self.__checkAlternativeBDIISite(ldapSite, name)
if result["OK"]:
bdiiSites = result["Value"]
if len(bdiiSites) == 0:
self.log.warn(name, "Error in BDII: leng = 0")
else:
if not len(bdiiSites) == 1:
self.log.warn(name, "Warning in BDII: leng = %d" % len(bdiiSites))
bdiiSite = bdiiSites[0]
try:
longitude = bdiiSite["GlueSiteLongitude"]
latitude = bdiiSite["GlueSiteLatitude"]
newcoor = "%s:%s" % (longitude, latitude)
except:
self.log.warn("Error in BDII coordinates")
newcoor = "Unknown"
try:
newmail = bdiiSite["GlueSiteSysAdminContact"].split(":")[-1].strip()
except:
self.log.warn("Error in BDII mail")
newmail = "Unknown"
self.log.debug("%s %s %s" % (name, newcoor, newmail))
if newcoor != coor:
self.log.info("%s" % (name), "%s -> %s" % (coor, newcoor))
if coor == "Unknown":
self.csAPI.setOption(cfgPath(siteSection, "Coordinates"), newcoor)
else:
self.csAPI.modifyValue(cfgPath(siteSection, "Coordinates"), newcoor)
changed = True
if newmail != mail:
self.log.info("%s" % (name), "%s -> %s" % (mail, newmail))
if mail == "Unknown":
self.csAPI.setOption(cfgPath(siteSection, "Mail"), newmail)
else:
self.csAPI.modifyValue(cfgPath(siteSection, "Mail"), newmail)
changed = True
ceList = List.fromChar(opt.get("CE", ""))
if not ceList:
self.log.warn(site, "Empty site list")
continue
# result = gConfig.getSections( cfgPath( siteSection,'CEs' )
# if not result['OK']:
# self.log.debug( "Section CEs:", result['Message'] )
for ce in ceList:
ceSection = cfgPath(siteSection, "CEs", ce)
result = gConfig.getOptionsDict(ceSection)
if not result["OK"]:
self.log.debug("Section CE", result["Message"])
wnTmpDir = "Unknown"
arch = "Unknown"
os = "Unknown"
si00 = "Unknown"
pilot = "Unknown"
ceType = "Unknown"
else:
ceopt = result["Value"]
wnTmpDir = ceopt.get("wnTmpDir", "Unknown")
arch = ceopt.get("architecture", "Unknown")
os = ceopt.get("OS", "Unknown")
si00 = ceopt.get("SI00", "Unknown")
pilot = ceopt.get("Pilot", "Unknown")
ceType = ceopt.get("CEType", "Unknown")
result = ldapCE(ce)
if not result["OK"]:
self.log.warn("Error in BDII for %s" % ce, result["Message"])
result = self.__checkAlternativeBDIISite(ldapCE, ce)
continue
try:
bdiiCE = result["Value"][0]
except:
self.log.warn("Error in BDII for %s" % ce, result)
bdiiCE = None
if bdiiCE:
try:
newWNTmpDir = bdiiCE["GlueSubClusterWNTmpDir"]
except:
newWNTmpDir = "Unknown"
if wnTmpDir != newWNTmpDir and newWNTmpDir != "Unknown":
section = cfgPath(ceSection, "wnTmpDir")
self.log.info(section, " -> ".join((wnTmpDir, newWNTmpDir)))
if wnTmpDir == "Unknown":
self.csAPI.setOption(section, newWNTmpDir)
else:
self.csAPI.modifyValue(section, newWNTmpDir)
changed = True
try:
newArch = bdiiCE["GlueHostArchitecturePlatformType"]
except:
newArch = "Unknown"
if arch != newArch and newArch != "Unknown":
section = cfgPath(ceSection, "architecture")
self.log.info(section, " -> ".join((arch, newArch)))
if arch == "Unknown":
self.csAPI.setOption(section, newArch)
else:
self.csAPI.modifyValue(section, newArch)
changed = True
try:
newOS = "_".join(
(
bdiiCE["GlueHostOperatingSystemName"],
bdiiCE["GlueHostOperatingSystemVersion"],
bdiiCE["GlueHostOperatingSystemRelease"],
)
)
except:
newOS = "Unknown"
if os != newOS and newOS != "Unknown":
section = cfgPath(ceSection, "OS")
self.log.info(section, " -> ".join((os, newOS)))
if os == "Unknown":
self.csAPI.setOption(section, newOS)
else:
self.csAPI.modifyValue(section, newOS)
changed = True
body = body + "OS was changed %s -> %s for %s at %s\n" % (os, newOS, ce, site)
try:
newSI00 = bdiiCE["GlueHostBenchmarkSI00"]
except:
newSI00 = "Unknown"
if si00 != newSI00 and newSI00 != "Unknown":
section = cfgPath(ceSection, "SI00")
self.log.info(section, " -> ".join((si00, newSI00)))
if si00 == "Unknown":
self.csAPI.setOption(section, newSI00)
else:
self.csAPI.modifyValue(section, newSI00)
changed = True
try:
rte = bdiiCE["GlueHostApplicationSoftwareRunTimeEnvironment"]
for vo in self.voName:
if vo.lower() == "lhcb":
if "VO-lhcb-pilot" in rte:
newPilot = "True"
else:
newPilot = "False"
else:
newPilot = "Unknown"
except:
newPilot = "Unknown"
if pilot != newPilot and newPilot != "Unknown":
section = cfgPath(ceSection, "Pilot")
self.log.info(section, " -> ".join((pilot, newPilot)))
if pilot == "Unknown":
self.csAPI.setOption(section, newPilot)
else:
self.csAPI.modifyValue(section, newPilot)
changed = True
newVO = ""
for vo in self.voName:
result = ldapCEState(ce, vo) # getBDIICEVOView
if not result["OK"]:
self.log.warn("Error in BDII for queue %s" % ce, result["Message"])
result = self.__checkAlternativeBDIISite(ldapCEState, ce, vo)
continue
try:
queues = result["Value"]
except:
self.log.warn("Error in BDII for queue %s" % ce, result["Massage"])
continue
newCEType = "Unknown"
for queue in queues:
try:
queueType = queue["GlueCEImplementationName"]
except:
queueType = "Unknown"
if newCEType == "Unknown":
newCEType = queueType
else:
if queueType != newCEType:
self.log.warn(
"Error in BDII for CE %s " % ce,
"different CE types %s %s" % (newCEType, queueType),
)
if newCEType == "ARC-CE":
newCEType = "ARC"
if ceType != newCEType and newCEType != "Unknown":
section = cfgPath(ceSection, "CEType")
self.log.info(section, " -> ".join((ceType, newCEType)))
if ceType == "Unknown":
self.csAPI.setOption(section, newCEType)
else:
self.csAPI.modifyValue(section, newCEType)
changed = True
for queue in queues:
try:
queueName = queue["GlueCEUniqueID"].split("/")[-1]
except:
self.log.warn("Error in queueName ", queue)
continue
try:
newMaxCPUTime = queue["GlueCEPolicyMaxCPUTime"]
except:
newMaxCPUTime = None
newSI00 = None
try:
caps = queue["GlueCECapability"]
if type(caps) == type(""):
caps = [caps]
for cap in caps:
if cap.count("CPUScalingReferenceSI00"):
newSI00 = cap.split("=")[-1]
except:
newSI00 = None
queueSection = cfgPath(ceSection, "Queues", queueName)
result = gConfig.getOptionsDict(queueSection)
if not result["OK"]:
self.log.warn("Section Queues", result["Message"])
maxCPUTime = "Unknown"
si00 = "Unknown"
allowedVOs = [""]
else:
queueOpt = result["Value"]
maxCPUTime = queueOpt.get("maxCPUTime", "Unknown")
si00 = queueOpt.get("SI00", "Unknown")
if newVO == "": # Remember previous iteration, if none - read from conf
allowedVOs = queueOpt.get("VO", "").split(",")
else: # Else use newVO, as it can contain changes, which aren't in conf yet
allowedVOs = newVO.split(",")
if newMaxCPUTime and (maxCPUTime != newMaxCPUTime):
section = cfgPath(queueSection, "maxCPUTime")
self.log.info(section, " -> ".join((maxCPUTime, newMaxCPUTime)))
if maxCPUTime == "Unknown":
self.csAPI.setOption(section, newMaxCPUTime)
else:
self.csAPI.modifyValue(section, newMaxCPUTime)
changed = True
if newSI00 and (si00 != newSI00):
section = cfgPath(queueSection, "SI00")
self.log.info(section, " -> ".join((si00, newSI00)))
if si00 == "Unknown":
self.csAPI.setOption(section, newSI00)
else:
self.csAPI.modifyValue(section, newSI00)
changed = True
modifyVO = True # Flag saying if we need VO option to change
newVO = ""
if allowedVOs != [""]:
for allowedVO in allowedVOs:
allowedVO = allowedVO.strip() # Get rid of spaces
newVO += allowedVO
if allowedVO == vo: # Current VO has been already in list
newVO = ""
modifyVO = False # Don't change anything
break # Skip next 'if', proceed to next VO
newVO += ", "
if modifyVO:
section = cfgPath(queueSection, "VO")
newVO += vo
self.log.info(section, " -> ".join(("%s" % allowedVOs, newVO)))
if allowedVOs == [""]:
self.csAPI.setOption(section, newVO)
else:
self.csAPI.modifyValue(section, newVO)
changed = True
if changed:
self.log.info(body)
if body and self.addressTo and self.addressFrom:
notification = NotificationClient()
result = notification.sendMail(self.addressTo, self.subject, body, self.addressFrom, localAttempt=False)
return self.csAPI.commit()
else:
self.log.info("No changes found")
return S_OK()
class Bdii2CSAgent(AgentModule):
def __init__(self, *args, **kwargs):
""" Defines default parameters
"""
super(Bdii2CSAgent, self).__init__(*args, **kwargs)
self.addressTo = ''
self.addressFrom = ''
self.voName = []
self.subject = "Bdii2CSAgent"
self.alternativeBDIIs = []
self.voBdiiCEDict = {}
self.voBdiiSEDict = {}
self.host = 'lcg-bdii.cern.ch:2170'
self.glue2URLs = []
self.glue2Only = False
self.csAPI = None
# What to get
self.processCEs = True
self.processSEs = False
self.selectedSites = []
# Update the CS or not?
self.dryRun = False
def initialize(self):
""" Gets run paramaters from the configuration
"""
self.addressTo = self.am_getOption('MailTo', self.addressTo)
self.addressFrom = self.am_getOption('MailFrom', self.addressFrom)
# Create a list of alternative bdii urls
self.alternativeBDIIs = self.am_getOption('AlternativeBDIIs', self.alternativeBDIIs)
self.host = self.am_getOption('Host', self.host)
self.glue2URLs = self.am_getOption('GLUE2URLs', self.glue2URLs)
self.glue2Only = self.am_getOption('GLUE2Only', self.glue2Only)
# Check if the bdii url is appended by a port number, if not append the default 2170
for index, url in enumerate(self.alternativeBDIIs):
if not url.split(':')[-1].isdigit():
self.alternativeBDIIs[index] += ':2170'
if self.addressTo and self.addressFrom:
self.log.info("MailTo", self.addressTo)
self.log.info("MailFrom", self.addressFrom)
if self.alternativeBDIIs:
self.log.info("AlternativeBDII URLs:", self.alternativeBDIIs)
self.processCEs = self.am_getOption('ProcessCEs', self.processCEs)
self.processSEs = self.am_getOption('ProcessSEs', self.processSEs)
self.selectedSites = self.am_getOption('SelectedSites', [])
self.dryRun = self.am_getOption('DryRun', self.dryRun)
self.voName = self.am_getOption('VirtualOrganization', self.voName)
if not self.voName:
self.voName = self.am_getOption('VO', [])
if not self.voName or (len(self.voName) == 1 and self.voName[0].lower() == 'all'):
# Get all VOs defined in the configuration
self.voName = []
result = getVOs()
if result['OK']:
vos = result['Value']
for vo in vos:
vomsVO = getVOOption(vo, "VOMSName")
if vomsVO:
self.voName.append(vomsVO)
if self.voName:
self.log.info("Agent will manage VO(s) %s" % self.voName)
else:
self.log.fatal("VirtualOrganization option not defined for agent")
return S_ERROR()
self.csAPI = CSAPI()
return self.csAPI.initialize()
def execute(self):
""" General agent execution method
"""
self.voBdiiCEDict = {}
self.voBdiiSEDict = {}
# Get a "fresh" copy of the CS data
result = self.csAPI.downloadCSData()
if not result['OK']:
self.log.warn("Could not download a fresh copy of the CS data", result['Message'])
# Refresh the configuration from the master server
gConfig.forceRefresh(fromMaster=True)
if self.processCEs:
self.__lookForNewCEs()
self.__updateCEs()
if self.processSEs:
self.__lookForNewSEs()
self.__updateSEs()
return S_OK()
def __lookForNewCEs(self):
""" Look up BDII for CEs not yet present in the DIRAC CS
"""
bannedCEs = self.am_getOption('BannedCEs', [])
result = getCEsFromCS()
if not result['OK']:
return result
knownCEs = set(result['Value'])
knownCEs = knownCEs.union(set(bannedCEs))
for vo in self.voName:
result = self.__getBdiiCEInfo(vo)
if not result['OK']:
continue
bdiiInfo = result['Value']
result = getGridCEs(vo, bdiiInfo=bdiiInfo, ceBlackList=knownCEs)
if not result['OK']:
self.log.error('Failed to get unused CEs', result['Message'])
siteDict = result['Value']
body = ''
for site in siteDict:
newCEs = set(siteDict[site].keys()) # pylint: disable=no-member
if not newCEs:
continue
ceString = ''
for ce in newCEs:
queueString = ''
ceInfo = bdiiInfo[site]['CEs'][ce]
newCEString = "CE: %s, GOCDB Site Name: %s" % (ce, site)
systemTuple = siteDict[site][ce]['System']
osString = "%s_%s_%s" % (systemTuple)
newCEString = "\n%s\n%s\n" % (newCEString, osString)
for queue in ceInfo['Queues']:
queueStatus = ceInfo['Queues'][queue].get('GlueCEStateStatus', 'UnknownStatus')
if 'production' in queueStatus.lower():
ceType = ceInfo['Queues'][queue].get('GlueCEImplementationName', '')
queueString += " %s %s %s\n" % (queue, queueStatus, ceType)
if queueString:
ceString += newCEString
ceString += "Queues:\n"
ceString += queueString
if ceString:
body += ceString
if body:
body = "\nWe are glad to inform You about new CE(s) possibly suitable for %s:\n" % vo + body
body += "\n\nTo suppress information about CE add its name to BannedCEs list.\n"
body += "Add new Sites/CEs for vo %s with the command:\n" % vo
body += "dirac-admin-add-resources --vo %s --ce\n" % vo
self.log.info(body)
if self.addressTo and self.addressFrom:
notification = NotificationClient()
result = notification.sendMail(self.addressTo, self.subject, body, self.addressFrom,
localAttempt=False, avoidSpam=True)
if not result['OK']:
self.log.error('Can not send new site notification mail', result['Message'])
return S_OK()
def __getBdiiCEInfo(self, vo):
if vo in self.voBdiiCEDict:
return S_OK(self.voBdiiCEDict[vo])
self.log.info("Check for available CEs for VO", vo)
totalResult = S_OK({})
message = ''
mainResult = getBdiiCEInfo(vo, host=self.host, glue2=self.glue2Only)
if not mainResult['OK']:
self.log.error("Failed getting information from default bdii", mainResult['Message'])
message = mainResult['Message']
for bdii in reversed(self.alternativeBDIIs):
resultAlt = getBdiiCEInfo(vo, host=bdii, glue2=self.glue2Only)
if resultAlt['OK']:
totalResult['Value'].update(resultAlt['Value'])
else:
self.log.error("Failed getting information from %s " % bdii, resultAlt['Message'])
message = (message + "\n" + resultAlt['Message']).strip()
for glue2URL in self.glue2URLs:
if self.glue2Only:
break
resultGlue2 = getBdiiCEInfo(vo, host=glue2URL, glue2=True)
if resultGlue2['OK']:
totalResult['Value'].update(resultGlue2['Value'])
else:
self.log.error("Failed getting GLUE2 information for", "%s, %s: %s" %
(glue2URL, vo, resultGlue2['Message']))
message = (message + "\n" + resultGlue2['Message']).strip()
if mainResult['OK']:
totalResult['Value'].update(mainResult['Value'])
if not totalResult['Value'] and message: # Dict is empty and we have an error message
self.log.error("Error during BDII request", message)
totalResult = S_ERROR(message)
else:
self.voBdiiCEDict[vo] = totalResult['Value']
return totalResult
def __getBdiiSEInfo(self, vo):
if vo in self.voBdiiSEDict:
return S_OK(self.voBdiiSEDict[vo])
self.log.info("Check for available SEs for VO", vo)
result = getBdiiSEInfo(vo)
message = ''
if not result['OK']:
message = result['Message']
for bdii in self.alternativeBDIIs:
result = getBdiiSEInfo(vo, host=bdii)
if result['OK']:
break
if not result['OK']:
if message:
self.log.error("Error during BDII request", message)
else:
self.log.error("Error during BDII request", result['Message'])
else:
self.voBdiiSEDict[vo] = result['Value']
return result
def __updateCEs(self):
""" Update the Site/CE/queue settings in the CS if they were changed in the BDII
"""
bdiiChangeSet = set()
for vo in self.voName:
result = self.__getBdiiCEInfo(vo)
if not result['OK']:
continue
ceBdiiDict = result['Value']
self.__purgeSites(ceBdiiDict)
result = getSiteUpdates(vo, bdiiInfo=ceBdiiDict, log=self.log)
if not result['OK']:
continue
bdiiChangeSet = bdiiChangeSet.union(result['Value'])
# We have collected all the changes, consolidate VO settings
result = self.__updateCS(bdiiChangeSet)
return result
def __purgeSites(self, ceBdiiDict):
"""Remove all sites that are not in self.selectedSites.
Modifies the ceBdiiDict!
"""
if not self.selectedSites:
return
for site in list(ceBdiiDict):
ces = list(ceBdiiDict[site]['CEs'])
if not ces:
self.log.error("No CE information for site:", site)
continue
diracSiteName = getSiteForCE(ces[0])
if not diracSiteName['OK']:
self.log.error("Failed to get DIRAC site name for ce", "%s: %s" % (ces[0], diracSiteName['Message']))
continue
self.log.debug("Checking site %s (%s), aka %s" % (site, ces, diracSiteName['Value']))
if diracSiteName['Value'] in self.selectedSites:
continue
self.log.info("Dropping site %s, aka %s" % (site, diracSiteName))
ceBdiiDict.pop(site)
return
def __updateCS(self, bdiiChangeSet):
queueVODict = {}
changeSet = set()
for entry in bdiiChangeSet:
section, option, _value, new_value = entry
if option == "VO":
queueVODict.setdefault(section, set())
queueVODict[section] = queueVODict[section].union(set(new_value.split(',')))
else:
changeSet.add(entry)
for section, VOs in queueVODict.items():
changeSet.add((section, 'VO', '', ','.join(VOs)))
if changeSet:
changeList = sorted(changeSet)
body = '\n'.join(["%s/%s %s -> %s" % entry for entry in changeList])
if body and self.addressTo and self.addressFrom:
notification = NotificationClient()
result = notification.sendMail(self.addressTo, self.subject, body, self.addressFrom, localAttempt=False)
if body:
self.log.info('The following configuration changes were detected:')
self.log.info(body)
for section, option, value, new_value in changeSet:
if value == 'Unknown' or not value:
self.csAPI.setOption(cfgPath(section, option), new_value)
else:
self.csAPI.modifyValue(cfgPath(section, option), new_value)
if self.dryRun:
self.log.info("Dry Run: CS won't be updated")
self.csAPI.showDiff()
else:
result = self.csAPI.commit()
if not result['OK']:
self.log.error("Error while committing to CS", result['Message'])
else:
self.log.info("Successfully committed %d changes to CS" % len(changeList))
return result
else:
self.log.info("No changes found")
return S_OK()
def __lookForNewSEs(self):
""" Look up BDII for SEs not yet present in the DIRAC CS
"""
bannedSEs = self.am_getOption('BannedSEs', [])
result = getSEsFromCS()
if not result['OK']:
return result
knownSEs = set(result['Value'])
knownSEs = knownSEs.union(set(bannedSEs))
for vo in self.voName:
result = self.__getBdiiSEInfo(vo)
if not result['OK']:
continue
bdiiInfo = result['Value']
result = getGridSRMs(vo, bdiiInfo=bdiiInfo, srmBlackList=knownSEs)
if not result['OK']:
continue
siteDict = result['Value']
body = ''
for site in siteDict:
newSEs = set(siteDict[site].keys()) # pylint: disable=no-member
if not newSEs:
continue
for se in newSEs:
body += '\n New SE %s available at site %s:\n' % (se, site)
backend = siteDict[site][se]['SE'].get('GlueSEImplementationName', 'Unknown')
size = siteDict[site][se]['SE'].get('GlueSESizeTotal', 'Unknown')
body += ' Backend %s, Size %s' % (backend, size)
if body:
body = "\nWe are glad to inform You about new SE(s) possibly suitable for %s:\n" % vo + body
body += "\n\nTo suppress information about an SE add its name to BannedSEs list.\n"
body += "Add new SEs for vo %s with the command:\n" % vo
body += "dirac-admin-add-resources --vo %s --se\n" % vo
self.log.info(body)
if self.addressTo and self.addressFrom:
notification = NotificationClient()
result = notification.sendMail(self.addressTo, self.subject, body, self.addressFrom, localAttempt=False)
if not result['OK']:
self.log.error('Can not send new site notification mail', result['Message'])
return S_OK()
def __updateSEs(self):
""" Update the Storage Element settings in the CS if they were changed in the BDII
"""
bdiiChangeSet = set()
for vo in self.voName:
result = self.__getBdiiSEInfo(vo)
if not result['OK']:
continue
seBdiiDict = result['Value']
result = getSRMUpdates(vo, bdiiInfo=seBdiiDict)
if not result['OK']:
continue
bdiiChangeSet = bdiiChangeSet.union(result['Value'])
# We have collected all the changes, consolidate VO settings
result = self.__updateCS(bdiiChangeSet)
return result
def checkUnusedSEs():
global vo, dry
result = getGridSRMs(vo, unUsed=True)
if not result['OK']:
gLogger.error('Failed to look up SRMs in BDII', result['Message'])
siteSRMDict = result['Value']
# Evaluate VOs
result = getVOs()
if result['OK']:
csVOs = set(result['Value'])
else:
csVOs = {vo}
changeSetFull = set()
for site in siteSRMDict:
for gridSE in siteSRMDict[site]:
changeSet = set()
seDict = siteSRMDict[site][gridSE]['SE']
srmDict = siteSRMDict[site][gridSE]['SRM']
# Check the SRM version
version = srmDict.get('GlueServiceVersion', '')
if not (version and version.startswith('2')):
gLogger.debug('Skipping SRM service with version %s' % version)
continue
result = getDIRACSiteName(site)
if not result['OK']:
gLogger.notice('Unused se %s is detected at unused site %s' % (gridSE, site))
gLogger.notice('Consider adding site %s to the DIRAC CS' % site)
continue
diracSites = result['Value']
yn = raw_input(
'\nDo you want to add new SRM SE %s at site(s) %s ? default yes [yes|no]: ' %
(gridSE, str(diracSites)))
if not yn or yn.lower().startswith('y'):
if len(diracSites) > 1:
prompt = 'Which DIRAC site the new SE should be attached to ?'
for i, s in enumerate(diracSites):
prompt += '\n[%d] %s' % (i, s)
prompt += '\nEnter your choice number: '
inp = raw_input(prompt)
try:
ind = int(inp)
except BaseException:
gLogger.notice('Can not interpret your choice: %s, try again later' % inp)
continue
diracSite = diracSites[ind]
else:
diracSite = diracSites[0]
domain, siteName, country = diracSite.split('.')
recName = '%s-disk' % siteName
inp = raw_input('Give a DIRAC name to the grid SE %s, default %s : ' % (gridSE, recName))
diracSEName = inp
if not inp:
diracSEName = recName
gLogger.notice('Adding new SE %s at site %s' % (diracSEName, diracSite))
seSection = cfgPath('/Resources/StorageElements', diracSEName)
changeSet.add((seSection, 'BackendType', seDict.get('GlueSEImplementationName', 'Unknown')))
changeSet.add((seSection, 'Description', seDict.get('GlueSEName', 'Unknown')))
bdiiVOs = set([re.sub('^VO:', '', rule) for rule in srmDict.get('GlueServiceAccessControlBaseRule', [])])
seVOs = csVOs.intersection(bdiiVOs)
changeSet.add((seSection, 'VO', ','.join(seVOs)))
accessSection = cfgPath(seSection, 'AccessProtocol.1')
changeSet.add((accessSection, 'Protocol', 'srm'))
changeSet.add((accessSection, 'PluginName', 'SRM2'))
endPoint = srmDict.get('GlueServiceEndpoint', '')
host = urlparse(endPoint).hostname
port = urlparse(endPoint).port
changeSet.add((accessSection, 'Host', host))
changeSet.add((accessSection, 'Port', port))
changeSet.add((accessSection, 'Access', 'remote'))
voPathSection = cfgPath(accessSection, 'VOPath')
if 'VOPath' in seDict:
path = seDict['VOPath']
voFromPath = os.path.basename(path)
if voFromPath != diracVO:
gLogger.notice('\n!!! Warning: non-conventional VO path: %s\n' % path)
changeSet.add((voPathSection, diracVO, path))
path = os.path.dirname(path)
else:
# Try to guess the Path
domain = '.'.join(host.split('.')[-2:])
path = '/dpm/%s/home' % domain
changeSet.add((accessSection, 'Path', path))
changeSet.add((accessSection, 'SpaceToken', ''))
changeSet.add((accessSection, 'WSUrl', '/srm/managerv2?SFN='))
gLogger.notice('SE %s will be added with the following parameters' % diracSEName)
changeList = sorted(changeSet)
for entry in changeList:
gLogger.notice(entry)
yn = raw_input('Do you want to add new SE %s ? default yes [yes|no]: ' % diracSEName)
if not yn or yn.lower().startswith('y'):
changeSetFull = changeSetFull.union(changeSet)
if dry:
if changeSetFull:
gLogger.notice('Skipping commit of the new SE data in a dry run')
else:
gLogger.notice("No new SE to be added")
return S_OK()
if changeSetFull:
csAPI = CSAPI()
csAPI.initialize()
result = csAPI.downloadCSData()
if not result['OK']:
gLogger.error('Failed to initialize CSAPI object', result['Message'])
DIRACExit(-1)
changeList = sorted(changeSetFull)
for section, option, value in changeList:
csAPI.setOption(cfgPath(section, option), value)
yn = raw_input('New SE data is accumulated\n Do you want to commit changes to CS ? default yes [yes|no]: ')
if not yn or yn.lower().startswith('y'):
result = csAPI.commit()
if not result['OK']:
gLogger.error("Error while commit to CS", result['Message'])
else:
gLogger.notice("Successfully committed %d changes to CS" % len(changeSetFull))
else:
gLogger.notice("No new SE to be added")
return S_OK()
class VOMS2CSAgent( AgentModule ):
def initialize( self ):
self.__voDict = {}
voNames = self.am_getOption( 'VO', [] )
if not voNames[0].lower() == "none":
if voNames[0].lower() == "any":
voNames = []
result = getVOMSVOs( voNames )
if not result['OK']:
return result
self.__voDict = result['Value']
self.__adminMsgs = {}
self.csapi = CSAPI()
self.voChanged = False
self.log.notice( "VOs: %s" % self.__voDict.keys() )
self.autoAddUsers = self.am_getOption( 'AutoAddUsers', False )
self.autoModifyUsers = self.am_getOption( 'AutoModifyUsers', False )
self.autoSuspendUsers = self.am_getOption( 'AutoSuspendUsers', False )
return S_OK()
def execute( self ):
self.__adminMsgs = {}
self.csapi.downloadCSData()
for vo in self.__voDict:
self.voChanged = False
voAdminUser = getVOOption( vo, "VOAdmin")
voAdminMail = None
if voAdminUser:
voAdminMail = getUserOption( voAdminUser, "Email")
voAdminGroup = getVOOption( vo, "VOAdminGroup", getVOOption( vo, "DefaultGroup" ) )
self.log.info( 'Performing VOMS sync for VO %s with credentials %s@%s' % ( vo, voAdminUser, voAdminGroup ) )
result = self.__syncCSWithVOMS( vo, proxyUserName = voAdminUser, proxyUserGroup = voAdminGroup ) #pylint: disable=unexpected-keyword-arg
if not result['OK']:
self.log.error( 'Failed to perform VOMS to CS synchronization:', 'VO %s: %s' % ( vo, result["Message"] ) )
continue
if self.voChanged:
mailMsg = ""
if self.__adminMsgs[ 'Errors' ]:
mailMsg += "\nErrors list:\n %s" % "\n ".join( self.__adminMsgs[ 'Errors' ] )
if self.__adminMsgs[ 'Info' ]:
mailMsg += "\nRun result:\n %s" % "\n ".join( self.__adminMsgs[ 'Info' ] )
NotificationClient().sendMail( self.am_getOption( 'MailTo', voAdminMail ),
"VOMS2CSAgent run log", mailMsg,
self.am_getOption( 'mailFrom', "DIRAC system" ) )
# We have accumulated all the changes, commit them now
result = self.csapi.commitChanges()
if not result[ 'OK' ]:
self.log.error( "Could not commit configuration changes", result[ 'Message' ] )
return result
self.log.info( "Configuration committed" )
return S_OK()
@executeWithUserProxy
def __syncCSWithVOMS( self, vo ):
self.__adminMsgs = { 'Errors' : [], 'Info' : [] }
# Get DIRAC group vs VOMS Role Mappings
result = getVOMSRoleGroupMapping( vo )
if not result['OK']:
return result
vomsDIRACMapping = result['Value']['VOMSDIRAC']
diracVOMSMapping = result['Value']['DIRACVOMS']
noVOMSGroups = result['Value']['NoVOMS']
vomsSrv = VOMSService( vo )
# Get VOMS VO name
result = vomsSrv.admGetVOName()
if not result['OK']:
self.log.error( 'Could not retrieve VOMS VO name', "for %s" % vo )
return result
vomsVOName = result[ 'Value' ].lstrip( '/' )
self.log.verbose( "VOMS VO Name for %s is %s" % ( vo, vomsVOName ) )
# Get VOMS users
result = vomsSrv.getUsers()
if not result['OK']:
self.log.error( 'Could not retrieve user information from VOMS', result['Message'] )
return result
vomsUserDict = result[ 'Value' ]
message = "There are %s registered users in VOMS VO %s" % ( len( vomsUserDict ), vomsVOName )
self.__adminMsgs[ 'Info' ].append( message )
self.log.info( message )
# Get DIRAC users
diracUsers = getUsersInVO( vo )
if not diracUsers:
return S_ERROR( "No VO users found for %s" % vo )
result = self.csapi.describeUsers( diracUsers )
if not result['OK']:
self.log.error( 'Could not retrieve CS User description' )
return result
diracUserDict = result['Value']
self.__adminMsgs[ 'Info' ].append( "There are %s registered users in DIRAC for VO %s" % ( len( diracUserDict ), vo ) )
self.log.info( "There are %s registered users in DIRAC VO %s" % ( len( diracUserDict ), vo ) )
# Find new and obsoleted user DNs
existingDNs = []
obsoletedDNs = []
newDNs = []
for user in diracUserDict:
dn = diracUserDict[user]['DN']
existingDNs.append( dn )
if dn not in vomsUserDict:
obsoletedDNs.append( dn )
for dn in vomsUserDict:
if dn not in existingDNs:
newDNs.append( dn )
allDiracUsers = getAllUsers()
nonVOusers = list( set( allDiracUsers ) - set(diracUsers) )
result = self.csapi.describeUsers( nonVOusers )
if not result['OK']:
self.log.error( 'Could not retrieve CS User description' )
return result
nonVOUserDict = result['Value']
# Process users
defaultVOGroup = getVOOption( vo, "DefaultGroup", "%s_user" % vo )
for dn in vomsUserDict:
if dn in newDNs:
# Find if the DN is already registered in the DIRAC CS
diracName = ''
for user in nonVOUserDict:
if dn == nonVOUserDict[user]['DN']:
diracName = user
# We have a real new user
if not diracName:
nickName = ''
result = vomsSrv.attGetUserNickname( dn, vomsUserDict[dn]['CA'] )
if result['OK']:
nickName = result['Value']
if nickName:
newDiracName = nickName
else:
newDiracName = getUserName( dn, vomsUserDict[dn]['mail'] )
ind = 1
trialName = newDiracName
while newDiracName in allDiracUsers:
# We have a user with the same name but with a different DN
newDiracName = "%s_%d" % ( trialName, ind )
ind += 1
# We now have everything to add the new user
userDict = { "DN": dn, "CA": vomsUserDict[dn]['CA'], "Email": vomsUserDict[dn]['mail'] }
groupsWithRole = []
for role in vomsUserDict[dn]['Roles']:
fullRole = "/%s/%s" % ( vomsVOName, role )
group = vomsDIRACMapping.get( fullRole )
if group:
groupsWithRole.append( group )
userDict['Groups'] = list( set( groupsWithRole + [defaultVOGroup] ) )
self.__adminMsgs[ 'Info' ].append( "Adding new user %s: %s" % ( newDiracName, str( userDict ) ) )
self.voChanged = True
if self.autoAddUsers:
self.log.info( "Adding new user %s: %s" % ( newDiracName, str( userDict ) ) )
result = self.csapi.modifyUser( newDiracName, userDict, createIfNonExistant = True )
if not result['OK']:
self.log.warn( 'Failed adding new user %s' % newDiracName )
continue
# We have an already existing user
userDict = { "DN": dn, "CA": vomsUserDict[dn]['CA'], "Email": vomsUserDict[dn]['mail'] }
nonVOGroups = nonVOUserDict.get( diracName, {} ).get( 'Groups', [] )
existingGroups = diracUserDict.get( diracName, {} ).get( 'Groups', [] )
groupsWithRole = []
for role in vomsUserDict[dn]['Roles']:
fullRole = "/%s/%s" % ( vomsVOName, role )
group = vomsDIRACMapping.get( fullRole )
if group:
groupsWithRole.append( group )
keepGroups = nonVOGroups + groupsWithRole + [defaultVOGroup]
for group in existingGroups:
role = diracVOMSMapping[group]
# Among already existing groups for the user keep those without a special VOMS Role
# because this membership is done by hand in the CS
if not "Role" in role:
keepGroups.append( group )
# Keep existing groups with no VOMS attribute if any
if group in noVOMSGroups:
keepGroups.append( group )
userDict['Groups'] = keepGroups
if self.autoModifyUsers:
result = self.csapi.modifyUser( diracName, userDict )
if result['OK'] and result['Value']:
self.voChanged = True
# Check if there are potentially obsoleted users
oldUsers = set()
for user in diracUserDict:
dn = diracUserDict[user]['DN']
if not dn in vomsUserDict and not user in nonVOUserDict:
for group in diracUserDict[user]['Groups']:
if not group in noVOMSGroups:
oldUsers.add( user )
if oldUsers:
self.voChanged = True
self.__adminMsgs[ 'Info' ].append( 'The following users to be checked for deletion: %s' % str( oldUsers ) )
self.log.info( 'The following users to be checked for deletion: %s' % str( oldUsers ) )
return S_OK()
class VOMS2CSSynchronizer(object):
def __init__(self, vo, autoModifyUsers=True, autoAddUsers=True, autoDeleteUsers=False):
self.log = gLogger.getSubLogger("VOMS2CSSynchronizer")
self.csapi = CSAPI()
self.vo = vo
self.vomsVOName = getVOOption(vo, "VOMSName", "")
if not self.vomsVOName:
raise Exception("VOMS name not defined for VO %s" % vo)
self.adminMsgs = {'Errors': [], 'Info': []}
self.vomsUserDict = {}
self.autoModifyUsers = autoModifyUsers
self.autoAddUsers = autoAddUsers
self.autoDeleteUsers = autoDeleteUsers
def syncCSWithVOMS(self):
""" Performs the synchronization of the DIRAC registry with the VOMS data. The resulting
CSAPI object containing modifications is returned as part of the output dictionary.
Those changes can be applied by the caller depending on the mode (dry or a real run)
:return: S_OK with a dictionary containing the results of the synchronization operation
"""
resultDict = defaultdict(list)
# Get DIRAC group vs VOMS Role Mappings
result = getVOMSRoleGroupMapping(self.vo)
if not result['OK']:
return result
vomsDIRACMapping = result['Value']['VOMSDIRAC']
diracVOMSMapping = result['Value']['DIRACVOMS']
noVOMSGroups = result['Value']['NoVOMS']
noSyncVOMSGroups = result['Value']['NoSyncVOMS']
vomsSrv = VOMSService(self.vo)
# Get VOMS users
result = vomsSrv.getUsers()
if not result['OK']:
self.log.error('Could not retrieve user information from VOMS', result['Message'])
return result
self.vomsUserDict = result['Value']
message = "There are %s user entries in VOMS for VO %s" % (len(self.vomsUserDict), self.vomsVOName)
self.adminMsgs['Info'].append(message)
self.log.info(message)
# Get DIRAC users
result = self.getVOUserData(self.vo)
if not result['OK']:
return result
diracUserDict = result['Value']
self.adminMsgs['Info'].append("There are %s registered users in DIRAC for VO %s" % (len(diracUserDict), self.vo))
self.log.info("There are %s registered users in DIRAC VO %s" % (len(diracUserDict), self.vo))
# Find new and obsoleted user DNs
existingDNs = []
obsoletedDNs = []
newDNs = []
for user in diracUserDict:
dn = diracUserDict[user]['DN']
# We can have users with more than one DN registered
dnList = fromChar(dn)
existingDNs.extend(dnList)
for dn in dnList:
if dn not in self.vomsUserDict:
obsoletedDNs.append(dn)
for dn in self.vomsUserDict:
if dn not in existingDNs:
newDNs.append(dn)
allDiracUsers = getAllUsers()
nonVOUserDict = {}
nonVOUsers = list(set(allDiracUsers) - set(diracUserDict))
if nonVOUsers:
result = self.csapi.describeUsers(nonVOUsers)
if not result['OK']:
self.log.error('Could not retrieve CS User description')
return result
nonVOUserDict = result['Value']
# Process users
defaultVOGroup = getVOOption(self.vo, "DefaultGroup", "%s_user" % self.vo)
# If a user is (previously put by hand) in a "QuarantineGroup",
# then the default group will be ignored.
# So, this option is only considered for the case of existing users.
quarantineVOGroup = getVOOption(self.vo, "QuarantineGroup")
newAddedUserDict = {}
for dn in self.vomsUserDict:
newDNForExistingUser = ''
diracName = ''
if dn in existingDNs:
for user in diracUserDict:
if dn == diracUserDict[user]['DN']:
diracName = user
if dn in newDNs:
# Find if the DN is already registered in the DIRAC CS
for user in nonVOUserDict:
if dn == nonVOUserDict[user]['DN']:
diracName = user
# Check the nickName in the same VO to see if the user is already registered
# with another DN
nickName = self.vomsUserDict[dn].get('nickname')
if nickName in diracUserDict or nickName in newAddedUserDict:
diracName = nickName
# This is a flag for adding the new DN to an already existing user
newDNForExistingUser = dn
# We have a real new user
if not diracName:
if nickName:
newDiracName = nickName
else:
newDiracName = self.getUserName(dn)
# Do not consider users with Suspended status in VOMS
if self.vomsUserDict[dn]['suspended'] or self.vomsUserDict[dn]['certSuspended']:
resultDict["SuspendedUsers"].append(newDiracName)
continue
# If the chosen user name exists already, append a distinguishing suffix
ind = 1
trialName = newDiracName
while newDiracName in allDiracUsers:
# We have a user with the same name but with a different DN
newDiracName = "%s_%d" % (trialName, ind)
ind += 1
# We now have everything to add the new user
userDict = {"DN": dn, "CA": self.vomsUserDict[dn]['CA'], "Email": self.vomsUserDict[dn]['mail']}
groupsWithRole = []
for role in self.vomsUserDict[dn]['Roles']:
groupList = vomsDIRACMapping.get(role, [])
for group in groupList:
if group not in noSyncVOMSGroups:
groupsWithRole.append(group)
userDict['Groups'] = list(set(groupsWithRole + [defaultVOGroup]))
message = "\n Added new user %s:\n" % newDiracName
for key in userDict:
message += " %s: %s\n" % (key, str(userDict[key]))
self.adminMsgs['Info'].append(message)
self.voChanged = True
if self.autoAddUsers:
self.log.info("Adding new user %s: %s" % (newDiracName, str(userDict)))
result = self.csapi.modifyUser(newDiracName, userDict, createIfNonExistant=True)
if not result['OK']:
self.log.warn('Failed adding new user %s' % newDiracName)
resultDict['NewUsers'].append(newDiracName)
newAddedUserDict[newDiracName] = userDict
continue
# We have an already existing user
modified = False
userDict = {"DN": dn, "CA": self.vomsUserDict[dn]['CA'], "Email": self.vomsUserDict[dn]['mail']}
if newDNForExistingUser:
userDict['DN'] = ','.join([dn, diracUserDict.get(diracName, newAddedUserDict.get(diracName))['DN']])
modified = True
existingGroups = diracUserDict.get(diracName, {}).get('Groups', [])
nonVOGroups = list(set(existingGroups) - set(diracVOMSMapping))
groupsWithRole = []
for role in self.vomsUserDict[dn]['Roles']:
groupList = vomsDIRACMapping.get(role, [])
for group in groupList:
if group not in noSyncVOMSGroups:
groupsWithRole.append(group)
keepGroups = nonVOGroups + groupsWithRole
if not quarantineVOGroup or quarantineVOGroup not in existingGroups:
keepGroups += [defaultVOGroup]
for group in existingGroups:
if group in nonVOGroups:
continue
role = diracVOMSMapping.get(group, '')
# Among already existing groups for the user keep those without a special VOMS Role
# because this membership is done by hand in the CS
if "Role" not in role:
keepGroups.append(group)
# Keep existing groups with no VOMS attribute if any
if group in noVOMSGroups:
keepGroups.append(group)
# Keep groups for which syncronization with VOMS is forbidden
if group in noSyncVOMSGroups:
keepGroups.append(group)
userDict['Groups'] = list(set(keepGroups))
# Merge together groups for the same user but different DNs
if diracName in newAddedUserDict:
otherGroups = newAddedUserDict[diracName].get('Groups', [])
userDict['Groups'] = list(set(keepGroups + otherGroups))
modified = True
# Check if something changed before asking CSAPI to modify
if diracName in diracUserDict:
message = "\n Modified user %s:\n" % diracName
modMsg = ''
for key in userDict:
if key == "Groups":
addedGroups = set(userDict[key]) - set(diracUserDict.get(diracName, {}).get(key, []))
removedGroups = set(diracUserDict.get(diracName, {}).get(key, [])) - set(userDict[key])
if addedGroups:
modMsg += " Added to group(s) %s\n" % ','.join(addedGroups)
if removedGroups:
modMsg += " Removed from group(s) %s\n" % ','.join(removedGroups)
else:
oldValue = str(diracUserDict.get(diracName, {}).get(key, ''))
if str(userDict[key]) != oldValue:
modMsg += " %s: %s -> %s\n" % (key, oldValue, str(userDict[key]))
if modMsg:
self.adminMsgs['Info'].append(message + modMsg)
modified = True
if self.autoModifyUsers and modified:
result = self.csapi.modifyUser(diracName, userDict)
if result['OK'] and result['Value']:
self.log.info("Modified user %s: %s" % (diracName, str(userDict)))
self.voChanged = True
resultDict['ModifiedUsers'].append(diracName)
# Check if there are potentially obsoleted users
oldUsers = set()
for user in diracUserDict:
dnSet = set(fromChar(diracUserDict[user]['DN']))
if not dnSet.intersection(set(self.vomsUserDict)) and user not in nonVOUserDict:
for group in diracUserDict[user]['Groups']:
if group not in noVOMSGroups:
oldUsers.add(user)
# Check for obsoleted DNs
for user in diracUserDict:
dnSet = set(fromChar(diracUserDict[user]['DN']))
for dn in dnSet:
if dn in obsoletedDNs and user not in oldUsers:
self.log.verbose("Modified user %s: dropped DN %s" % (user, dn))
if self.autoModifyUsers:
userDict = diracUserDict[user]
modDNSet = dnSet - set([dn])
if modDNSet:
userDict['DN'] = ','.join(modDNSet)
result = self.csapi.modifyUser(user, userDict)
if result['OK'] and result['Value']:
self.log.info("Modified user %s: dropped DN %s" % (user, dn))
self.adminMsgs['Info'].append("Modified user %s: dropped DN %s" % (user, dn))
self.voChanged = True
resultDict['ModifiedUsers'].append(diracName)
else:
oldUsers.add(user)
if oldUsers:
self.voChanged = True
if self.autoDeleteUsers:
self.log.info('The following users will be deleted: %s' % str(oldUsers))
result = self.csapi.deleteUsers(oldUsers)
if result['OK']:
self.adminMsgs['Info'].append('The following users are deleted from CS:\n %s\n' % str(oldUsers))
resultDict['DeletedUsers'] = oldUsers
else:
self.adminMsgs['Errors'].append('Error in deleting users from CS:\n %s' % str(oldUsers))
self.log.error('Error while user deletion from CS', result)
else:
self.adminMsgs['Info'].append('The following users to be checked for deletion:\n %s' % str(oldUsers))
self.log.info('The following users to be checked for deletion: %s' % str(oldUsers))
resultDict['CSAPI'] = self.csapi
resultDict['AdminMessages'] = self.adminMsgs
return S_OK(resultDict)
def getVOUserData(self, refreshFlag=False):
""" Get a report for users of a given VO
:param bool refreshFlag: flag to indicate that the configuration must be refreshed
before looking up user data
:return: S_OK/S_ERROR, Value = user description dictionary
"""
if refreshFlag:
gConfig.forceRefresh()
# Get DIRAC users
diracUsers = getUsersInVO(self.vo)
if not diracUsers:
return S_ERROR("No VO users found for %s" % self.vo)
if refreshFlag:
result = self.csapi.downloadCSData()
if not result['OK']:
return result
result = self.csapi.describeUsers(diracUsers)
if not result['OK']:
self.log.error('Could not retrieve CS User description')
return result
def getVOUserReport(self):
""" Get a report string with the current status of the DIRAC Registry for the
Virtual Organization
:return: S_OK with the report string as Value
"""
result = self.getVOUserData(refreshFlag=True)
if not result['OK']:
return result
userDict = result['Value']
# Get DIRAC group vs VOMS Role Mappings
result = getVOMSRoleGroupMapping(self.vo)
if not result['OK']:
return result
diracVOMSMapping = result['Value']['DIRACVOMS']
records = []
groupDict = defaultdict(int)
multiDNUsers = {}
suspendedUsers = []
for user in userDict:
for group in userDict[user]['Groups']:
groupDict[group] += 1
dnList = fromChar(userDict[user]['DN'])
if len(dnList) > 1:
multiDNUsers[user] = dnList
if userDict[user].get('Status', 'Active') == 'Suspended':
suspendedUsers.append(user)
for group in diracVOMSMapping:
records.append((group, str(groupDict[group]), diracVOMSMapping.get(group, '')))
fields = ['Group', 'Number of users', 'VOMS Role']
output = printTable(fields, records, sortField='Group', printOut=False, numbering=False)
if multiDNUsers:
output += '\nUsers with multiple DNs:\n'
for user in multiDNUsers:
output += ' %s:\n' % user
for dn in multiDNUsers[user]:
output += ' %s\n' % dn
if suspendedUsers:
output += '\n%d suspended users:\n' % len(suspendedUsers)
output += ' %s' % ','.join(suspendedUsers)
return S_OK(output)
def getUserName(self, dn):
""" Utility to construct user name
:param str dn: user DN
:return str: user name
"""
name = self.vomsUserDict[dn].get('name')
surname = self.vomsUserDict[dn].get('surname')
if name and surname:
surnameName = _getUserNameFromSurname(name, surname)
return surnameName
mail = self.vomsUserDict[dn]['mail']
dnName = _getUserNameFromDN(dn, self.vo)
mailName = _getUserNameFromMail(mail)
# If robot, take the dn based name
if dnName.startswith('robot'):
return dnName
# Is mailName reasonable ?
if len(mailName) > 5 and mailName.isalpha():
return mailName
# dnName too long
if len(dnName) >= 12:
return dnName[:11]
# May be the mail name is still more reasonable
if len(dnName) < len(mailName) and mailName.isalpha():
return mailName
return dnName
def __syncCSWithVOMS(self):
self.__adminMsgs = {'Errors': [], 'Info': []}
#Get DIRAC VOMS Mapping
self.log.info("Getting DIRAC VOMS mapping")
mappingSection = '/Registry/VOMS/Mapping'
ret = gConfig.getOptionsDict(mappingSection)
if not ret['OK']:
self.log.fatal('No VOMS to DIRAC Group Mapping Available')
return ret
vomsMapping = ret['Value']
self.log.info("There are %s registered voms mappings in DIRAC" %
len(vomsMapping))
#Get VOMS VO name
self.log.info("Getting VOMS VO name")
result = self.vomsSrv.admGetVOName()
if not ret['OK']:
self.log.fatal('Could not retrieve VOMS VO name')
voNameInVOMS = result['Value']
self.log.info("VOMS VO Name is %s" % voNameInVOMS)
#Get VOMS roles
self.log.info("Getting the list of registered roles in VOMS")
result = self.vomsSrv.admListRoles()
if not ret['OK']:
self.log.fatal('Could not retrieve registered roles in VOMS')
rolesInVOMS = result['Value']
self.log.info("There are %s registered roles in VOMS" %
len(rolesInVOMS))
print rolesInVOMS
rolesInVOMS.append('')
#Map VOMS roles
vomsRoles = {}
for role in rolesInVOMS:
if role:
role = "%s/%s" % (voNameInVOMS, role)
else:
role = voNameInVOMS
groupsForRole = []
for group in vomsMapping:
if vomsMapping[group] == role:
groupsForRole.append(group)
if groupsForRole:
vomsRoles[role] = {'Groups': groupsForRole, 'Users': []}
self.log.info("DIRAC valid VOMS roles are:\n\t",
"\n\t ".join(vomsRoles.keys()))
#Get DIRAC users
self.log.info("Getting the list of registered users in DIRAC")
csapi = CSAPI()
ret = csapi.listUsers()
if not ret['OK']:
self.log.fatal('Could not retrieve current list of Users')
return ret
currentUsers = ret['Value']
ret = csapi.describeUsers(currentUsers)
if not ret['OK']:
self.log.fatal('Could not retrieve current User description')
return ret
currentUsers = ret['Value']
self.__adminMsgs['Info'].append(
"There are %s registered users in DIRAC" % len(currentUsers))
self.log.info("There are %s registered users in DIRAC" %
len(currentUsers))
#Get VOMS user entries
self.log.info("Getting the list of registered user entries in VOMS")
result = self.vomsSrv.admListMembers()
if not ret['OK']:
self.log.fatal(
'Could not retrieve registered user entries in VOMS')
usersInVOMS = result['Value']
self.__adminMsgs['Info'].append(
"There are %s registered user entries in VOMS" % len(usersInVOMS))
self.log.info("There are %s registered user entries in VOMS" %
len(usersInVOMS))
#Consolidate users by nickname
usersData = {}
newUserNames = []
knownUserNames = []
obsoleteUserNames = []
self.log.info("Retrieving usernames...")
usersInVOMS.sort()
for iUPos in range(len(usersInVOMS)):
userName = ''
user = usersInVOMS[iUPos]
for oldUser in currentUsers:
if user['DN'].strip() in List.fromChar(
currentUsers[oldUser]['DN']):
userName = oldUser
if not userName:
result = self.vomsSrv.attGetUserNickname(
user['DN'], user['CA'])
if result['OK']:
userName = result['Value']
else:
self.__adminMsgs['Errors'].append(
"Could not retrieve nickname for DN %s" % user['DN'])
self.log.error("Could not get nickname for DN", user['DN'])
userName = user['mail'][:user['mail'].find('@')]
if not userName:
self.log.error("Empty nickname for DN", user['DN'])
self.__adminMsgs['Errors'].append("Empty nickname for DN %s" %
user['DN'])
continue
self.log.info(
" (%02d%%) Found username %s : %s " %
((iUPos * 100 / len(usersInVOMS)), userName, user['DN']))
if userName not in usersData:
usersData[userName] = {
'DN': [],
'CA': [],
'Email': [],
'Groups': ['user']
}
for key in ('DN', 'CA', 'mail'):
value = user[key]
if value:
if key == "mail":
List.appendUnique(usersData[userName]['Email'], value)
else:
usersData[userName][key].append(value.strip())
if userName not in currentUsers:
List.appendUnique(newUserNames, userName)
else:
List.appendUnique(knownUserNames, userName)
self.log.info("Finished retrieving usernames")
if newUserNames:
self.log.info("There are %s new users" % len(newUserNames))
else:
self.log.info("There are no new users")
#Get the list of users for each group
result = csapi.listGroups()
if not result['OK']:
self.log.error("Could not get the list of groups in DIRAC",
result['Message'])
return result
staticGroups = result['Value']
vomsGroups = []
self.log.info("Mapping users in VOMS to groups")
for vomsRole in vomsRoles:
self.log.info(" Getting users for role %s" % vomsRole)
groupsForRole = vomsRoles[vomsRole]['Groups']
vomsMap = vomsRole.split("Role=")
for g in groupsForRole:
if g in staticGroups:
staticGroups.pop(staticGroups.index(g))
else:
vomsGroups.append(g)
if len(vomsMap) == 1:
# no Role
users = usersInVOMS
else:
vomsGroup = "Role=".join(vomsMap[:-1])
if vomsGroup[-1] == "/":
vomsGroup = vomsGroup[:-1]
vomsRole = "Role=%s" % vomsMap[-1]
result = self.vomsSrv.admListUsersWithRole(vomsGroup, vomsRole)
if not result['OK']:
errorMsg = "Could not get list of users for VOMS %s" % (
vomsMapping[group])
self.__adminMsgs['Errors'].append(errorMsg)
self.log.error(errorMsg, result['Message'])
return result
users = result['Value']
numUsersInGroup = 0
for vomsUser in users:
for userName in usersData:
if vomsUser['DN'] in usersData[userName]['DN']:
numUsersInGroup += 1
usersData[userName]['Groups'].extend(groupsForRole)
infoMsg = "There are %s users in group(s) %s for VOMS Role %s" % (
numUsersInGroup, ",".join(groupsForRole), vomsRole)
self.__adminMsgs['Info'].append(infoMsg)
self.log.info(" %s" % infoMsg)
self.log.info("Checking static groups")
staticUsers = []
for group in staticGroups:
self.log.info(" Checking static group %s" % group)
numUsersInGroup = 0
result = csapi.listUsers(group)
if not result['OK']:
self.log.error(
"Could not get the list of users in DIRAC group %s" %
group, result['Message'])
return result
for userName in result['Value']:
if userName in usersData:
numUsersInGroup += 1
usersData[userName]['Groups'].append(group)
else:
if group not in vomsGroups and userName not in staticUsers:
staticUsers.append(userName)
infoMsg = "There are %s users in group %s" % (numUsersInGroup,
group)
self.__adminMsgs['Info'].append(infoMsg)
self.log.info(" %s" % infoMsg)
if staticUsers:
infoMsg = "There are %s static users: %s" % (
len(staticUsers), ', '.join(staticUsers))
self.__adminMsgs['Info'].append(infoMsg)
self.log.info("%s" % infoMsg)
for user in currentUsers:
if user not in usersData and user not in staticUsers:
self.log.info('User %s is no longer valid' % user)
obsoleteUserNames.append(user)
#Do the CS Sync
self.log.info("Updating CS...")
ret = csapi.downloadCSData()
if not ret['OK']:
self.log.fatal('Can not update from CS', ret['Message'])
return ret
usersWithMoreThanOneDN = {}
for user in usersData:
csUserData = dict(usersData[user])
if len(csUserData['DN']) > 1:
usersWithMoreThanOneDN[user] = csUserData['DN']
result = csapi.describeUsers([user])
if result['OK']:
if result['Value']:
prevUser = result['Value'][user]
prevDNs = List.fromChar(prevUser['DN'])
newDNs = csUserData['DN']
for DN in newDNs:
if DN not in prevDNs:
self.__adminMsgs['Info'].append(
"User %s has new DN %s" % (user, DN))
for DN in prevDNs:
if DN not in newDNs:
self.__adminMsgs['Info'].append(
"User %s has lost a DN %s" % (user, DN))
else:
newDNs = csUserData['DN']
for DN in newDNs:
self.__adminMsgs['Info'].append(
"New user %s has new DN %s" % (user, DN))
for k in ('DN', 'CA', 'Email'):
csUserData[k] = ", ".join(csUserData[k])
result = csapi.modifyUser(user,
csUserData,
createIfNonExistant=True)
if not result['OK']:
self.__adminMsgs['Error'].append("Cannot modify user %s: %s" %
(user, result['Message']))
self.log.error("Cannot modify user", user)
if usersWithMoreThanOneDN:
self.__adminMsgs['Info'].append("\nUsers with more than one DN:")
for uwmtod in sorted(usersWithMoreThanOneDN):
self.__adminMsgs['Info'].append(" %s" % uwmtod)
self.__adminMsgs['Info'].append(" + DN list:")
for DN in usersWithMoreThanOneDN[uwmtod]:
self.__adminMsgs['Info'].append(" - %s" % DN)
if obsoleteUserNames:
self.__adminMsgs['Info'].append("\nObsolete users:")
address = self.am_getOption('MailTo', '*****@*****.**')
fromAddress = self.am_getOption('mailFrom', '*****@*****.**')
subject = 'Obsolete LFC Users found'
body = 'Delete entries into LFC: \n'
for obsoleteUser in obsoleteUserNames:
self.log.info(subject, ", ".join(obsoleteUserNames))
body += 'for ' + obsoleteUser + '\n'
self.__adminMsgs['Info'].append(" %s" % obsoleteUser)
self.log.info("Deleting %s users" % len(obsoleteUserNames))
NotificationClient().sendMail(address,
'UsersAndGroupsAgent: %s' % subject,
body, fromAddress)
csapi.deleteUsers(obsoleteUserNames)
if newUserNames:
self.__adminMsgs['Info'].append("\nNew users:")
for newUser in newUserNames:
self.__adminMsgs['Info'].append(" %s" % newUser)
self.__adminMsgs['Info'].append(" + DN list:")
for DN in usersData[newUser]['DN']:
self.__adminMsgs['Info'].append(" - %s" % DN)
self.__adminMsgs['Info'].append(" + EMail: %s" %
usersData[newUser]['Email'])
result = csapi.commitChanges()
if not result['OK']:
self.log.error("Could not commit configuration changes",
result['Message'])
return result
self.log.info("Configuration committed")
#LFC Check
if self.am_getOption("LFCCheckEnabled", True):
result = self.checkLFCRegisteredUsers(usersData)
if not result['OK']:
return result
return S_OK()
def checkUnusedSEs():
global vo, dry
result = getGridSRMs( vo, unUsed = True )
if not result['OK']:
gLogger.error( 'Failed to look up SRMs in BDII', result['Message'] )
siteSRMDict = result['Value']
# Evaluate VOs
result = getVOs()
if result['OK']:
csVOs = set( result['Value'] )
else:
csVOs = set( [vo] )
changeSetFull = set()
for site in siteSRMDict:
for gridSE in siteSRMDict[site]:
changeSet = set()
seDict = siteSRMDict[site][gridSE]['SE']
srmDict = siteSRMDict[site][gridSE]['SRM']
# Check the SRM version
version = srmDict.get( 'GlueServiceVersion', '' )
if not ( version and version.startswith( '2' ) ):
gLogger.debug( 'Skipping SRM service with version %s' % version )
continue
result = getDIRACSiteName( site )
if not result['OK']:
gLogger.notice( 'Unused se %s is detected at unused site %s' % ( gridSE, site ) )
gLogger.notice( 'Consider adding site %s to the DIRAC CS' % site )
continue
diracSites = result['Value']
yn = raw_input( '\nDo you want to add new SRM SE %s at site(s) %s ? default yes [yes|no]: ' % ( gridSE, str( diracSites ) ) )
if not yn or yn.lower().startswith( 'y' ):
if len( diracSites ) > 1:
prompt = 'Which DIRAC site the new SE should be attached to ?'
for i, s in enumerate( diracSites ):
prompt += '\n[%d] %s' % ( i, s )
prompt += '\nEnter your choice number: '
inp = raw_input( prompt )
try:
ind = int( inp )
except:
gLogger.notice( 'Can not interpret your choice: %s, try again later' % inp )
continue
diracSite = diracSites[ind]
else:
diracSite = diracSites[0]
domain, siteName, country = diracSite.split( '.' )
recName = '%s-disk' % siteName
inp = raw_input( 'Give a DIRAC name to the grid SE %s, default %s : ' % ( gridSE, recName ) )
diracSEName = inp
if not inp:
diracSEName = recName
gLogger.notice( 'Adding new SE %s at site %s' % ( diracSEName, diracSite ) )
seSection = cfgPath( '/Resources/StorageElements', diracSEName )
changeSet.add( ( seSection, 'BackendType', seDict.get( 'GlueSEImplementationName', 'Unknown' ) ) )
changeSet.add( ( seSection, 'Description', seDict.get( 'GlueSEName', 'Unknown' ) ) )
bdiiVOs = set( [ re.sub( '^VO:', '', rule ) for rule in srmDict.get( 'GlueServiceAccessControlBaseRule', [] ) ] )
seVOs = csVOs.intersection( bdiiVOs )
changeSet.add( ( seSection, 'VO', ','.join( seVOs ) ) )
accessSection = cfgPath( seSection, 'AccessProtocol.1' )
changeSet.add( ( accessSection, 'Protocol', 'srm' ) )
changeSet.add( ( accessSection, 'ProtocolName', 'SRM2' ) )
endPoint = srmDict.get( 'GlueServiceEndpoint', '' )
result = pfnparse( endPoint )
if not result['OK']:
gLogger.error( 'Can not get the SRM service end point. Skipping ...' )
continue
host = result['Value']['Host']
port = result['Value']['Port']
changeSet.add( ( accessSection, 'Host', host ) )
changeSet.add( ( accessSection, 'Port', port ) )
changeSet.add( ( accessSection, 'Access', 'remote' ) )
voPathSection = cfgPath( accessSection, 'VOPath' )
if 'VOPath' in seDict:
path = seDict['VOPath']
voFromPath = os.path.basename( path )
if voFromPath != diracVO:
gLogger.notice( '\n!!! Warning: non-conventional VO path: %s\n' % path )
changeSet.add( ( voPathSection, diracVO, path ) )
path = os.path.dirname( path )
else:
# Try to guess the Path
domain = '.'.join( host.split( '.' )[-2:] )
path = '/dpm/%s/home' % domain
changeSet.add( ( accessSection, 'Path', path ) )
changeSet.add( ( accessSection, 'SpaceToken', '' ) )
changeSet.add( ( accessSection, 'WSUrl', '/srm/managerv2?SFN=' ) )
gLogger.notice( 'SE %s will be added with the following parameters' % diracSEName )
changeList = list( changeSet )
changeList.sort()
for entry in changeList:
gLogger.notice( entry )
yn = raw_input( 'Do you want to add new SE %s ? default yes [yes|no]: ' % diracSEName )
if not yn or yn.lower().startswith( 'y' ):
changeSetFull = changeSetFull.union( changeSet )
if dry:
if changeSetFull:
gLogger.notice( 'Skipping commit of the new SE data in a dry run' )
else:
gLogger.notice( "No new SE to be added" )
return S_OK()
if changeSetFull:
csAPI = CSAPI()
csAPI.initialize()
result = csAPI.downloadCSData()
if not result['OK']:
gLogger.error( 'Failed to initialize CSAPI object', result['Message'] )
DIRACExit( -1 )
changeList = list( changeSetFull )
changeList.sort()
for section, option, value in changeList:
csAPI.setOption( cfgPath( section, option ), value )
yn = raw_input( 'New SE data is accumulated\n Do you want to commit changes to CS ? default yes [yes|no]: ' )
if not yn or yn.lower().startswith( 'y' ):
result = csAPI.commit()
if not result['OK']:
gLogger.error( "Error while commit to CS", result['Message'] )
else:
gLogger.notice( "Successfully committed %d changes to CS" % len( changeSetFull ) )
else:
gLogger.notice( "No new SE to be added" )
return S_OK()
class Bdii2CSAgent(AgentModule):
def __init__(self, *args, **kwargs):
""" Defines default parameters
"""
super(Bdii2CSAgent, self).__init__(*args, **kwargs)
self.addressTo = ''
self.addressFrom = ''
self.voName = []
self.subject = "Bdii2CSAgent"
self.alternativeBDIIs = []
self.voBdiiCEDict = {}
self.voBdiiSEDict = {}
self.csAPI = None
# What to get
self.processCEs = True
self.processSEs = False
# Update the CS or not?
self.dryRun = False
def initialize(self):
""" Gets run paramaters from the configuration
"""
self.addressTo = self.am_getOption('MailTo', self.addressTo)
self.addressFrom = self.am_getOption('MailFrom', self.addressFrom)
# Create a list of alternative bdii urls
self.alternativeBDIIs = self.am_getOption('AlternativeBDIIs',
self.alternativeBDIIs)
# Check if the bdii url is appended by a port number, if not append the default 2170
for index, url in enumerate(self.alternativeBDIIs):
if not url.split(':')[-1].isdigit():
self.alternativeBDIIs[index] += ':2170'
if self.addressTo and self.addressFrom:
self.log.info("MailTo", self.addressTo)
self.log.info("MailFrom", self.addressFrom)
if self.alternativeBDIIs:
self.log.info("AlternativeBDII URLs:", self.alternativeBDIIs)
self.processCEs = self.am_getOption('ProcessCEs', self.processCEs)
self.processSEs = self.am_getOption('ProcessSEs', self.processSEs)
self.dryRun = self.am_getOption('DryRun', self.dryRun)
self.voName = self.am_getOption('VirtualOrganization', self.voName)
if not self.voName:
self.voName = self.am_getOption('VO', [])
if not self.voName or (len(self.voName) == 1
and self.voName[0].lower() == 'all'):
# Get all VOs defined in the configuration
self.voName = []
result = getVOs()
if result['OK']:
vos = result['Value']
for vo in vos:
vomsVO = getVOOption(vo, "VOMSName")
if vomsVO:
self.voName.append(vomsVO)
if self.voName:
self.log.info("Agent will manage VO(s) %s" % self.voName)
else:
self.log.fatal("VirtualOrganization option not defined for agent")
return S_ERROR()
self.csAPI = CSAPI()
return self.csAPI.initialize()
def execute(self):
""" General agent execution method
"""
self.voBdiiCEDict = {}
self.voBdiiSEDict = {}
# Get a "fresh" copy of the CS data
result = self.csAPI.downloadCSData()
if not result['OK']:
self.log.warn("Could not download a fresh copy of the CS data",
result['Message'])
# Refresh the configuration from the master server
gConfig.forceRefresh(fromMaster=True)
if self.processCEs:
self.__lookForNewCEs()
self.__updateCEs()
if self.processSEs:
self.__lookForNewSEs()
self.__updateSEs()
return S_OK()
def __lookForNewCEs(self):
""" Look up BDII for CEs not yet present in the DIRAC CS
"""
bannedCEs = self.am_getOption('BannedCEs', [])
result = getCEsFromCS()
if not result['OK']:
return result
knownCEs = set(result['Value'])
knownCEs = knownCEs.union(set(bannedCEs))
for vo in self.voName:
result = self.__getBdiiCEInfo(vo)
if not result['OK']:
continue
bdiiInfo = result['Value']
result = getGridCEs(vo, bdiiInfo=bdiiInfo, ceBlackList=knownCEs)
if not result['OK']:
self.log.error('Failed to get unused CEs', result['Message'])
siteDict = result['Value']
body = ''
for site in siteDict:
newCEs = set(siteDict[site].keys()) # pylint: disable=no-member
if not newCEs:
continue
ceString = ''
for ce in newCEs:
queueString = ''
ceInfo = bdiiInfo[site]['CEs'][ce]
newCEString = "CE: %s, GOCDB Site Name: %s" % (ce, site)
systemTuple = siteDict[site][ce]['System']
osString = "%s_%s_%s" % (systemTuple)
newCEString = "\n%s\n%s\n" % (newCEString, osString)
for queue in ceInfo['Queues']:
queueStatus = ceInfo['Queues'][queue].get(
'GlueCEStateStatus', 'UnknownStatus')
if 'production' in queueStatus.lower():
ceType = ceInfo['Queues'][queue].get(
'GlueCEImplementationName', '')
queueString += " %s %s %s\n" % (
queue, queueStatus, ceType)
if queueString:
ceString += newCEString
ceString += "Queues:\n"
ceString += queueString
if ceString:
body += ceString
if body:
body = "\nWe are glad to inform You about new CE(s) possibly suitable for %s:\n" % vo + body
body += "\n\nTo suppress information about CE add its name to BannedCEs list.\n"
body += "Add new Sites/CEs for vo %s with the command:\n" % vo
body += "dirac-admin-add-resources --vo %s --ce\n" % vo
self.log.info(body)
if self.addressTo and self.addressFrom:
notification = NotificationClient()
result = notification.sendMail(self.addressTo,
self.subject,
body,
self.addressFrom,
localAttempt=False)
if not result['OK']:
self.log.error(
'Can not send new site notification mail',
result['Message'])
return S_OK()
def __getBdiiCEInfo(self, vo):
if vo in self.voBdiiCEDict:
return S_OK(self.voBdiiCEDict[vo])
self.log.info("Check for available CEs for VO", vo)
totalResult = S_OK({})
message = ''
mainResult = getBdiiCEInfo(vo)
if not mainResult['OK']:
self.log.error("Failed getting information from default bdii",
mainResult['Message'])
message = mainResult['Message']
for bdii in reversed(self.alternativeBDIIs):
resultAlt = getBdiiCEInfo(vo, host=bdii)
if resultAlt['OK']:
totalResult['Value'].update(resultAlt['Value'])
else:
self.log.error("Failed getting information from %s " % bdii,
resultAlt['Message'])
message = (message + "\n" + resultAlt['Message']).strip()
if mainResult['OK']:
totalResult['Value'].update(mainResult['Value'])
if not totalResult[
'Value'] and message: ## Dict is empty and we have an error message
self.log.error("Error during BDII request", message)
totalResult = S_ERROR(message)
else:
self.voBdiiCEDict[vo] = totalResult['Value']
return totalResult
def __getBdiiSEInfo(self, vo):
if vo in self.voBdiiSEDict:
return S_OK(self.voBdiiSEDict[vo])
self.log.info("Check for available SEs for VO", vo)
result = getBdiiSEInfo(vo)
message = ''
if not result['OK']:
message = result['Message']
for bdii in self.alternativeBDIIs:
result = getBdiiSEInfo(vo, host=bdii)
if result['OK']:
break
if not result['OK']:
if message:
self.log.error("Error during BDII request", message)
else:
self.log.error("Error during BDII request", result['Message'])
else:
self.voBdiiSEDict[vo] = result['Value']
return result
def __updateCEs(self):
""" Update the Site/CE/queue settings in the CS if they were changed in the BDII
"""
bdiiChangeSet = set()
for vo in self.voName:
result = self.__getBdiiCEInfo(vo)
if not result['OK']:
continue
ceBdiiDict = result['Value']
result = getSiteUpdates(vo, bdiiInfo=ceBdiiDict, log=self.log)
if not result['OK']:
continue
bdiiChangeSet = bdiiChangeSet.union(result['Value'])
# We have collected all the changes, consolidate VO settings
result = self.__updateCS(bdiiChangeSet)
return result
def __updateCS(self, bdiiChangeSet):
queueVODict = {}
changeSet = set()
for entry in bdiiChangeSet:
section, option, _value, new_value = entry
if option == "VO":
queueVODict.setdefault(section, set())
queueVODict[section] = queueVODict[section].union(
set(new_value.split(',')))
else:
changeSet.add(entry)
for section, VOs in queueVODict.items():
changeSet.add((section, 'VO', '', ','.join(VOs)))
if changeSet:
changeList = list(changeSet)
changeList.sort()
body = '\n'.join(
["%s/%s %s -> %s" % entry for entry in changeList])
if body and self.addressTo and self.addressFrom:
notification = NotificationClient()
result = notification.sendMail(self.addressTo,
self.subject,
body,
self.addressFrom,
localAttempt=False)
if body:
self.log.info(
'The following configuration changes were detected:')
self.log.info(body)
for section, option, value, new_value in changeSet:
if value == 'Unknown' or not value:
self.csAPI.setOption(cfgPath(section, option), new_value)
else:
self.csAPI.modifyValue(cfgPath(section, option), new_value)
if self.dryRun:
self.log.info("Dry Run: CS won't be updated")
self.csAPI.showDiff()
else:
result = self.csAPI.commit()
if not result['OK']:
self.log.error("Error while committing to CS",
result['Message'])
else:
self.log.info("Successfully committed %d changes to CS" %
len(changeList))
return result
else:
self.log.info("No changes found")
return S_OK()
def __lookForNewSEs(self):
""" Look up BDII for SEs not yet present in the DIRAC CS
"""
bannedSEs = self.am_getOption('BannedSEs', [])
result = getSEsFromCS()
if not result['OK']:
return result
knownSEs = set(result['Value'])
knownSEs = knownSEs.union(set(bannedSEs))
for vo in self.voName:
result = self.__getBdiiSEInfo(vo)
if not result['OK']:
continue
bdiiInfo = result['Value']
result = getGridSRMs(vo, bdiiInfo=bdiiInfo, srmBlackList=knownSEs)
if not result['OK']:
continue
siteDict = result['Value']
body = ''
for site in siteDict:
newSEs = set(siteDict[site].keys()) # pylint: disable=no-member
if not newSEs:
continue
for se in newSEs:
body += '\n New SE %s available at site %s:\n' % (se, site)
backend = siteDict[site][se]['SE'].get(
'GlueSEImplementationName', 'Unknown')
size = siteDict[site][se]['SE'].get(
'GlueSESizeTotal', 'Unknown')
body += ' Backend %s, Size %s' % (backend, size)
if body:
body = "\nWe are glad to inform You about new SE(s) possibly suitable for %s:\n" % vo + body
body += "\n\nTo suppress information about an SE add its name to BannedSEs list.\n"
body += "Add new SEs for vo %s with the command:\n" % vo
body += "dirac-admin-add-resources --vo %s --se\n" % vo
self.log.info(body)
if self.addressTo and self.addressFrom:
notification = NotificationClient()
result = notification.sendMail(self.addressTo,
self.subject,
body,
self.addressFrom,
localAttempt=False)
if not result['OK']:
self.log.error(
'Can not send new site notification mail',
result['Message'])
return S_OK()
def __updateSEs(self):
""" Update the Storage Element settings in the CS if they were changed in the BDII
"""
bdiiChangeSet = set()
for vo in self.voName:
result = self.__getBdiiSEInfo(vo)
if not result['OK']:
continue
seBdiiDict = result['Value']
result = getSRMUpdates(vo, bdiiInfo=seBdiiDict)
if not result['OK']:
continue
bdiiChangeSet = bdiiChangeSet.union(result['Value'])
# We have collected all the changes, consolidate VO settings
result = self.__updateCS(bdiiChangeSet)
return result
class VOMS2CSAgent( AgentModule ):
def __init__( self, *args, **kwargs ):
""" Defines default parameters
"""
super(VOMS2CSAgent, self).__init__( *args, **kwargs )
self.__voDict = {}
self.__adminMsgs = {}
self.csapi = None
self.voChanged = False
self.dryRun = False
self.autoAddUsers = False
self.autoModifyUsers = False
self.autoDeleteUsers = False
self.detailedReport = True
self.makeFCEntry = False
def initialize( self ):
""" Initialize the default parameters
"""
voNames = self.am_getOption( 'VO', [] )
if not voNames[0].lower() == "none":
if voNames[0].lower() == "any":
voNames = []
result = getVOMSVOs( voNames )
if not result['OK']:
return result
self.__voDict = result['Value']
self.log.notice( "VOs: %s" % self.__voDict.keys() )
self.csapi = CSAPI()
self.dryRun = self.am_getOption( 'DryRun', self.dryRun )
self.autoAddUsers = self.am_getOption( 'AutoAddUsers', self.autoAddUsers )
self.autoModifyUsers = self.am_getOption( 'AutoModifyUsers', self.autoModifyUsers )
self.autoDeleteUsers = self.am_getOption( 'AutoDeleteUsers', self.autoDeleteUsers )
self.detailedReport = self.am_getOption( 'DetailedReport', self.detailedReport )
self.makeFCEntry = self.am_getOption( 'MakeHomeDirectory', self.makeFCEntry )
return S_OK()
def execute( self ):
self.__adminMsgs = {}
self.csapi.downloadCSData()
for vo in self.__voDict:
self.voChanged = False
voAdminUser = getVOOption( vo, "VOAdmin")
voAdminMail = None
if voAdminUser:
voAdminMail = getUserOption( voAdminUser, "Email")
voAdminGroup = getVOOption( vo, "VOAdminGroup", getVOOption( vo, "DefaultGroup" ) )
self.log.info( 'Performing VOMS sync for VO %s with credentials %s@%s' % ( vo, voAdminUser, voAdminGroup ) )
result = self.__syncCSWithVOMS( vo, proxyUserName = voAdminUser, proxyUserGroup = voAdminGroup ) #pylint: disable=unexpected-keyword-arg
if not result['OK']:
self.log.error( 'Failed to perform VOMS to CS synchronization:', 'VO %s: %s' % ( vo, result["Message"] ) )
continue
resultDict = result['Value']
newUsers = resultDict.get( "NewUsers", [] )
modUsers = resultDict.get( "ModifiedUsers", [] )
delUsers = resultDict.get( "DeletedUsers", [] )
self.log.info( "Run results: new users %d, modified users %d, deleted users %d" % \
( len( newUsers ), len( modUsers ), len( delUsers ) ) )
if self.csapi.csModified:
# We have accumulated all the changes, commit them now
self.log.info( "There are changes to the CS for vo %s ready to be committed" % vo )
if self.dryRun:
self.log.info( "Dry Run: CS won't be updated" )
self.csapi.showDiff()
else:
result = self.csapi.commitChanges()
if not result[ 'OK' ]:
self.log.error( "Could not commit configuration changes", result[ 'Message' ] )
return result
self.log.notice( "Configuration committed for VO %s" % vo )
else:
self.log.info( "No changes to the CS for VO %s recorded at this cycle" % vo )
# Add user home directory in the file catalog
if self.makeFCEntry and newUsers:
self.log.info( "Creating home directories for users %s" % str( newUsers ) )
result = self.__addHomeDirectory( vo, newUsers, proxyUserName = voAdminUser, proxyUserGroup = voAdminGroup ) #pylint: disable=unexpected-keyword-arg
if not result['OK']:
self.log.error( 'Failed to create user home directories:', 'VO %s: %s' % ( vo, result["Message"] ) )
else:
for user in result['Value']['Failed']:
self.log.error( "Failed to create home directory", "user: %s, operation: %s" % \
( user, result['Value']['Failed'][user] ) )
self.__adminMsgs[ 'Errors' ].append( "Failed to create home directory for user %s: operation %s" % \
( user, result['Value']['Failed'][user] ) )
for user in result['Value']['Successful']:
self.__adminMsgs[ 'Info' ].append( "Created home directory for user %s" % user )
if self.voChanged or self.detailedReport:
mailMsg = ""
if self.__adminMsgs[ 'Errors' ]:
mailMsg += "\nErrors list:\n %s" % "\n ".join( self.__adminMsgs[ 'Errors' ] )
if self.__adminMsgs[ 'Info' ]:
mailMsg += "\nRun result:\n %s" % "\n ".join( self.__adminMsgs[ 'Info' ] )
if self.detailedReport:
result = self.getVOUserReport( vo )
if result['OK']:
mailMsg += '\n\n'
mailMsg += result['Value']
else:
mailMsg += 'Failed to produce a detailed user report'
mailMsg += result['Message']
NotificationClient().sendMail( self.am_getOption( 'MailTo', voAdminMail ),
"VOMS2CSAgent run log", mailMsg,
self.am_getOption( 'MailFrom', self.am_getOption( 'mailFrom', "DIRAC system" ) ) )
return S_OK()
def getVOUserData( self, vo, refreshFlag = False ):
""" Get a report for users of a given VO
:param str vo: VO name
:return: S_OK/S_ERROR, Value = user description dictionary
"""
if refreshFlag:
gConfig.forceRefresh()
# Get DIRAC users
diracUsers = getUsersInVO( vo )
if not diracUsers:
return S_ERROR( "No VO users found for %s" % vo )
if refreshFlag:
result = self.csapi.downloadCSData()
if not result['OK']:
return result
result = self.csapi.describeUsers( diracUsers )
if not result['OK']:
self.log.error( 'Could not retrieve CS User description' )
return result
def getVOUserReport( self, vo ):
"""
:param str vo: VO name
:return: report string
"""
result = self.getVOUserData( vo, refreshFlag = True )
if not result['OK']:
return result
userDict = result['Value']
# Get DIRAC group vs VOMS Role Mappings
result = getVOMSRoleGroupMapping( vo )
if not result['OK']:
return result
diracVOMSMapping = result['Value']['DIRACVOMS']
records = []
groupDict = defaultdict( int )
multiDNUsers = {}
suspendedUsers = []
for user in userDict:
for group in userDict[user]['Groups']:
groupDict[group] += 1
dnList = fromChar( userDict[user]['DN'] )
if len( dnList ) > 1:
multiDNUsers[user] = dnList
if userDict[user].get( 'Status', 'Active' ) == 'Suspended':
suspendedUsers.append( user )
for group in diracVOMSMapping:
records.append( ( group, str( groupDict[group] ), diracVOMSMapping.get( group, '' ) ) )
fields = [ 'Group', 'Number of users', 'VOMS Role' ]
output = printTable( fields, records, sortField = 'Group', printOut = False, numbering = False )
if multiDNUsers:
output += '\nUsers with multiple DNs:\n'
for user in multiDNUsers:
output += ' %s:\n' % user
for dn in multiDNUsers[user]:
output += ' %s\n' % dn
if suspendedUsers:
output += '\n%d suspended users:\n' % len( suspendedUsers )
output += ' %s' % ','.join( suspendedUsers )
return S_OK( output )
@executeWithUserProxy
def __syncCSWithVOMS( self, vo ):
self.__adminMsgs = { 'Errors' : [], 'Info' : [] }
resultDict = defaultdict( list )
# Get DIRAC group vs VOMS Role Mappings
result = getVOMSRoleGroupMapping( vo )
if not result['OK']:
return result
vomsDIRACMapping = result['Value']['VOMSDIRAC']
diracVOMSMapping = result['Value']['DIRACVOMS']
noVOMSGroups = result['Value']['NoVOMS']
noSyncVOMSGroups = result['Value']['NoSyncVOMS']
vomsSrv = VOMSService( vo )
# Get VOMS VO name
result = vomsSrv.admGetVOName()
if not result['OK']:
self.log.error( 'Could not retrieve VOMS VO name', "for %s" % vo )
return result
vomsVOName = result[ 'Value' ].lstrip( '/' )
self.log.verbose( "VOMS VO Name for %s is %s" % ( vo, vomsVOName ) )
# Get VOMS users
result = vomsSrv.getUsers()
if not result['OK']:
self.log.error( 'Could not retrieve user information from VOMS', result['Message'] )
return result
vomsUserDict = result[ 'Value' ]
message = "There are %s user entries in VOMS for VO %s" % ( len( vomsUserDict ), vomsVOName )
self.__adminMsgs[ 'Info' ].append( message )
self.log.info( message )
# Get DIRAC users
result = self.getVOUserData( vo )
if not result['OK']:
return result
diracUserDict = result['Value']
self.__adminMsgs[ 'Info' ].append( "There are %s registered users in DIRAC for VO %s" % ( len( diracUserDict ), vo ) )
self.log.info( "There are %s registered users in DIRAC VO %s" % ( len( diracUserDict ), vo ) )
# Find new and obsoleted user DNs
existingDNs = []
obsoletedDNs = []
newDNs = []
for user in diracUserDict:
dn = diracUserDict[user]['DN']
# We can have users with more than one DN registered
dnList = fromChar( dn )
existingDNs.extend( dnList )
for dn in dnList:
if dn not in vomsUserDict:
obsoletedDNs.append( dn )
for dn in vomsUserDict:
if dn not in existingDNs:
newDNs.append( dn )
allDiracUsers = getAllUsers()
nonVOUserDict = {}
nonVOUsers = list( set( allDiracUsers ) - set( diracUserDict.keys() ) )
if nonVOUsers:
result = self.csapi.describeUsers( nonVOUsers )
if not result['OK']:
self.log.error( 'Could not retrieve CS User description' )
return result
nonVOUserDict = result['Value']
# Process users
defaultVOGroup = getVOOption( vo, "DefaultGroup", "%s_user" % vo )
newAddedUserDict = {}
for dn in vomsUserDict:
nickName = ''
newDNForExistingUser = ''
diracName = ''
if dn in existingDNs:
for user in diracUserDict:
if dn == diracUserDict[user]['DN']:
diracName = user
if dn in newDNs:
# Find if the DN is already registered in the DIRAC CS
for user in nonVOUserDict:
if dn == nonVOUserDict[user]['DN']:
diracName = user
# Check the nickName in the same VO to see if the user is already registered
# with another DN
result = vomsSrv.attGetUserNickname( dn, vomsUserDict[dn]['CA'] )
if result['OK']:
nickName = result['Value']
if nickName in diracUserDict or nickName in newAddedUserDict:
diracName = nickName
# This is a flag for adding the new DN to an already existing user
newDNForExistingUser = dn
# We have a real new user
if not diracName:
if nickName:
newDiracName = nickName
else:
newDiracName = getUserName( dn, vomsUserDict[dn]['mail'], vo )
# If the chosen user name exists already, append a distinguishing suffix
ind = 1
trialName = newDiracName
while newDiracName in allDiracUsers:
# We have a user with the same name but with a different DN
newDiracName = "%s_%d" % ( trialName, ind )
ind += 1
# We now have everything to add the new user
userDict = { "DN": dn, "CA": vomsUserDict[dn]['CA'], "Email": vomsUserDict[dn]['mail'] }
groupsWithRole = []
for role in vomsUserDict[dn]['Roles']:
fullRole = "/%s/%s" % ( vomsVOName, role )
groupList = vomsDIRACMapping.get( fullRole, [] )
for group in groupList:
if group not in noSyncVOMSGroups:
groupsWithRole.append( group )
userDict['Groups'] = list( set( groupsWithRole + [defaultVOGroup] ) )
message = "\n Added new user %s:\n" % newDiracName
for key in userDict:
message += " %s: %s\n" % ( key, str( userDict[key] ) )
self.__adminMsgs[ 'Info' ].append( message )
self.voChanged = True
if self.autoAddUsers:
self.log.info( "Adding new user %s: %s" % ( newDiracName, str( userDict ) ) )
result = self.csapi.modifyUser( newDiracName, userDict, createIfNonExistant = True )
if not result['OK']:
self.log.warn( 'Failed adding new user %s' % newDiracName )
resultDict['NewUsers'].append( newDiracName )
newAddedUserDict[newDiracName] = userDict
continue
# We have an already existing user
modified = False
userDict = { "DN": dn, "CA": vomsUserDict[dn]['CA'], "Email": vomsUserDict[dn]['mail'] }
if newDNForExistingUser:
userDict['DN'] = ','.join( [ dn, diracUserDict[diracName]['DN'] ] )
modified = True
existingGroups = diracUserDict.get( diracName, {} ).get( 'Groups', [] )
nonVOGroups = list( set( existingGroups ) - set( diracVOMSMapping.keys() ) )
groupsWithRole = []
for role in vomsUserDict[dn]['Roles']:
fullRole = "/%s/%s" % ( vomsVOName, role )
groupList = vomsDIRACMapping.get( fullRole, [] )
for group in groupList:
if group not in noSyncVOMSGroups:
groupsWithRole.append( group )
keepGroups = nonVOGroups + groupsWithRole + [defaultVOGroup]
for group in existingGroups:
if group in nonVOGroups:
continue
role = diracVOMSMapping.get( group, '' )
# Among already existing groups for the user keep those without a special VOMS Role
# because this membership is done by hand in the CS
if not "Role" in role:
keepGroups.append( group )
# Keep existing groups with no VOMS attribute if any
if group in noVOMSGroups:
keepGroups.append( group )
# Keep groups for which syncronization with VOMS is forbidden
if group in noSyncVOMSGroups:
keepGroups.append( group )
userDict['Groups'] = list( set( keepGroups ) )
# Merge together groups for the same user but different DNs
if diracName in newAddedUserDict:
otherGroups = newAddedUserDict[diracName].get( 'Groups', [] )
userDict['Groups'] = list( set( keepGroups + otherGroups ) )
modified = True
# Check if something changed before asking CSAPI to modify
if diracName in diracUserDict:
message = "\n Modified user %s:\n" % diracName
modMsg = ''
for key in userDict:
if key == "Groups":
addedGroups = set( userDict[key] ) - set( diracUserDict.get( diracName, {} ).get( key, [] ) )
removedGroups = set( diracUserDict.get( diracName, {} ).get( key, [] ) ) - set( userDict[key] )
if addedGroups:
modMsg += " Added to group(s) %s\n" % ','.join( addedGroups )
if removedGroups:
modMsg += " Removed from group(s) %s\n" % ','.join( removedGroups )
else:
oldValue = str( diracUserDict.get( diracName, {} ).get( key, '' ) )
if str( userDict[key] ) != oldValue:
modMsg += " %s: %s -> %s\n" % ( key, oldValue, str( userDict[key] ) )
if modMsg:
self.__adminMsgs[ 'Info' ].append( message + modMsg )
modified = True
if self.autoModifyUsers and modified:
result = self.csapi.modifyUser( diracName, userDict )
if result['OK'] and result['Value']:
self.log.info( "Modified user %s: %s" % ( diracName, str( userDict ) ) )
self.voChanged = True
resultDict['ModifiedUsers'].append( diracName )
# Check if there are potentially obsoleted users
oldUsers = set()
for user in diracUserDict:
dnSet = set( fromChar( diracUserDict[user]['DN'] ) )
if not dnSet.intersection( set( vomsUserDict.keys() ) ) and user not in nonVOUserDict:
for group in diracUserDict[user]['Groups']:
if group not in noVOMSGroups:
oldUsers.add( user )
# Check for obsoleted DNs
for user in diracUserDict:
dnSet = set( fromChar( diracUserDict[user]['DN'] ) )
for dn in dnSet:
if dn in obsoletedDNs and user not in oldUsers:
self.log.verbose( "Modified user %s: dropped DN %s" % ( user, dn ) )
if self.autoModifyUsers:
userDict = diracUserDict[user]
modDNSet = dnSet - set( [dn] )
if modDNSet:
userDict['DN'] = ','.join( modDNSet )
result = self.csapi.modifyUser( user, userDict )
if result['OK'] and result['Value']:
self.log.info( "Modified user %s: dropped DN %s" % ( user, dn ) )
self.__adminMsgs[ 'Info' ].append( "Modified user %s: dropped DN %s" % ( user, dn ) )
self.voChanged = True
resultDict['ModifiedUsers'].append( diracName )
else:
oldUsers.add( user )
if oldUsers:
self.voChanged = True
if self.autoDeleteUsers:
self.log.info( 'The following users will be deleted: %s' % str( oldUsers ) )
result = self.csapi.deleteUsers( oldUsers )
if result['OK']:
self.__adminMsgs[ 'Info' ].append( 'The following users are deleted from CS:\n %s\n' % str( oldUsers ) )
resultDict['DeletedUsers'] = oldUsers
else:
self.__adminMsgs[ 'Errors' ].append( 'Error in deleting users from CS:\n %s' % str( oldUsers ) )
self.log.error( 'Error while user deletion from CS', result )
else:
self.__adminMsgs[ 'Info' ].append( 'The following users to be checked for deletion:\n %s' % str( oldUsers ) )
self.log.info( 'The following users to be checked for deletion: %s' % str( oldUsers ) )
return S_OK( resultDict )
@executeWithUserProxy
def __addHomeDirectory( self, vo, newUsers ):
fc = FileCatalog( vo = vo )
defaultVOGroup = getVOOption( vo, "DefaultGroup", "%s_user" % vo )
failed = {}
successful = {}
for user in newUsers:
result = fc.addUser( user )
if not result['OK']:
failed[user] = "addUser"
continue
dirName = '/%s/user/%s/%s' % ( vo, user[0], user )
result = fc.createDirectory( dirName )
if not result['OK']:
failed[user] = "createDirectory"
continue
result = fc.changePathOwner( { dirName: user }, recursive = True )
if not result['OK']:
failed[user] = "changePathOwner"
continue
result = fc.changePathGroup( { dirName: defaultVOGroup }, recursive = True )
if not result['OK']:
failed[user] = "changePathGroup"
continue
successful[user] = True
return S_OK( { "Successful": successful, "Failed": failed } )
class VOMS2CSAgent(AgentModule):
def __init__(self, *args, **kwargs):
""" Defines default parameters
"""
super(VOMS2CSAgent, self).__init__(*args, **kwargs)
self.__voDict = {}
self.__adminMsgs = {}
self.csapi = None
self.voChanged = False
self.dryRun = False
self.autoAddUsers = False
self.autoModifyUsers = False
def initialize(self):
""" Initialize the default parameters
"""
voNames = self.am_getOption('VO', [])
if not voNames[0].lower() == "none":
if voNames[0].lower() == "any":
voNames = []
result = getVOMSVOs(voNames)
if not result['OK']:
return result
self.__voDict = result['Value']
self.log.notice("VOs: %s" % self.__voDict.keys())
self.csapi = CSAPI()
self.dryRun = self.am_getOption('DryRun', self.dryRun)
self.autoAddUsers = self.am_getOption('AutoAddUsers',
self.autoAddUsers)
self.autoModifyUsers = self.am_getOption('AutoModifyUsers',
self.autoModifyUsers)
return S_OK()
def execute(self):
self.__adminMsgs = {}
self.csapi.downloadCSData()
for vo in self.__voDict:
self.voChanged = False
voAdminUser = getVOOption(vo, "VOAdmin")
voAdminMail = None
if voAdminUser:
voAdminMail = getUserOption(voAdminUser, "Email")
voAdminGroup = getVOOption(vo, "VOAdminGroup",
getVOOption(vo, "DefaultGroup"))
self.log.info(
'Performing VOMS sync for VO %s with credentials %s@%s' %
(vo, voAdminUser, voAdminGroup))
result = self.__syncCSWithVOMS(vo,
proxyUserName=voAdminUser,
proxyUserGroup=voAdminGroup) #pylint: disable=unexpected-keyword-arg
if not result['OK']:
self.log.error('Failed to perform VOMS to CS synchronization:',
'VO %s: %s' % (vo, result["Message"]))
continue
if self.voChanged:
mailMsg = ""
if self.__adminMsgs['Errors']:
mailMsg += "\nErrors list:\n %s" % "\n ".join(
self.__adminMsgs['Errors'])
if self.__adminMsgs['Info']:
mailMsg += "\nRun result:\n %s" % "\n ".join(
self.__adminMsgs['Info'])
NotificationClient().sendMail(
self.am_getOption('MailTo', voAdminMail),
"VOMS2CSAgent run log", mailMsg,
self.am_getOption('mailFrom', "DIRAC system"))
if self.csapi.csModified:
# We have accumulated all the changes, commit them now
self.log.info("There are changes to the CS ready to be committed")
if self.dryRun:
self.log.info("Dry Run: CS won't be updated")
self.csapi.showDiff()
else:
result = self.csapi.commitChanges()
if not result['OK']:
self.log.error("Could not commit configuration changes",
result['Message'])
return result
self.log.notice("Configuration committed")
else:
self.log.info("No changes to the CS recorded at this cycle")
return S_OK()
@executeWithUserProxy
def __syncCSWithVOMS(self, vo):
self.__adminMsgs = {'Errors': [], 'Info': []}
# Get DIRAC group vs VOMS Role Mappings
result = getVOMSRoleGroupMapping(vo)
if not result['OK']:
return result
vomsDIRACMapping = result['Value']['VOMSDIRAC']
diracVOMSMapping = result['Value']['DIRACVOMS']
noVOMSGroups = result['Value']['NoVOMS']
vomsSrv = VOMSService(vo)
# Get VOMS VO name
result = vomsSrv.admGetVOName()
if not result['OK']:
self.log.error('Could not retrieve VOMS VO name', "for %s" % vo)
return result
vomsVOName = result['Value'].lstrip('/')
self.log.verbose("VOMS VO Name for %s is %s" % (vo, vomsVOName))
# Get VOMS users
result = vomsSrv.getUsers()
if not result['OK']:
self.log.error('Could not retrieve user information from VOMS',
result['Message'])
return result
vomsUserDict = result['Value']
message = "There are %s registered users in VOMS VO %s" % (
len(vomsUserDict), vomsVOName)
self.__adminMsgs['Info'].append(message)
self.log.info(message)
# Get DIRAC users
diracUsers = getUsersInVO(vo)
if not diracUsers:
return S_ERROR("No VO users found for %s" % vo)
result = self.csapi.describeUsers(diracUsers)
if not result['OK']:
self.log.error('Could not retrieve CS User description')
return result
diracUserDict = result['Value']
self.__adminMsgs['Info'].append(
"There are %s registered users in DIRAC for VO %s" %
(len(diracUserDict), vo))
self.log.info("There are %s registered users in DIRAC VO %s" %
(len(diracUserDict), vo))
# Find new and obsoleted user DNs
existingDNs = []
obsoletedDNs = []
newDNs = []
for user in diracUserDict:
dn = diracUserDict[user]['DN']
existingDNs.append(dn)
if dn not in vomsUserDict:
obsoletedDNs.append(dn)
for dn in vomsUserDict:
if dn not in existingDNs:
newDNs.append(dn)
allDiracUsers = getAllUsers()
nonVOusers = list(set(allDiracUsers) - set(diracUsers))
result = self.csapi.describeUsers(nonVOusers)
if not result['OK']:
self.log.error('Could not retrieve CS User description')
return result
nonVOUserDict = result['Value']
# Process users
defaultVOGroup = getVOOption(vo, "DefaultGroup", "%s_user" % vo)
for dn in vomsUserDict:
if dn in newDNs:
# Find if the DN is already registered in the DIRAC CS
diracName = ''
for user in nonVOUserDict:
if dn == nonVOUserDict[user]['DN']:
diracName = user
# We have a real new user
if not diracName:
nickName = ''
result = vomsSrv.attGetUserNickname(
dn, vomsUserDict[dn]['CA'])
if result['OK']:
nickName = result['Value']
if nickName:
newDiracName = nickName
else:
newDiracName = getUserName(dn,
vomsUserDict[dn]['mail'])
ind = 1
trialName = newDiracName
while newDiracName in allDiracUsers:
# We have a user with the same name but with a different DN
newDiracName = "%s_%d" % (trialName, ind)
ind += 1
# We now have everything to add the new user
userDict = {
"DN": dn,
"CA": vomsUserDict[dn]['CA'],
"Email": vomsUserDict[dn]['mail']
}
groupsWithRole = []
for role in vomsUserDict[dn]['Roles']:
fullRole = "/%s/%s" % (vomsVOName, role)
group = vomsDIRACMapping.get(fullRole)
if group:
groupsWithRole.extend(group)
userDict['Groups'] = list(
set(groupsWithRole + [defaultVOGroup]))
self.__adminMsgs['Info'].append(
"Adding new user %s: %s" %
(newDiracName, str(userDict)))
self.voChanged = True
if self.autoAddUsers:
self.log.info("Adding new user %s: %s" %
(newDiracName, str(userDict)))
result = self.csapi.modifyUser(
newDiracName, userDict, createIfNonExistant=True)
if not result['OK']:
self.log.warn('Failed adding new user %s' %
newDiracName)
continue
# We have an already existing user
userDict = {
"DN": dn,
"CA": vomsUserDict[dn]['CA'],
"Email": vomsUserDict[dn]['mail']
}
nonVOGroups = nonVOUserDict.get(diracName,
{}).get('Groups', [])
existingGroups = diracUserDict.get(diracName,
{}).get('Groups', [])
groupsWithRole = []
for role in vomsUserDict[dn]['Roles']:
fullRole = "/%s/%s" % (vomsVOName, role)
group = vomsDIRACMapping.get(fullRole)
if group:
groupsWithRole.extend(group)
keepGroups = nonVOGroups + groupsWithRole + [defaultVOGroup]
for group in existingGroups:
role = diracVOMSMapping[group]
# Among already existing groups for the user keep those without a special VOMS Role
# because this membership is done by hand in the CS
if not "Role" in role:
keepGroups.append(group)
# Keep existing groups with no VOMS attribute if any
if group in noVOMSGroups:
keepGroups.append(group)
userDict['Groups'] = keepGroups
if self.autoModifyUsers:
result = self.csapi.modifyUser(diracName, userDict)
if result['OK'] and result['Value']:
self.voChanged = True
# Check if there are potentially obsoleted users
oldUsers = set()
for user in diracUserDict:
dn = diracUserDict[user]['DN']
if not dn in vomsUserDict and not user in nonVOUserDict:
for group in diracUserDict[user]['Groups']:
if not group in noVOMSGroups:
oldUsers.add(user)
if oldUsers:
self.voChanged = True
self.__adminMsgs['Info'].append(
'The following users to be checked for deletion: %s' %
str(oldUsers))
self.log.info(
'The following users to be checked for deletion: %s' %
str(oldUsers))
return S_OK()
class VOMS2CSSynchronizer(object):
def __init__(
self,
vo,
autoModifyUsers=True,
autoAddUsers=True,
autoDeleteUsers=False,
autoLiftSuspendedStatus=False,
syncPluginName=None,
):
"""VOMS2CSSynchronizer class constructor
:param str vo: VO to be synced
:param boolean autoModifyUsers: flag to automatically modify user data in CS
:param autoAddUsers: flag to automatically add new users to CS
:param autoDeleteUsers: flag to automatically delete users from CS if no more in VOMS
:param autoLiftSuspendedStatus: flag to automatically remove Suspended status in CS
:param syncPluginName: name of the plugin to validate or extend users' info
:return: None
"""
self.log = gLogger.getSubLogger(self.__class__.__name__)
self.csapi = CSAPI()
self.vo = vo
self.vomsVOName = getVOOption(vo, "VOMSName", "")
if not self.vomsVOName:
raise Exception("VOMS name not defined for VO %s" % vo)
self.adminMsgs = {"Errors": [], "Info": []}
self.vomsUserDict = {}
self.autoModifyUsers = autoModifyUsers
self.autoAddUsers = autoAddUsers
self.autoDeleteUsers = autoDeleteUsers
self.autoLiftSuspendedStatus = autoLiftSuspendedStatus
self.voChanged = False
self.syncPlugin = None
if syncPluginName:
objLoader = ObjectLoader()
_class = objLoader.loadObject(
"ConfigurationSystem.Client.SyncPlugins.%sSyncPlugin" % syncPluginName, "%sSyncPlugin" % syncPluginName
)
if not _class["OK"]:
raise Exception(_class["Message"])
self.syncPlugin = _class["Value"]()
def syncCSWithVOMS(self):
"""Performs the synchronization of the DIRAC registry with the VOMS data. The resulting
CSAPI object containing modifications is returned as part of the output dictionary.
Those changes can be applied by the caller depending on the mode (dry or a real run)
:return: S_OK with a dictionary containing the results of the synchronization operation
"""
resultDict = defaultdict(list)
# Get DIRAC group vs VOMS Role Mappings
result = getVOMSRoleGroupMapping(self.vo)
if not result["OK"]:
return result
vomsDIRACMapping = result["Value"]["VOMSDIRAC"]
diracVOMSMapping = result["Value"]["DIRACVOMS"]
noVOMSGroups = result["Value"]["NoVOMS"]
noSyncVOMSGroups = result["Value"]["NoSyncVOMS"]
vomsSrv = VOMSService(self.vo)
# Get VOMS users
result = vomsSrv.getUsers()
if not result["OK"]:
self.log.error("Could not retrieve user information from VOMS", result["Message"])
return result
self.vomsUserDict = result["Value"]
message = "There are %s user entries in VOMS for VO %s" % (len(self.vomsUserDict), self.vomsVOName)
self.adminMsgs["Info"].append(message)
self.log.info("VOMS user entries", message)
self.log.debug(self.vomsUserDict)
# Get DIRAC users
result = self.getVOUserData(self.vo)
if not result["OK"]:
return result
diracUserDict = result["Value"]
self.adminMsgs["Info"].append(
"There are %s registered users in DIRAC for VO %s" % (len(diracUserDict), self.vo)
)
self.log.info(
"Users already registered", ": there are %s registered users in DIRAC VO %s" % (len(diracUserDict), self.vo)
)
# Find new and obsoleted user DNs
existingDNs = []
obsoletedDNs = []
newDNs = []
for user in diracUserDict:
dn = diracUserDict[user]["DN"]
# We can have users with more than one DN registered
dnList = fromChar(dn)
existingDNs.extend(dnList)
for dn in dnList:
if dn not in self.vomsUserDict:
obsoletedDNs.append(dn)
for dn in self.vomsUserDict:
if dn not in existingDNs:
newDNs.append(dn)
allDiracUsers = getAllUsers()
nonVOUserDict = {}
nonVOUsers = list(set(allDiracUsers) - set(diracUserDict))
if nonVOUsers:
result = self.csapi.describeUsers(nonVOUsers)
if not result["OK"]:
self.log.error("Could not retrieve CS User description")
return result
nonVOUserDict = result["Value"]
# Process users
defaultVOGroup = getVOOption(self.vo, "DefaultGroup", "%s_user" % self.vo)
# If a user is (previously put by hand) in a "QuarantineGroup",
# then the default group will be ignored.
# So, this option is only considered for the case of existing users.
quarantineVOGroup = getVOOption(self.vo, "QuarantineGroup")
newAddedUserDict = {}
for dn in self.vomsUserDict:
newDNForExistingUser = ""
diracName = ""
if dn in existingDNs:
for user in diracUserDict:
if dn in fromChar(diracUserDict[user]["DN"]):
diracName = user
break
if dn in newDNs:
# Find if the DN is already registered in the DIRAC CS
for user in nonVOUserDict:
if dn in fromChar(nonVOUserDict[user]["DN"]):
diracName = user
diracUserDict[diracName] = nonVOUserDict[user]
break
# Check the nickName in the same VO to see if the user is already registered
# with another DN
nickName = self.vomsUserDict[dn].get("nickname")
if nickName in diracUserDict or nickName in newAddedUserDict:
diracName = nickName
# This is a flag for adding the new DN to an already existing user
newDNForExistingUser = dn
# We have a real new user
if not diracName:
if nickName:
newDiracName = nickName
else:
newDiracName = self.getUserName(dn)
# Do not consider users with Suspended status in VOMS
if self.vomsUserDict[dn]["suspended"] or self.vomsUserDict[dn]["certSuspended"]:
resultDict["SuspendedUsers"].append(newDiracName)
continue
# If the chosen user name exists already, append a distinguishing suffix
ind = 1
trialName = newDiracName
while newDiracName in allDiracUsers:
# We have a user with the same name but with a different DN
newDiracName = "%s_%d" % (trialName, ind)
ind += 1
# We now have everything to add the new user
userDict = {"DN": dn, "CA": self.vomsUserDict[dn]["CA"], "Email": self.vomsUserDict[dn]["mail"]}
groupsWithRole = []
for role in self.vomsUserDict[dn]["Roles"]:
groupList = vomsDIRACMapping.get(role, [])
for group in groupList:
if group not in noSyncVOMSGroups:
groupsWithRole.append(group)
userDict["Groups"] = list(set(groupsWithRole + [defaultVOGroup]))
# Run the sync plugins for extra info and/or validations
if self.syncPlugin:
try:
self.syncPlugin.verifyAndUpdateUserInfo(newDiracName, userDict)
except ValueError as e:
self.log.error("Error validating new user", "nickname %s\n error %s" % (newDiracName, e))
self.adminMsgs["Errors"].append(
"Error validating new user %s: %s\n %s" % (newDiracName, userDict, e)
)
continue
message = "\n Added new user %s:\n" % newDiracName
for key in userDict:
message += " %s: %s\n" % (key, str(userDict[key]))
self.adminMsgs["Info"].append(message)
self.voChanged = True
if self.autoAddUsers:
self.log.info("Adding new user %s: %s" % (newDiracName, str(userDict)))
result = self.csapi.modifyUser(newDiracName, userDict, createIfNonExistant=True)
if not result["OK"]:
self.log.warn("Failed adding new user %s" % newDiracName)
resultDict["NewUsers"].append(newDiracName)
newAddedUserDict[newDiracName] = userDict
continue
# We have an already existing user
modified = False
suspendedInVOMS = self.vomsUserDict[dn]["suspended"] or self.vomsUserDict[dn]["certSuspended"]
suspendedVOList = getUserOption(diracName, "Suspended", [])
knownEmail = getUserOption(diracName, "Email", None)
userDict = {
"DN": diracUserDict[diracName]["DN"],
"CA": diracUserDict[diracName]["CA"],
"Email": self.vomsUserDict[dn].get("mail", self.vomsUserDict[dn].get("emailAddress")) or knownEmail,
}
# Set Suspended status for the user for this particular VO
if suspendedInVOMS and self.vo not in suspendedVOList:
suspendedVOList.append(self.vo)
userDict["Suspended"] = ",".join(suspendedVOList)
modified = True
# Remove the lifted Suspended status
if not suspendedInVOMS and self.vo in suspendedVOList and self.autoLiftSuspendedStatus:
newList = []
for vo in suspendedVOList:
if vo != self.vo:
newList.append(vo)
if not newList:
newList = ["None"]
userDict["Suspended"] = ",".join(newList)
modified = True
if newDNForExistingUser:
userDict["DN"] = ",".join([dn, diracUserDict.get(diracName, newAddedUserDict.get(diracName))["DN"]])
userDict["CA"] = ",".join(
[self.vomsUserDict[dn]["CA"], diracUserDict.get(diracName, newAddedUserDict.get(diracName))["CA"]]
)
modified = True
existingGroups = diracUserDict.get(diracName, {}).get("Groups", [])
nonVOGroups = list(set(existingGroups) - set(diracVOMSMapping))
groupsWithRole = []
for role in self.vomsUserDict[dn]["Roles"]:
groupList = vomsDIRACMapping.get(role, [])
for group in groupList:
if group not in noSyncVOMSGroups:
groupsWithRole.append(group)
keepGroups = nonVOGroups + groupsWithRole
if not quarantineVOGroup or quarantineVOGroup not in existingGroups:
keepGroups += [defaultVOGroup]
if quarantineVOGroup and quarantineVOGroup in existingGroups:
keepGroups = [quarantineVOGroup]
for group in existingGroups:
if group in nonVOGroups:
continue
role = diracVOMSMapping.get(group, "")
# Among already existing groups for the user keep those without a special VOMS Role
# because this membership is done by hand in the CS
if "Role" not in role:
keepGroups.append(group)
# Keep existing groups with no VOMS attribute if any
if group in noVOMSGroups:
keepGroups.append(group)
# Keep groups for which syncronization with VOMS is forbidden
if group in noSyncVOMSGroups:
keepGroups.append(group)
userDict["Groups"] = list(set(keepGroups))
# Merge together groups for the same user but different DNs
if diracName in newAddedUserDict:
otherGroups = newAddedUserDict[diracName].get("Groups", [])
userDict["Groups"] = list(set(keepGroups + otherGroups))
modified = True
if not existingGroups and diracName in allDiracUsers:
groups = getGroupsForUser(diracName)
if groups["OK"]:
self.log.info("Found groups for user %s %s" % (diracName, groups["Value"]))
userDict["Groups"] = list(set(groups["Value"] + keepGroups))
addedGroups = list(set(userDict["Groups"]) - set(groups["Value"]))
modified = True
message = "\n Modified user %s:\n" % diracName
message += " Added to group(s) %s\n" % ",".join(addedGroups)
self.adminMsgs["Info"].append(message)
# Check if something changed before asking CSAPI to modify
if diracName in diracUserDict:
message = "\n Modified user %s:\n" % diracName
modMsg = ""
for key in userDict:
if key == "Groups":
addedGroups = set(userDict[key]) - set(diracUserDict.get(diracName, {}).get(key, []))
removedGroups = set(diracUserDict.get(diracName, {}).get(key, [])) - set(userDict[key])
if addedGroups:
modMsg += " Added to group(s) %s\n" % ",".join(addedGroups)
if removedGroups:
modMsg += " Removed from group(s) %s\n" % ",".join(removedGroups)
elif key == "Suspended":
if userDict["Suspended"] == "None":
modMsg += " Suspended status removed\n"
else:
modMsg += " User Suspended in VOs: %s\n" % userDict["Suspended"]
else:
oldValue = str(diracUserDict.get(diracName, {}).get(key, ""))
if str(userDict[key]) != oldValue:
modMsg += " %s: %s -> %s\n" % (key, oldValue, str(userDict[key]))
if modMsg:
self.adminMsgs["Info"].append(message + modMsg)
modified = True
if self.autoModifyUsers and modified:
result = self.csapi.modifyUser(diracName, userDict)
if result["OK"] and result["Value"]:
self.log.info("Modified user %s: %s" % (diracName, str(userDict)))
self.voChanged = True
resultDict["ModifiedUsers"].append(diracName)
# Check if there are potentially obsoleted users
oldUsers = set()
for user in diracUserDict:
dnSet = set(fromChar(diracUserDict[user]["DN"]))
if not dnSet.intersection(set(self.vomsUserDict)) and user not in nonVOUserDict:
existingGroups = diracUserDict.get(user, {}).get("Groups", [])
nonVOGroups = list(set(existingGroups) - set(diracVOMSMapping))
removedGroups = list(set(existingGroups) - set(nonVOGroups))
if removedGroups:
self.log.info("Checking user for deletion", "%s: %s" % (user, existingGroups))
self.log.info("User has groups in other VOs", "%s: %s" % (user, nonVOGroups))
userDict = diracUserDict[user]
userDict["Groups"] = nonVOGroups
if self.autoModifyUsers:
result = self.csapi.modifyUser(user, userDict)
if result["OK"] and result["Value"]:
self.log.info("Modified user %s: %s" % (user, str(userDict)))
self.voChanged = True
message = "\n Modified user %s:\n" % user
modMsg = " Removed from group(s) %s\n" % ",".join(removedGroups)
self.adminMsgs["Info"].append(message + modMsg)
resultDict["ModifiedUsers"].append(user)
continue
if not any(group in noVOMSGroups for group in existingGroups):
oldUsers.add(user)
# Check for obsoleted DNs
for user in diracUserDict:
dnSet = set(fromChar(diracUserDict[user]["DN"]))
for dn in dnSet:
if dn in obsoletedDNs and user not in oldUsers:
existingGroups = diracUserDict.get(user, {}).get("Groups", [])
nonVOGroups = list(set(existingGroups) - set(diracVOMSMapping))
if nonVOGroups:
self.log.verbose("User has groups in other VOs", "%s: %s" % (user, nonVOGroups))
continue
self.log.verbose("Modified user %s: dropped DN %s" % (user, dn))
if self.autoModifyUsers:
userDict = diracUserDict[user]
modDNSet = dnSet - set([dn])
if modDNSet:
userDict["DN"] = ",".join(modDNSet)
result = self.csapi.modifyUser(user, userDict)
if result["OK"] and result["Value"]:
self.log.info("Modified user %s: dropped DN %s" % (user, dn))
self.adminMsgs["Info"].append("Modified user %s: dropped DN %s" % (user, dn))
self.voChanged = True
resultDict["ModifiedUsers"].append(diracName)
else:
oldUsers.add(user)
if oldUsers:
self.voChanged = True
if self.autoDeleteUsers:
self.log.info("The following users will be deleted: %s" % str(oldUsers))
result = self.csapi.deleteUsers(oldUsers)
if result["OK"]:
self.adminMsgs["Info"].append("The following users are deleted from CS:\n %s\n" % str(oldUsers))
resultDict["DeletedUsers"] = oldUsers
else:
self.adminMsgs["Errors"].append("Error in deleting users from CS:\n %s" % str(oldUsers))
self.log.error("Error while user deletion from CS", result)
else:
self.adminMsgs["Info"].append(
"The following users to be checked for deletion:\n\t%s" % "\n\t".join(sorted(oldUsers))
)
self.log.info("The following users to be checked for deletion:", "\n\t".join(sorted(oldUsers)))
resultDict["CSAPI"] = self.csapi
resultDict["AdminMessages"] = self.adminMsgs
resultDict["VOChanged"] = self.voChanged
return S_OK(resultDict)
def getVOUserData(self, refreshFlag=False):
"""Get a report for users of a given VO
:param bool refreshFlag: flag to indicate that the configuration must be refreshed
before looking up user data
:return: S_OK/S_ERROR, Value = user description dictionary
"""
if refreshFlag:
gConfig.forceRefresh()
# Get DIRAC users
diracUsers = getUsersInVO(self.vo)
if not diracUsers:
return S_ERROR("No VO users found for %s" % self.vo)
if refreshFlag:
result = self.csapi.downloadCSData()
if not result["OK"]:
return result
result = self.csapi.describeUsers(diracUsers)
if not result["OK"]:
self.log.error("Could not retrieve CS User description")
return result
def getVOUserReport(self):
"""Get a report string with the current status of the DIRAC Registry for the
Virtual Organization
:return: S_OK with the report string as Value
"""
result = self.getVOUserData(refreshFlag=True)
if not result["OK"]:
return result
userDict = result["Value"]
# Get DIRAC group vs VOMS Role Mappings
result = getVOMSRoleGroupMapping(self.vo)
if not result["OK"]:
return result
diracVOMSMapping = result["Value"]["DIRACVOMS"]
records = []
groupDict = defaultdict(int)
multiDNUsers = {}
suspendedUsers = []
for user in userDict:
for group in userDict[user]["Groups"]:
groupDict[group] += 1
dnList = fromChar(userDict[user]["DN"])
if len(dnList) > 1:
multiDNUsers[user] = dnList
if userDict[user].get("Status", "Active") == "Suspended":
suspendedUsers.append(user)
for group in diracVOMSMapping:
records.append((group, str(groupDict[group]), diracVOMSMapping.get(group, "")))
fields = ["Group", "Number of users", "VOMS Role"]
output = printTable(fields, records, sortField="Group", printOut=False, numbering=False)
if multiDNUsers:
output += "\nUsers with multiple DNs:\n"
for user in multiDNUsers:
output += " %s:\n" % user
for dn in multiDNUsers[user]:
output += " %s\n" % dn
if suspendedUsers:
output += "\n%d suspended users:\n" % len(suspendedUsers)
output += " %s" % ",".join(suspendedUsers)
return S_OK(output)
def getUserName(self, dn):
"""Utility to construct user name
:param str dn: user DN
:return str: user name
"""
name = self.vomsUserDict[dn].get("name")
surname = self.vomsUserDict[dn].get("surname")
if name and surname:
surnameName = _getUserNameFromSurname(name, surname)
return surnameName
dnName = _getUserNameFromDN(dn, self.vo)
# If robot, take the dn based name
if dnName.startswith("robot"):
return dnName
# Is mailName reasonable ?
mail = self.vomsUserDict[dn]["mail"]
if mail:
mailName = _getUserNameFromMail(mail)
if len(mailName) > 5 and mailName.isalpha():
return mailName
# dnName too long
if len(dnName) >= 12:
dnName = dnName[:11]
# May be the mail name is still more reasonable
if mail and len(dnName) < len(mailName) and mailName.isalpha():
return mailName
return dnName
def checkUnusedSEs():
global vo, dry
result = getGridSRMs(vo, unUsed=True)
if not result["OK"]:
gLogger.error("Failed to look up SRMs in BDII", result["Message"])
siteSRMDict = result["Value"]
# Evaluate VOs
result = getVOs()
if result["OK"]:
csVOs = set(result["Value"])
else:
csVOs = {vo}
changeSetFull = set()
for site in siteSRMDict:
for gridSE in siteSRMDict[site]:
changeSet = set()
seDict = siteSRMDict[site][gridSE]["SE"]
srmDict = siteSRMDict[site][gridSE]["SRM"]
# Check the SRM version
version = srmDict.get("GlueServiceVersion", "")
if not (version and version.startswith("2")):
gLogger.debug("Skipping SRM service with version %s" % version)
continue
result = getDIRACSiteName(site)
if not result["OK"]:
gLogger.notice("Unused se %s is detected at unused site %s" % (gridSE, site))
gLogger.notice("Consider adding site %s to the DIRAC CS" % site)
continue
diracSites = result["Value"]
yn = raw_input(
"\nDo you want to add new SRM SE %s at site(s) %s ? default yes [yes|no]: " % (gridSE, str(diracSites))
)
if not yn or yn.lower().startswith("y"):
if len(diracSites) > 1:
prompt = "Which DIRAC site the new SE should be attached to ?"
for i, s in enumerate(diracSites):
prompt += "\n[%d] %s" % (i, s)
prompt += "\nEnter your choice number: "
inp = raw_input(prompt)
try:
ind = int(inp)
except:
gLogger.notice("Can not interpret your choice: %s, try again later" % inp)
continue
diracSite = diracSites[ind]
else:
diracSite = diracSites[0]
domain, siteName, country = diracSite.split(".")
recName = "%s-disk" % siteName
inp = raw_input("Give a DIRAC name to the grid SE %s, default %s : " % (gridSE, recName))
diracSEName = inp
if not inp:
diracSEName = recName
gLogger.notice("Adding new SE %s at site %s" % (diracSEName, diracSite))
seSection = cfgPath("/Resources/StorageElements", diracSEName)
changeSet.add((seSection, "BackendType", seDict.get("GlueSEImplementationName", "Unknown")))
changeSet.add((seSection, "Description", seDict.get("GlueSEName", "Unknown")))
bdiiVOs = set(
[re.sub("^VO:", "", rule) for rule in srmDict.get("GlueServiceAccessControlBaseRule", [])]
)
seVOs = csVOs.intersection(bdiiVOs)
changeSet.add((seSection, "VO", ",".join(seVOs)))
accessSection = cfgPath(seSection, "AccessProtocol.1")
changeSet.add((accessSection, "Protocol", "srm"))
changeSet.add((accessSection, "PluginName", "SRM2"))
endPoint = srmDict.get("GlueServiceEndpoint", "")
host = urlparse(endPoint).hostname
port = urlparse(endPoint).port
changeSet.add((accessSection, "Host", host))
changeSet.add((accessSection, "Port", port))
changeSet.add((accessSection, "Access", "remote"))
voPathSection = cfgPath(accessSection, "VOPath")
if "VOPath" in seDict:
path = seDict["VOPath"]
voFromPath = os.path.basename(path)
if voFromPath != diracVO:
gLogger.notice("\n!!! Warning: non-conventional VO path: %s\n" % path)
changeSet.add((voPathSection, diracVO, path))
path = os.path.dirname(path)
else:
# Try to guess the Path
domain = ".".join(host.split(".")[-2:])
path = "/dpm/%s/home" % domain
changeSet.add((accessSection, "Path", path))
changeSet.add((accessSection, "SpaceToken", ""))
changeSet.add((accessSection, "WSUrl", "/srm/managerv2?SFN="))
gLogger.notice("SE %s will be added with the following parameters" % diracSEName)
changeList = list(changeSet)
changeList.sort()
for entry in changeList:
gLogger.notice(entry)
yn = raw_input("Do you want to add new SE %s ? default yes [yes|no]: " % diracSEName)
if not yn or yn.lower().startswith("y"):
changeSetFull = changeSetFull.union(changeSet)
if dry:
if changeSetFull:
gLogger.notice("Skipping commit of the new SE data in a dry run")
else:
gLogger.notice("No new SE to be added")
return S_OK()
if changeSetFull:
csAPI = CSAPI()
csAPI.initialize()
result = csAPI.downloadCSData()
if not result["OK"]:
gLogger.error("Failed to initialize CSAPI object", result["Message"])
DIRACExit(-1)
changeList = list(changeSetFull)
changeList.sort()
for section, option, value in changeList:
csAPI.setOption(cfgPath(section, option), value)
yn = raw_input("New SE data is accumulated\n Do you want to commit changes to CS ? default yes [yes|no]: ")
if not yn or yn.lower().startswith("y"):
result = csAPI.commit()
if not result["OK"]:
gLogger.error("Error while commit to CS", result["Message"])
else:
gLogger.notice("Successfully committed %d changes to CS" % len(changeSetFull))
else:
gLogger.notice("No new SE to be added")
return S_OK()
class VOMS2CSSynchronizer(object):
def __init__(self,
vo,
autoModifyUsers=True,
autoAddUsers=True,
autoDeleteUsers=False):
self.log = gLogger.getSubLogger("VOMS2CSSynchronizer")
self.csapi = CSAPI()
self.vo = vo
self.vomsVOName = getVOOption(vo, "VOMSName", "")
if not self.vomsVOName:
raise Exception("VOMS name not defined for VO %s" % vo)
self.adminMsgs = {'Errors': [], 'Info': []}
self.vomsUserDict = {}
self.autoModifyUsers = autoModifyUsers
self.autoAddUsers = autoAddUsers
self.autoDeleteUsers = autoDeleteUsers
def syncCSWithVOMS(self):
""" Performs the synchronization of the DIRAC registry with the VOMS data. The resulting
CSAPI object containing modifications is returned as part of the output dictionary.
Those changes can be applied by the caller depending on the mode (dry or a real run)
:return: S_OK with a dictionary containing the results of the synchronization operation
"""
resultDict = defaultdict(list)
# Get DIRAC group vs VOMS Role Mappings
result = getVOMSRoleGroupMapping(self.vo)
if not result['OK']:
return result
vomsDIRACMapping = result['Value']['VOMSDIRAC']
diracVOMSMapping = result['Value']['DIRACVOMS']
noVOMSGroups = result['Value']['NoVOMS']
noSyncVOMSGroups = result['Value']['NoSyncVOMS']
vomsSrv = VOMSService(self.vo)
# Get VOMS users
result = vomsSrv.getUsers()
if not result['OK']:
self.log.error('Could not retrieve user information from VOMS',
result['Message'])
return result
self.vomsUserDict = result['Value']
message = "There are %s user entries in VOMS for VO %s" % (len(
self.vomsUserDict), self.vomsVOName)
self.adminMsgs['Info'].append(message)
self.log.info(message)
# Get DIRAC users
result = self.getVOUserData(self.vo)
if not result['OK']:
return result
diracUserDict = result['Value']
self.adminMsgs['Info'].append(
"There are %s registered users in DIRAC for VO %s" %
(len(diracUserDict), self.vo))
self.log.info("There are %s registered users in DIRAC VO %s" %
(len(diracUserDict), self.vo))
# Find new and obsoleted user DNs
existingDNs = []
obsoletedDNs = []
newDNs = []
for user in diracUserDict:
dn = diracUserDict[user]['DN']
# We can have users with more than one DN registered
dnList = fromChar(dn)
existingDNs.extend(dnList)
for dn in dnList:
if dn not in self.vomsUserDict:
obsoletedDNs.append(dn)
for dn in self.vomsUserDict:
if dn not in existingDNs:
newDNs.append(dn)
allDiracUsers = getAllUsers()
nonVOUserDict = {}
nonVOUsers = list(set(allDiracUsers) - set(diracUserDict))
if nonVOUsers:
result = self.csapi.describeUsers(nonVOUsers)
if not result['OK']:
self.log.error('Could not retrieve CS User description')
return result
nonVOUserDict = result['Value']
# Process users
defaultVOGroup = getVOOption(self.vo, "DefaultGroup",
"%s_user" % self.vo)
# If a user is (previously put by hand) in a "QuarantineGroup",
# then the default group will be ignored.
# So, this option is only considered for the case of existing users.
quarantineVOGroup = getVOOption(self.vo, "QuarantineGroup")
newAddedUserDict = {}
for dn in self.vomsUserDict:
newDNForExistingUser = ''
diracName = ''
if dn in existingDNs:
for user in diracUserDict:
if dn == diracUserDict[user]['DN']:
diracName = user
if dn in newDNs:
# Find if the DN is already registered in the DIRAC CS
for user in nonVOUserDict:
if dn == nonVOUserDict[user]['DN']:
diracName = user
# Check the nickName in the same VO to see if the user is already registered
# with another DN
nickName = self.vomsUserDict[dn].get('nickname')
if nickName in diracUserDict or nickName in newAddedUserDict:
diracName = nickName
# This is a flag for adding the new DN to an already existing user
newDNForExistingUser = dn
# We have a real new user
if not diracName:
if nickName:
newDiracName = nickName
else:
newDiracName = self.getUserName(dn)
# Do not consider users with Suspended status in VOMS
if self.vomsUserDict[dn]['suspended'] or self.vomsUserDict[
dn]['certSuspended']:
resultDict["SuspendedUsers"].append(newDiracName)
continue
# If the chosen user name exists already, append a distinguishing suffix
ind = 1
trialName = newDiracName
while newDiracName in allDiracUsers:
# We have a user with the same name but with a different DN
newDiracName = "%s_%d" % (trialName, ind)
ind += 1
# We now have everything to add the new user
userDict = {
"DN": dn,
"CA": self.vomsUserDict[dn]['CA'],
"Email": self.vomsUserDict[dn]['mail']
}
groupsWithRole = []
for role in self.vomsUserDict[dn]['Roles']:
groupList = vomsDIRACMapping.get(role, [])
for group in groupList:
if group not in noSyncVOMSGroups:
groupsWithRole.append(group)
userDict['Groups'] = list(
set(groupsWithRole + [defaultVOGroup]))
message = "\n Added new user %s:\n" % newDiracName
for key in userDict:
message += " %s: %s\n" % (key, str(userDict[key]))
self.adminMsgs['Info'].append(message)
self.voChanged = True
if self.autoAddUsers:
self.log.info("Adding new user %s: %s" %
(newDiracName, str(userDict)))
result = self.csapi.modifyUser(
newDiracName, userDict, createIfNonExistant=True)
if not result['OK']:
self.log.warn('Failed adding new user %s' %
newDiracName)
resultDict['NewUsers'].append(newDiracName)
newAddedUserDict[newDiracName] = userDict
continue
# We have an already existing user
modified = False
userDict = {
"DN": dn,
"CA": self.vomsUserDict[dn]['CA'],
"Email": self.vomsUserDict[dn]['mail']
}
if newDNForExistingUser:
userDict['DN'] = ','.join([
dn,
diracUserDict.get(diracName,
newAddedUserDict.get(diracName))['DN']
])
modified = True
existingGroups = diracUserDict.get(diracName, {}).get('Groups', [])
nonVOGroups = list(set(existingGroups) - set(diracVOMSMapping))
groupsWithRole = []
for role in self.vomsUserDict[dn]['Roles']:
groupList = vomsDIRACMapping.get(role, [])
for group in groupList:
if group not in noSyncVOMSGroups:
groupsWithRole.append(group)
keepGroups = nonVOGroups + groupsWithRole
if not quarantineVOGroup or quarantineVOGroup not in existingGroups:
keepGroups += [defaultVOGroup]
for group in existingGroups:
if group in nonVOGroups:
continue
role = diracVOMSMapping.get(group, '')
# Among already existing groups for the user keep those without a special VOMS Role
# because this membership is done by hand in the CS
if "Role" not in role:
keepGroups.append(group)
# Keep existing groups with no VOMS attribute if any
if group in noVOMSGroups:
keepGroups.append(group)
# Keep groups for which syncronization with VOMS is forbidden
if group in noSyncVOMSGroups:
keepGroups.append(group)
userDict['Groups'] = list(set(keepGroups))
# Merge together groups for the same user but different DNs
if diracName in newAddedUserDict:
otherGroups = newAddedUserDict[diracName].get('Groups', [])
userDict['Groups'] = list(set(keepGroups + otherGroups))
modified = True
# Check if something changed before asking CSAPI to modify
if diracName in diracUserDict:
message = "\n Modified user %s:\n" % diracName
modMsg = ''
for key in userDict:
if key == "Groups":
addedGroups = set(userDict[key]) - set(
diracUserDict.get(diracName, {}).get(key, []))
removedGroups = set(
diracUserDict.get(diracName, {}).get(
key, [])) - set(userDict[key])
if addedGroups:
modMsg += " Added to group(s) %s\n" % ','.join(
addedGroups)
if removedGroups:
modMsg += " Removed from group(s) %s\n" % ','.join(
removedGroups)
else:
oldValue = str(
diracUserDict.get(diracName, {}).get(key, ''))
if str(userDict[key]) != oldValue:
modMsg += " %s: %s -> %s\n" % (
key, oldValue, str(userDict[key]))
if modMsg:
self.adminMsgs['Info'].append(message + modMsg)
modified = True
if self.autoModifyUsers and modified:
result = self.csapi.modifyUser(diracName, userDict)
if result['OK'] and result['Value']:
self.log.info("Modified user %s: %s" %
(diracName, str(userDict)))
self.voChanged = True
resultDict['ModifiedUsers'].append(diracName)
# Check if there are potentially obsoleted users
oldUsers = set()
for user in diracUserDict:
dnSet = set(fromChar(diracUserDict[user]['DN']))
if not dnSet.intersection(set(
self.vomsUserDict)) and user not in nonVOUserDict:
for group in diracUserDict[user]['Groups']:
if group not in noVOMSGroups:
oldUsers.add(user)
# Check for obsoleted DNs
for user in diracUserDict:
dnSet = set(fromChar(diracUserDict[user]['DN']))
for dn in dnSet:
if dn in obsoletedDNs and user not in oldUsers:
self.log.verbose("Modified user %s: dropped DN %s" %
(user, dn))
if self.autoModifyUsers:
userDict = diracUserDict[user]
modDNSet = dnSet - set([dn])
if modDNSet:
userDict['DN'] = ','.join(modDNSet)
result = self.csapi.modifyUser(user, userDict)
if result['OK'] and result['Value']:
self.log.info(
"Modified user %s: dropped DN %s" %
(user, dn))
self.adminMsgs['Info'].append(
"Modified user %s: dropped DN %s" %
(user, dn))
self.voChanged = True
resultDict['ModifiedUsers'].append(diracName)
else:
oldUsers.add(user)
if oldUsers:
self.voChanged = True
if self.autoDeleteUsers:
self.log.info('The following users will be deleted: %s' %
str(oldUsers))
result = self.csapi.deleteUsers(oldUsers)
if result['OK']:
self.adminMsgs['Info'].append(
'The following users are deleted from CS:\n %s\n' %
str(oldUsers))
resultDict['DeletedUsers'] = oldUsers
else:
self.adminMsgs['Errors'].append(
'Error in deleting users from CS:\n %s' %
str(oldUsers))
self.log.error('Error while user deletion from CS', result)
else:
self.adminMsgs['Info'].append(
'The following users to be checked for deletion:\n %s' %
str(oldUsers))
self.log.info(
'The following users to be checked for deletion: %s' %
str(oldUsers))
resultDict['CSAPI'] = self.csapi
resultDict['AdminMessages'] = self.adminMsgs
return S_OK(resultDict)
def getVOUserData(self, refreshFlag=False):
""" Get a report for users of a given VO
:param bool refreshFlag: flag to indicate that the configuration must be refreshed
before looking up user data
:return: S_OK/S_ERROR, Value = user description dictionary
"""
if refreshFlag:
gConfig.forceRefresh()
# Get DIRAC users
diracUsers = getUsersInVO(self.vo)
if not diracUsers:
return S_ERROR("No VO users found for %s" % self.vo)
if refreshFlag:
result = self.csapi.downloadCSData()
if not result['OK']:
return result
result = self.csapi.describeUsers(diracUsers)
if not result['OK']:
self.log.error('Could not retrieve CS User description')
return result
def getVOUserReport(self):
""" Get a report string with the current status of the DIRAC Registry for the
Virtual Organization
:return: S_OK with the report string as Value
"""
result = self.getVOUserData(refreshFlag=True)
if not result['OK']:
return result
userDict = result['Value']
# Get DIRAC group vs VOMS Role Mappings
result = getVOMSRoleGroupMapping(self.vo)
if not result['OK']:
return result
diracVOMSMapping = result['Value']['DIRACVOMS']
records = []
groupDict = defaultdict(int)
multiDNUsers = {}
suspendedUsers = []
for user in userDict:
for group in userDict[user]['Groups']:
groupDict[group] += 1
dnList = fromChar(userDict[user]['DN'])
if len(dnList) > 1:
multiDNUsers[user] = dnList
if userDict[user].get('Status', 'Active') == 'Suspended':
suspendedUsers.append(user)
for group in diracVOMSMapping:
records.append(
(group, str(groupDict[group]), diracVOMSMapping.get(group,
'')))
fields = ['Group', 'Number of users', 'VOMS Role']
output = printTable(fields,
records,
sortField='Group',
printOut=False,
numbering=False)
if multiDNUsers:
output += '\nUsers with multiple DNs:\n'
for user in multiDNUsers:
output += ' %s:\n' % user
for dn in multiDNUsers[user]:
output += ' %s\n' % dn
if suspendedUsers:
output += '\n%d suspended users:\n' % len(suspendedUsers)
output += ' %s' % ','.join(suspendedUsers)
return S_OK(output)
def getUserName(self, dn):
""" Utility to construct user name
:param str dn: user DN
:return str: user name
"""
name = self.vomsUserDict[dn].get('name')
surname = self.vomsUserDict[dn].get('surname')
if name and surname:
surnameName = _getUserNameFromSurname(name, surname)
return surnameName
mail = self.vomsUserDict[dn]['mail']
dnName = _getUserNameFromDN(dn, self.vo)
mailName = _getUserNameFromMail(mail)
# If robot, take the dn based name
if dnName.startswith('robot'):
return dnName
# Is mailName reasonable ?
if len(mailName) > 5 and mailName.isalpha():
return mailName
# dnName too long
if len(dnName) >= 12:
return dnName[:11]
# May be the mail name is still more reasonable
if len(dnName) < len(mailName) and mailName.isalpha():
return mailName
return dnName
