def __setCSElementStatus( self, elementName, elementType, statusType, status ):
"""
Sets on the CS the Elements status
"""
# DIRAC doesn't store the status of ComputingElements nor FTS in the CS, so here we can just do nothing
if elementType in ('ComputingElement', 'FTS'):
return S_OK()
# If we are here it is because elementType is either 'StorageElement' or 'Catalog'
statuses = self.rssConfig.getConfigStatusType( elementType )
if statusType not in statuses:
gLogger.error( "%s is not a valid statusType" % statusType )
return S_ERROR( "%s is not a valid statusType: %s" % ( statusType, statuses ) )
if elementType == 'StorageElement':
cs_path = "/Resources/StorageElements"
elif elementType == 'Catalog':
cs_path = "/Resources/FileCatalogs"
#FIXME: This a probably outdated location (new one is in /Operations/[]/Services/Catalogs)
# but needs to be VO-aware
statusType = 'Status'
csAPI = CSAPI()
csAPI.setOption( "%s/%s/%s/%s" % ( cs_path, elementName, elementType, statusType ), status )
res = csAPI.commitChanges()
if not res[ 'OK' ]:
gLogger.warn( 'CS: %s' % res[ 'Message' ] )
return res
def __setCSElementStatus(self, elementName, elementType, statusType, status):
"""
Sets on the CS the Elements status
"""
# DIRAC doesn't store the status of ComputingElements nor FTS in the CS, so here we can just do nothing
if elementType in ('ComputingElement', 'FTS'):
return S_OK()
# If we are here it is because elementType is either 'StorageElement' or 'Catalog'
statuses = self.rssConfig.getConfigStatusType(elementType)
if statusType not in statuses:
gLogger.error("%s is not a valid statusType" % statusType)
return S_ERROR("%s is not a valid statusType: %s" % (statusType, statuses))
if elementType == 'StorageElement':
cs_path = "/Resources/StorageElements"
elif elementType == 'Catalog':
cs_path = "/Resources/FileCatalogs"
# FIXME: This a probably outdated location (new one is in /Operations/[]/Services/Catalogs)
# but needs to be VO-aware
statusType = 'Status'
csAPI = CSAPI()
csAPI.setOption("%s/%s/%s/%s" % (cs_path, elementName, elementType, statusType), status)
res = csAPI.commitChanges()
if not res['OK']:
gLogger.warn('CS: %s' % res['Message'])
return res
def test_configurationAutoUpdate(value1, value2):
"""
Test if service refresh his configuration. It sent a random value to the CS
and check if Service can return it.
"""
csapi = CSAPI()
# SETTING FIRST VALUE
csapi.modifyValue("/DIRAC/Configuration/TestUpdateValue", value1)
csapi.commitChanges()
# Wait for automatic refresh (+1 to be sure that request is done)
time.sleep(gConfigurationData.getPropagationTime() + 1)
RPCClient("Framework/User").getTestValue()
assert RPCClient("Framework/User").getTestValue()["Value"] == value1
# SETTING SECOND VALUE
csapi.modifyValue("/DIRAC/Configuration/TestUpdateValue", value2)
csapi.commitChanges()
time.sleep(gConfigurationData.getPropagationTime() + 1)
assert RPCClient("Framework/User").getTestValue()["Value"] == value2
def __setCSStorageElementStatus( self, elementName, statusType, status ):
"""
Sets on the CS the StorageElements status
"""
statuses = self.rssConfig.getConfigStatusType( 'StorageElement' )
if not statusType in statuses:
gLogger.error( "%s is not a valid statusType" % statusType )
return S_ERROR( "%s is not a valid statusType: %s" % ( statusType, statuses ) )
csAPI = CSAPI()
cs_path = "/Resources/StorageElements"
csAPI.setOption( "%s/%s/%s" % ( cs_path, elementName, statusType ), status )
res = csAPI.commitChanges()
if not res[ 'OK' ]:
gLogger.warn( 'CS: %s' % res[ 'Message' ] )
return res
def __setCSStorageElementStatus( self, elementName, statusType, status ):
"""
Sets on the CS the StorageElements status
"""
statuses = self.rssConfig.getConfigStatusType( 'StorageElement' )
if not statusType in statuses:
gLogger.error( "%s is not a valid statusType" % statusType )
return S_ERROR( "%s is not a valid statusType: %s" % ( statusType, statuses ) )
csAPI = CSAPI()
cs_path = "/Resources/StorageElements"
csAPI.setOption( "%s/%s/%s" % ( cs_path, elementName, statusType ), status )
res = csAPI.commitChanges()
if not res[ 'OK' ]:
gLogger.warn( 'CS: %s' % res[ 'Message' ] )
return res
def __syncCSWithVOMS( self ):
self.__adminMsgs = { 'Errors' : [], 'Info' : [] }
#Get DIRAC VOMS Mapping
self.log.info( "Getting DIRAC VOMS mapping" )
mappingSection = '/Registry/VOMS/Mapping'
ret = gConfig.getOptionsDict( mappingSection )
if not ret['OK']:
self.log.fatal( 'No VOMS to DIRAC Group Mapping Available' )
return ret
vomsMapping = ret['Value']
self.log.info( "There are %s registered voms mappings in DIRAC" % len( vomsMapping ) )
#Get VOMS VO name
self.log.info( "Getting VOMS VO name" )
result = self.vomsSrv.admGetVOName()
if not ret['OK']:
self.log.fatal( 'Could not retrieve VOMS VO name' )
voNameInVOMS = result[ 'Value' ]
self.log.info( "VOMS VO Name is %s" % voNameInVOMS )
#Get VOMS roles
self.log.info( "Getting the list of registered roles in VOMS" )
result = self.vomsSrv.admListRoles()
if not ret['OK']:
self.log.fatal( 'Could not retrieve registered roles in VOMS' )
rolesInVOMS = result[ 'Value' ]
self.log.info( "There are %s registered roles in VOMS" % len( rolesInVOMS ) )
print rolesInVOMS
rolesInVOMS.append( '' )
#Map VOMS roles
vomsRoles = {}
for role in rolesInVOMS:
if role:
role = "%s/%s" % ( voNameInVOMS, role )
else:
role = voNameInVOMS
groupsForRole = []
for group in vomsMapping:
if vomsMapping[ group ] == role:
groupsForRole.append( group )
if groupsForRole:
vomsRoles[ role ] = { 'Groups' : groupsForRole, 'Users' : [] }
self.log.info( "DIRAC valid VOMS roles are:\n\t", "\n\t ".join( vomsRoles.keys() ) )
#Get DIRAC users
self.log.info( "Getting the list of registered users in DIRAC" )
csapi = CSAPI()
ret = csapi.listUsers()
if not ret['OK']:
self.log.fatal( 'Could not retrieve current list of Users' )
return ret
currentUsers = ret['Value']
ret = csapi.describeUsers( currentUsers )
if not ret['OK']:
self.log.fatal( 'Could not retrieve current User description' )
return ret
currentUsers = ret['Value']
self.__adminMsgs[ 'Info' ].append( "There are %s registered users in DIRAC" % len( currentUsers ) )
self.log.info( "There are %s registered users in DIRAC" % len( currentUsers ) )
#Get VOMS user entries
self.log.info( "Getting the list of registered user entries in VOMS" )
result = self.vomsSrv.admListMembers()
if not ret['OK']:
self.log.fatal( 'Could not retrieve registered user entries in VOMS' )
usersInVOMS = result[ 'Value' ]
self.__adminMsgs[ 'Info' ].append( "There are %s registered user entries in VOMS" % len( usersInVOMS ) )
self.log.info( "There are %s registered user entries in VOMS" % len( usersInVOMS ) )
#Consolidate users by nickname
usersData = {}
newUserNames = []
knownUserNames = []
obsoleteUserNames = []
self.log.info( "Retrieving usernames..." )
usersInVOMS.sort()
for iUPos in range( len( usersInVOMS ) ):
userName = ''
user = usersInVOMS[ iUPos ]
for oldUser in currentUsers:
if user[ 'DN' ].strip() in List.fromChar( currentUsers[oldUser][ 'DN' ] ):
userName = oldUser
if not userName:
result = self.vomsSrv.attGetUserNickname( user[ 'DN' ], user[ 'CA' ] )
if result[ 'OK' ]:
userName = result[ 'Value' ]
else:
self.__adminMsgs[ 'Errors' ].append( "Could not retrieve nickname for DN %s" % user[ 'DN' ] )
self.log.error( "Could not get nickname for DN", user[ 'DN' ] )
userName = user[ 'mail' ][:user[ 'mail' ].find( '@' )]
if not userName:
self.log.error( "Empty nickname for DN", user[ 'DN' ] )
self.__adminMsgs[ 'Errors' ].append( "Empty nickname for DN %s" % user[ 'DN' ] )
continue
self.log.info( " (%02d%%) Found username %s : %s " % ( ( iUPos * 100 / len( usersInVOMS ) ), userName, user[ 'DN' ] ) )
if userName not in usersData:
usersData[ userName ] = { 'DN': [], 'CA': [], 'Email': [], 'Groups' : ['user'] }
for key in ( 'DN', 'CA', 'mail' ):
value = user[ key ]
if value:
if key == "mail":
List.appendUnique( usersData[ userName ][ 'Email' ], value )
else:
usersData[ userName ][ key ].append( value.strip() )
if userName not in currentUsers:
List.appendUnique( newUserNames, userName )
else:
List.appendUnique( knownUserNames, userName )
self.log.info( "Finished retrieving usernames" )
if newUserNames:
self.log.info( "There are %s new users" % len( newUserNames ) )
else:
self.log.info( "There are no new users" )
#Get the list of users for each group
result = csapi.listGroups()
if not result[ 'OK' ]:
self.log.error( "Could not get the list of groups in DIRAC", result[ 'Message' ] )
return result
staticGroups = result[ 'Value' ]
vomsGroups = []
self.log.info( "Mapping users in VOMS to groups" )
for vomsRole in vomsRoles:
self.log.info( " Getting users for role %s" % vomsRole )
groupsForRole = vomsRoles[ vomsRole ][ 'Groups' ]
vomsMap = vomsRole.split( "Role=" )
for g in groupsForRole:
if g in staticGroups:
staticGroups.pop( staticGroups.index( g ) )
else:
vomsGroups.append( g )
if len( vomsMap ) == 1:
# no Role
users = usersInVOMS
else:
vomsGroup = "Role=".join( vomsMap[:-1] )
if vomsGroup[-1] == "/":
vomsGroup = vomsGroup[:-1]
vomsRole = "Role=%s" % vomsMap[-1]
result = self.vomsSrv.admListUsersWithRole( vomsGroup, vomsRole )
if not result[ 'OK' ]:
errorMsg = "Could not get list of users for VOMS %s" % ( vomsMapping[ group ] )
self.__adminMsgs[ 'Errors' ].append( errorMsg )
self.log.error( errorMsg, result[ 'Message' ] )
return result
users = result['Value']
numUsersInGroup = 0
for vomsUser in users:
for userName in usersData:
if vomsUser[ 'DN' ] in usersData[ userName ][ 'DN' ]:
numUsersInGroup += 1
usersData[ userName ][ 'Groups' ].extend( groupsForRole )
infoMsg = "There are %s users in group(s) %s for VOMS Role %s" % ( numUsersInGroup, ",".join( groupsForRole ), vomsRole )
self.__adminMsgs[ 'Info' ].append( infoMsg )
self.log.info( " %s" % infoMsg )
self.log.info( "Checking static groups" )
staticUsers = []
for group in staticGroups:
self.log.info( " Checking static group %s" % group )
numUsersInGroup = 0
result = csapi.listUsers( group )
if not result[ 'OK' ]:
self.log.error( "Could not get the list of users in DIRAC group %s" % group , result[ 'Message' ] )
return result
for userName in result[ 'Value' ]:
if userName in usersData:
numUsersInGroup += 1
usersData[ userName ][ 'Groups' ].append( group )
else:
if group not in vomsGroups and userName not in staticUsers:
staticUsers.append( userName )
infoMsg = "There are %s users in group %s" % ( numUsersInGroup, group )
self.__adminMsgs[ 'Info' ].append( infoMsg )
self.log.info( " %s" % infoMsg )
if staticUsers:
infoMsg = "There are %s static users: %s" % ( len( staticUsers ) , ', '.join( staticUsers ) )
self.__adminMsgs[ 'Info' ].append( infoMsg )
self.log.info( "%s" % infoMsg )
for user in currentUsers:
if user not in usersData and user not in staticUsers:
self.log.info( 'User %s is no longer valid' % user )
obsoleteUserNames.append( user )
#Do the CS Sync
self.log.info( "Updating CS..." )
ret = csapi.downloadCSData()
if not ret['OK']:
self.log.fatal( 'Can not update from CS', ret['Message'] )
return ret
usersWithMoreThanOneDN = {}
for user in usersData:
csUserData = dict( usersData[ user ] )
if len( csUserData[ 'DN' ] ) > 1:
usersWithMoreThanOneDN[ user ] = csUserData[ 'DN' ]
result = csapi.describeUsers( [ user ] )
if result[ 'OK' ]:
if result[ 'Value' ]:
prevUser = result[ 'Value' ][ user ]
prevDNs = List.fromChar( prevUser[ 'DN' ] )
newDNs = csUserData[ 'DN' ]
for DN in newDNs:
if DN not in prevDNs:
self.__adminMsgs[ 'Info' ].append( "User %s has new DN %s" % ( user, DN ) )
for DN in prevDNs:
if DN not in newDNs:
self.__adminMsgs[ 'Info' ].append( "User %s has lost a DN %s" % ( user, DN ) )
else:
newDNs = csUserData[ 'DN' ]
for DN in newDNs:
self.__adminMsgs[ 'Info' ].append( "New user %s has new DN %s" % ( user, DN ) )
for k in ( 'DN', 'CA', 'Email' ):
csUserData[ k ] = ", ".join( csUserData[ k ] )
result = csapi.modifyUser( user, csUserData, createIfNonExistant = True )
if not result[ 'OK' ]:
self.__adminMsgs[ 'Error' ].append( "Cannot modify user %s: %s" % ( user, result[ 'Message' ] ) )
self.log.error( "Cannot modify user", user )
if usersWithMoreThanOneDN:
self.__adminMsgs[ 'Info' ].append( "\nUsers with more than one DN:" )
for uwmtod in sorted( usersWithMoreThanOneDN ):
self.__adminMsgs[ 'Info' ].append( " %s" % uwmtod )
self.__adminMsgs[ 'Info' ].append( " + DN list:" )
for DN in usersWithMoreThanOneDN[uwmtod]:
self.__adminMsgs[ 'Info' ].append( " - %s" % DN )
if obsoleteUserNames:
self.__adminMsgs[ 'Info' ].append( "\nObsolete users:" )
address = self.am_getOption( 'MailTo', '*****@*****.**' )
fromAddress = self.am_getOption( 'mailFrom', '*****@*****.**' )
subject = 'Obsolete LFC Users found'
body = 'Delete entries into LFC: \n'
for obsoleteUser in obsoleteUserNames:
self.log.info( subject, ", ".join( obsoleteUserNames ) )
body += 'for ' + obsoleteUser + '\n'
self.__adminMsgs[ 'Info' ].append( " %s" % obsoleteUser )
self.log.info( "Deleting %s users" % len( obsoleteUserNames ) )
NotificationClient().sendMail( address, 'UsersAndGroupsAgent: %s' % subject, body, fromAddress )
csapi.deleteUsers( obsoleteUserNames )
if newUserNames:
self.__adminMsgs[ 'Info' ].append( "\nNew users:" )
for newUser in newUserNames:
self.__adminMsgs[ 'Info' ].append( " %s" % newUser )
self.__adminMsgs[ 'Info' ].append( " + DN list:" )
for DN in usersData[newUser][ 'DN' ]:
self.__adminMsgs[ 'Info' ].append( " - %s" % DN )
self.__adminMsgs[ 'Info' ].append( " + EMail: %s" % usersData[newUser][ 'Email' ] )
result = csapi.commitChanges()
if not result[ 'OK' ]:
self.log.error( "Could not commit configuration changes", result[ 'Message' ] )
return result
self.log.info( "Configuration committed" )
#LFC Check
if self.am_getOption( "LFCCheckEnabled", True ):
result = self.checkLFCRegisteredUsers( usersData )
if not result[ 'OK' ]:
return result
return S_OK()
class VOMS2CSAgent(AgentModule):
def __init__(self, *args, **kwargs):
""" Defines default parameters
"""
super(VOMS2CSAgent, self).__init__(*args, **kwargs)
self.__voDict = {}
self.__adminMsgs = {}
self.csapi = None
self.voChanged = False
self.dryRun = False
self.autoAddUsers = False
self.autoModifyUsers = False
self.autoDeleteUsers = False
self.detailedReport = True
self.makeFCEntry = False
def initialize(self):
""" Initialize the default parameters
"""
voNames = self.am_getOption('VO', [])
if not voNames[0].lower() == "none":
if voNames[0].lower() == "any":
voNames = []
result = getVOMSVOs(voNames)
if not result['OK']:
return result
self.__voDict = result['Value']
self.log.notice("VOs: %s" % self.__voDict.keys())
self.csapi = CSAPI()
self.dryRun = self.am_getOption('DryRun', self.dryRun)
self.autoAddUsers = self.am_getOption('AutoAddUsers',
self.autoAddUsers)
self.autoModifyUsers = self.am_getOption('AutoModifyUsers',
self.autoModifyUsers)
self.autoDeleteUsers = self.am_getOption('AutoDeleteUsers',
self.autoDeleteUsers)
self.detailedReport = self.am_getOption('DetailedReport',
self.detailedReport)
self.makeFCEntry = self.am_getOption('MakeHomeDirectory',
self.makeFCEntry)
return S_OK()
def execute(self):
self.__adminMsgs = {}
self.csapi.downloadCSData()
for vo in self.__voDict:
self.voChanged = False
voAdminUser = getVOOption(vo, "VOAdmin")
voAdminMail = None
if voAdminUser:
voAdminMail = getUserOption(voAdminUser, "Email")
voAdminGroup = getVOOption(vo, "VOAdminGroup",
getVOOption(vo, "DefaultGroup"))
self.log.info(
'Performing VOMS sync for VO %s with credentials %s@%s' %
(vo, voAdminUser, voAdminGroup))
result = self.__syncCSWithVOMS(vo,
proxyUserName=voAdminUser,
proxyUserGroup=voAdminGroup) #pylint: disable=unexpected-keyword-arg
if not result['OK']:
self.log.error('Failed to perform VOMS to CS synchronization:',
'VO %s: %s' % (vo, result["Message"]))
continue
resultDict = result['Value']
newUsers = resultDict.get("NewUsers", [])
modUsers = resultDict.get("ModifiedUsers", [])
delUsers = resultDict.get("DeletedUsers", [])
self.log.info( "Run results: new users %d, modified users %d, deleted users %d" % \
( len( newUsers ), len( modUsers ), len( delUsers ) ) )
if self.csapi.csModified:
# We have accumulated all the changes, commit them now
self.log.info(
"There are changes to the CS for vo %s ready to be committed"
% vo)
if self.dryRun:
self.log.info("Dry Run: CS won't be updated")
self.csapi.showDiff()
else:
result = self.csapi.commitChanges()
if not result['OK']:
self.log.error(
"Could not commit configuration changes",
result['Message'])
return result
self.log.notice("Configuration committed for VO %s" % vo)
else:
self.log.info(
"No changes to the CS for VO %s recorded at this cycle" %
vo)
# Add user home directory in the file catalog
if self.makeFCEntry and newUsers:
self.log.info("Creating home directories for users %s" %
str(newUsers))
result = self.__addHomeDirectory(vo,
newUsers,
proxyUserName=voAdminUser,
proxyUserGroup=voAdminGroup) #pylint: disable=unexpected-keyword-arg
if not result['OK']:
self.log.error('Failed to create user home directories:',
'VO %s: %s' % (vo, result["Message"]))
else:
for user in result['Value']['Failed']:
self.log.error( "Failed to create home directory", "user: %s, operation: %s" % \
( user, result['Value']['Failed'][user] ) )
self.__adminMsgs[ 'Errors' ].append( "Failed to create home directory for user %s: operation %s" % \
( user, result['Value']['Failed'][user] ) )
for user in result['Value']['Successful']:
self.__adminMsgs['Info'].append(
"Created home directory for user %s" % user)
if self.voChanged or self.detailedReport:
mailMsg = ""
if self.__adminMsgs['Errors']:
mailMsg += "\nErrors list:\n %s" % "\n ".join(
self.__adminMsgs['Errors'])
if self.__adminMsgs['Info']:
mailMsg += "\nRun result:\n %s" % "\n ".join(
self.__adminMsgs['Info'])
if self.detailedReport:
result = self.getVOUserReport(vo)
if result['OK']:
mailMsg += '\n\n'
mailMsg += result['Value']
else:
mailMsg += 'Failed to produce a detailed user report'
mailMsg += result['Message']
NotificationClient().sendMail(
self.am_getOption('MailTo', voAdminMail),
"VOMS2CSAgent run log", mailMsg,
self.am_getOption(
'MailFrom',
self.am_getOption('mailFrom', "DIRAC system")))
return S_OK()
def getVOUserData(self, vo, refreshFlag=False):
""" Get a report for users of a given VO
:param str vo: VO name
:return: S_OK/S_ERROR, Value = user description dictionary
"""
if refreshFlag:
gConfig.forceRefresh()
# Get DIRAC users
diracUsers = getUsersInVO(vo)
if not diracUsers:
return S_ERROR("No VO users found for %s" % vo)
if refreshFlag:
result = self.csapi.downloadCSData()
if not result['OK']:
return result
result = self.csapi.describeUsers(diracUsers)
if not result['OK']:
self.log.error('Could not retrieve CS User description')
return result
def getVOUserReport(self, vo):
"""
:param str vo: VO name
:return: report string
"""
result = self.getVOUserData(vo, refreshFlag=True)
if not result['OK']:
return result
userDict = result['Value']
# Get DIRAC group vs VOMS Role Mappings
result = getVOMSRoleGroupMapping(vo)
if not result['OK']:
return result
diracVOMSMapping = result['Value']['DIRACVOMS']
records = []
groupDict = defaultdict(int)
multiDNUsers = {}
suspendedUsers = []
for user in userDict:
for group in userDict[user]['Groups']:
groupDict[group] += 1
dnList = fromChar(userDict[user]['DN'])
if len(dnList) > 1:
multiDNUsers[user] = dnList
if userDict[user].get('Status', 'Active') == 'Suspended':
suspendedUsers.append(user)
for group in diracVOMSMapping:
records.append(
(group, str(groupDict[group]), diracVOMSMapping.get(group,
'')))
fields = ['Group', 'Number of users', 'VOMS Role']
output = printTable(fields,
records,
sortField='Group',
printOut=False,
numbering=False)
if multiDNUsers:
output += '\nUsers with multiple DNs:\n'
for user in multiDNUsers:
output += ' %s:\n' % user
for dn in multiDNUsers[user]:
output += ' %s\n' % dn
if suspendedUsers:
output += '\n%d suspended users:\n' % len(suspendedUsers)
output += ' %s' % ','.join(suspendedUsers)
return S_OK(output)
@executeWithUserProxy
def __syncCSWithVOMS(self, vo):
self.__adminMsgs = {'Errors': [], 'Info': []}
resultDict = defaultdict(list)
# Get DIRAC group vs VOMS Role Mappings
result = getVOMSRoleGroupMapping(vo)
if not result['OK']:
return result
vomsDIRACMapping = result['Value']['VOMSDIRAC']
diracVOMSMapping = result['Value']['DIRACVOMS']
noVOMSGroups = result['Value']['NoVOMS']
noSyncVOMSGroups = result['Value']['NoSyncVOMS']
vomsSrv = VOMSService(vo)
# Get VOMS VO name
result = vomsSrv.admGetVOName()
if not result['OK']:
self.log.error('Could not retrieve VOMS VO name', "for %s" % vo)
return result
vomsVOName = result['Value'].lstrip('/')
self.log.verbose("VOMS VO Name for %s is %s" % (vo, vomsVOName))
# Get VOMS users
result = vomsSrv.getUsers()
if not result['OK']:
self.log.error('Could not retrieve user information from VOMS',
result['Message'])
return result
vomsUserDict = result['Value']
message = "There are %s user entries in VOMS for VO %s" % (
len(vomsUserDict), vomsVOName)
self.__adminMsgs['Info'].append(message)
self.log.info(message)
# Get DIRAC users
result = self.getVOUserData(vo)
if not result['OK']:
return result
diracUserDict = result['Value']
self.__adminMsgs['Info'].append(
"There are %s registered users in DIRAC for VO %s" %
(len(diracUserDict), vo))
self.log.info("There are %s registered users in DIRAC VO %s" %
(len(diracUserDict), vo))
# Find new and obsoleted user DNs
existingDNs = []
obsoletedDNs = []
newDNs = []
for user in diracUserDict:
dn = diracUserDict[user]['DN']
# We can have users with more than one DN registered
dnList = fromChar(dn)
existingDNs.extend(dnList)
for dn in dnList:
if dn not in vomsUserDict:
obsoletedDNs.append(dn)
for dn in vomsUserDict:
if dn not in existingDNs:
newDNs.append(dn)
allDiracUsers = getAllUsers()
nonVOUserDict = {}
nonVOUsers = list(set(allDiracUsers) - set(diracUserDict.keys()))
if nonVOUsers:
result = self.csapi.describeUsers(nonVOUsers)
if not result['OK']:
self.log.error('Could not retrieve CS User description')
return result
nonVOUserDict = result['Value']
# Process users
defaultVOGroup = getVOOption(vo, "DefaultGroup", "%s_user" % vo)
newAddedUserDict = {}
for dn in vomsUserDict:
nickName = ''
newDNForExistingUser = ''
diracName = ''
if dn in existingDNs:
for user in diracUserDict:
if dn == diracUserDict[user]['DN']:
diracName = user
if dn in newDNs:
# Find if the DN is already registered in the DIRAC CS
for user in nonVOUserDict:
if dn == nonVOUserDict[user]['DN']:
diracName = user
# Check the nickName in the same VO to see if the user is already registered
# with another DN
result = vomsSrv.attGetUserNickname(dn, vomsUserDict[dn]['CA'])
if result['OK']:
nickName = result['Value']
if nickName in diracUserDict or nickName in newAddedUserDict:
diracName = nickName
# This is a flag for adding the new DN to an already existing user
newDNForExistingUser = dn
# We have a real new user
if not diracName:
if nickName:
newDiracName = nickName
else:
newDiracName = getUserName(dn,
vomsUserDict[dn]['mail'],
vo)
# If the chosen user name exists already, append a distinguishing suffix
ind = 1
trialName = newDiracName
while newDiracName in allDiracUsers:
# We have a user with the same name but with a different DN
newDiracName = "%s_%d" % (trialName, ind)
ind += 1
# We now have everything to add the new user
userDict = {
"DN": dn,
"CA": vomsUserDict[dn]['CA'],
"Email": vomsUserDict[dn]['mail']
}
groupsWithRole = []
for role in vomsUserDict[dn]['Roles']:
fullRole = "/%s/%s" % (vomsVOName, role)
groupList = vomsDIRACMapping.get(fullRole, [])
for group in groupList:
if group not in noSyncVOMSGroups:
groupsWithRole.append(group)
userDict['Groups'] = list(
set(groupsWithRole + [defaultVOGroup]))
message = "\n Added new user %s:\n" % newDiracName
for key in userDict:
message += " %s: %s\n" % (key, str(userDict[key]))
self.__adminMsgs['Info'].append(message)
self.voChanged = True
if self.autoAddUsers:
self.log.info("Adding new user %s: %s" %
(newDiracName, str(userDict)))
result = self.csapi.modifyUser(
newDiracName, userDict, createIfNonExistant=True)
if not result['OK']:
self.log.warn('Failed adding new user %s' %
newDiracName)
resultDict['NewUsers'].append(newDiracName)
newAddedUserDict[newDiracName] = userDict
continue
# We have an already existing user
modified = False
userDict = {
"DN": dn,
"CA": vomsUserDict[dn]['CA'],
"Email": vomsUserDict[dn]['mail']
}
if newDNForExistingUser:
userDict['DN'] = ','.join([dn, diracUserDict[diracName]['DN']])
modified = True
existingGroups = diracUserDict.get(diracName, {}).get('Groups', [])
nonVOGroups = list(
set(existingGroups) - set(diracVOMSMapping.keys()))
groupsWithRole = []
for role in vomsUserDict[dn]['Roles']:
fullRole = "/%s/%s" % (vomsVOName, role)
groupList = vomsDIRACMapping.get(fullRole, [])
for group in groupList:
if group not in noSyncVOMSGroups:
groupsWithRole.append(group)
keepGroups = nonVOGroups + groupsWithRole + [defaultVOGroup]
for group in existingGroups:
if group in nonVOGroups:
continue
role = diracVOMSMapping.get(group, '')
# Among already existing groups for the user keep those without a special VOMS Role
# because this membership is done by hand in the CS
if not "Role" in role:
keepGroups.append(group)
# Keep existing groups with no VOMS attribute if any
if group in noVOMSGroups:
keepGroups.append(group)
# Keep groups for which syncronization with VOMS is forbidden
if group in noSyncVOMSGroups:
keepGroups.append(group)
userDict['Groups'] = list(set(keepGroups))
# Merge together groups for the same user but different DNs
if diracName in newAddedUserDict:
otherGroups = newAddedUserDict[diracName].get('Groups', [])
userDict['Groups'] = list(set(keepGroups + otherGroups))
modified = True
# Check if something changed before asking CSAPI to modify
if diracName in diracUserDict:
message = "\n Modified user %s:\n" % diracName
modMsg = ''
for key in userDict:
if key == "Groups":
addedGroups = set(userDict[key]) - set(
diracUserDict.get(diracName, {}).get(key, []))
removedGroups = set(
diracUserDict.get(diracName, {}).get(
key, [])) - set(userDict[key])
if addedGroups:
modMsg += " Added to group(s) %s\n" % ','.join(
addedGroups)
if removedGroups:
modMsg += " Removed from group(s) %s\n" % ','.join(
removedGroups)
else:
oldValue = str(
diracUserDict.get(diracName, {}).get(key, ''))
if str(userDict[key]) != oldValue:
modMsg += " %s: %s -> %s\n" % (
key, oldValue, str(userDict[key]))
if modMsg:
self.__adminMsgs['Info'].append(message + modMsg)
modified = True
if self.autoModifyUsers and modified:
result = self.csapi.modifyUser(diracName, userDict)
if result['OK'] and result['Value']:
self.log.info("Modified user %s: %s" %
(diracName, str(userDict)))
self.voChanged = True
resultDict['ModifiedUsers'].append(diracName)
# Check if there are potentially obsoleted users
oldUsers = set()
for user in diracUserDict:
dnSet = set(fromChar(diracUserDict[user]['DN']))
if not dnSet.intersection(set(
vomsUserDict.keys())) and user not in nonVOUserDict:
for group in diracUserDict[user]['Groups']:
if group not in noVOMSGroups:
oldUsers.add(user)
if oldUsers:
self.voChanged = True
if self.autoDeleteUsers:
self.log.info('The following users will be deleted: %s' %
str(oldUsers))
result = self.csapi.deleteUsers(oldUsers)
if result['OK']:
self.__adminMsgs['Info'].append(
'The following users are deleted from CS:\n %s\n' %
str(oldUsers))
resultDict['DeletedUsers'] = oldUsers
else:
self.__adminMsgs['Errors'].append(
'Error in deleting users from CS:\n %s' %
str(oldUsers))
self.log.error('Error while user deletion from CS', result)
else:
self.__adminMsgs['Info'].append(
'The following users to be checked for deletion:\n %s' %
str(oldUsers))
self.log.info(
'The following users to be checked for deletion: %s' %
str(oldUsers))
return S_OK(resultDict)
@executeWithUserProxy
def __addHomeDirectory(self, vo, newUsers):
fc = FileCatalog(vo=vo)
defaultVOGroup = getVOOption(vo, "DefaultGroup", "%s_user" % vo)
failed = {}
successful = {}
for user in newUsers:
result = fc.addUser(user)
if not result['OK']:
failed[user] = "addUser"
continue
dirName = '/%s/user/%s/%s' % (vo, user[0], user)
result = fc.createDirectory(dirName)
if not result['OK']:
failed[user] = "createDirectory"
continue
result = fc.changePathOwner({dirName: user}, recursive=True)
if not result['OK']:
failed[user] = "changePathOwner"
continue
result = fc.changePathGroup({dirName: defaultVOGroup},
recursive=True)
if not result['OK']:
failed[user] = "changePathGroup"
continue
successful[user] = True
return S_OK({"Successful": successful, "Failed": failed})
class DiracAdmin(API):
""" Administrative functionalities
"""
#############################################################################
def __init__(self):
"""Internal initialization of the DIRAC Admin API.
"""
super(DiracAdmin, self).__init__()
self.csAPI = CSAPI()
self.dbg = False
if gConfig.getValue(self.section + '/LogLevel', 'DEBUG') == 'DEBUG':
self.dbg = True
self.scratchDir = gConfig.getValue(self.section + '/ScratchDir',
'/tmp')
self.currentDir = os.getcwd()
self.rssFlag = ResourceStatus().rssFlag
self.sitestatus = SiteStatus()
#############################################################################
def uploadProxy(self, group):
"""Upload a proxy to the DIRAC WMS. This method
Example usage:
>>> print diracAdmin.uploadProxy('lhcb_pilot')
{'OK': True, 'Value': 0L}
:param group: DIRAC Group
:type job: string
:return: S_OK,S_ERROR
:param permanent: Indefinitely update proxy
:type permanent: boolean
"""
return gProxyManager.uploadProxy(diracGroup=group)
#############################################################################
def setProxyPersistency(self, userDN, userGroup, persistent=True):
"""Set the persistence of a proxy in the Proxy Manager
Example usage:
>>> print diracAdmin.setProxyPersistency( 'some DN', 'dirac group', True )
{'OK': True }
:param userDN: User DN
:type userDN: string
:param userGroup: DIRAC Group
:type userGroup: string
:param persistent: Persistent flag
:type persistent: boolean
:return: S_OK,S_ERROR
"""
return gProxyManager.setPersistency(userDN, userGroup, persistent)
#############################################################################
def checkProxyUploaded(self, userDN, userGroup, requiredTime):
"""Set the persistence of a proxy in the Proxy Manager
Example usage:
>>> print diracAdmin.setProxyPersistency( 'some DN', 'dirac group', True )
{'OK': True, 'Value' : True/False }
:param userDN: User DN
:type userDN: string
:param userGroup: DIRAC Group
:type userGroup: string
:param requiredTime: Required life time of the uploaded proxy
:type requiredTime: boolean
:return: S_OK,S_ERROR
"""
return gProxyManager.userHasProxy(userDN, userGroup, requiredTime)
#############################################################################
def getSiteMask(self, printOutput=False, status='Active'):
"""Retrieve current site mask from WMS Administrator service.
Example usage:
>>> print diracAdmin.getSiteMask()
{'OK': True, 'Value': 0L}
:return: S_OK,S_ERROR
"""
result = self.sitestatus.getSites(siteState=status)
if result['OK']:
sites = result['Value']
if printOutput:
sites.sort()
for site in sites:
print site
return result
#############################################################################
def getBannedSites(self, gridType=[], printOutput=False):
"""Retrieve current list of banned and probing sites.
Example usage:
>>> print diracAdmin.getBannedSites()
{'OK': True, 'Value': []}
:return: S_OK,S_ERROR
"""
bannedSites = self.sitestatus.getSites(siteState='Banned')
if not bannedSites['OK']:
return bannedSites
probingSites = self.sitestatus.getSites(siteState='Probing')
if not probingSites['OK']:
return probingSites
mergedList = bannedSites['Value'] + probingSites['Value']
mergedList.sort()
if printOutput:
print '\n'.join(mergedList)
return S_OK(mergedList)
#############################################################################
def getSiteSection(self, site, printOutput=False):
"""Simple utility to get the list of CEs for DIRAC site name.
Example usage:
>>> print diracAdmin.getSiteSection('LCG.CERN.ch')
{'OK': True, 'Value':}
:return: S_OK,S_ERROR
"""
gridType = site.split('.')[0]
if not gConfig.getSections('/Resources/Sites/%s' % (gridType))['OK']:
return S_ERROR('/Resources/Sites/%s is not a valid site section' %
(gridType))
result = gConfig.getOptionsDict('/Resources/Sites/%s/%s' %
(gridType, site))
if printOutput and result['OK']:
print self.pPrint.pformat(result['Value'])
return result
#############################################################################
def allowSite(self, site, comment, printOutput=False):
"""Adds the site to the site mask.
Example usage:
>>> print diracAdmin.allowSite()
{'OK': True, 'Value': }
:return: S_OK,S_ERROR
"""
result = self.__checkSiteIsValid(site)
if not result['OK']:
return result
result = self.getSiteMask(status='Active')
if not result['OK']:
return result
siteMask = result['Value']
if site in siteMask:
if printOutput:
print 'Site %s is already Active' % site
return S_OK('Site %s is already Active' % site)
if self.rssFlag:
result = self.sitestatus.setSiteStatus(site, 'Active', comment)
else:
wmsAdmin = RPCClient('WorkloadManagement/WMSAdministrator')
result = wmsAdmin.allowSite(site, comment)
if not result['OK']:
return result
if printOutput:
print 'Site %s status is set to Active' % site
return result
#############################################################################
def getSiteMaskLogging(self, site=None, printOutput=False):
"""Retrieves site mask logging information.
Example usage:
>>> print diracAdmin.getSiteMaskLogging('LCG.AUVER.fr')
{'OK': True, 'Value': }
:return: S_OK,S_ERROR
"""
result = self.__checkSiteIsValid(site)
if not result['OK']:
return result
wmsAdmin = RPCClient('WorkloadManagement/WMSAdministrator')
result = wmsAdmin.getSiteMaskLogging(site)
if not result['OK']:
return result
if site:
if not result['Value'].has_key(site):
return S_ERROR('Site mask information not available for %s' %
(site))
if printOutput:
if site:
print '\nSite Mask Logging Info for %s\n' % site
else:
print '\nAll Site Mask Logging Info\n'
siteDict = result['Value']
for site, tupleList in siteDict.iteritems():
if not site:
print '\n===> %s\n' % site
for tup in tupleList:
print str( tup[0] ).ljust( 8 ) + str( tup[1] ).ljust( 20 ) + \
'( ' + str( tup[2] ).ljust( len( str( tup[2] ) ) ) + ' ) "' + str( tup[3] ) + '"'
print ' '
return result
#############################################################################
def banSite(self, site, comment, printOutput=False):
"""Removes the site from the site mask.
Example usage:
>>> print diracAdmin.banSite()
{'OK': True, 'Value': }
:return: S_OK,S_ERROR
"""
result = self.__checkSiteIsValid(site)
if not result['OK']:
return result
mask = self.getSiteMask(status='Banned')
if not mask['OK']:
return mask
siteMask = mask['Value']
if site in siteMask:
if printOutput:
print 'Site %s is already Banned' % site
return S_OK('Site %s is already Banned' % site)
if self.rssFlag:
result = self.sitestatus.setSiteStatus(site, 'Banned', comment)
else:
wmsAdmin = RPCClient('WorkloadManagement/WMSAdministrator')
result = wmsAdmin.banSite(site, comment)
if not result['OK']:
return result
if printOutput:
print 'Site %s status is set to Banned' % site
return result
#############################################################################
def __checkSiteIsValid(self, site):
"""Internal function to check that a site name is valid.
"""
sites = getSiteCEMapping()
if not sites['OK']:
return S_ERROR('Could not get site CE mapping')
siteList = sites['Value'].keys()
if not site in siteList:
return S_ERROR(
'Specified site %s is not in list of defined sites' % site)
return S_OK('%s is valid' % site)
#############################################################################
def clearMask(self):
"""Removes all sites from the site mask. Should be used with care.
Example usage:
>>> print diracAdmin.clearMask()
{'OK': True, 'Value':''}
:return: S_OK,S_ERROR
"""
wmsAdmin = RPCClient('WorkloadManagement/WMSAdministrator')
result = wmsAdmin.clearMask()
return result
#############################################################################
def getServicePorts(self, setup='', printOutput=False):
"""Checks the service ports for the specified setup. If not given this is
taken from the current installation (/DIRAC/Setup)
Example usage:
>>> print diracAdmin.getServicePorts()
{'OK': True, 'Value':''}
:return: S_OK,S_ERROR
"""
if not setup:
setup = gConfig.getValue('/DIRAC/Setup', '')
setupList = gConfig.getSections('/DIRAC/Setups', [])
if not setupList['OK']:
return S_ERROR('Could not get /DIRAC/Setups sections')
setupList = setupList['Value']
if not setup in setupList:
return S_ERROR('Setup %s is not in allowed list: %s' %
(setup, ', '.join(setupList)))
serviceSetups = gConfig.getOptionsDict('/DIRAC/Setups/%s' % setup)
if not serviceSetups['OK']:
return S_ERROR('Could not get /DIRAC/Setups/%s options' % setup)
serviceSetups = serviceSetups['Value'] # dict
systemList = gConfig.getSections('/Systems')
if not systemList['OK']:
return S_ERROR('Could not get Systems sections')
systemList = systemList['Value']
result = {}
for system in systemList:
if serviceSetups.has_key(system):
path = '/Systems/%s/%s/Services' % (system,
serviceSetups[system])
servicesList = gConfig.getSections(path)
if not servicesList['OK']:
self.log.warn('Could not get sections in %s' % path)
else:
servicesList = servicesList['Value']
if not servicesList:
servicesList = []
self.log.verbose('System: %s ServicesList: %s' %
(system, ', '.join(servicesList)))
for service in servicesList:
spath = '%s/%s/Port' % (path, service)
servicePort = gConfig.getValue(spath, 0)
if servicePort:
self.log.verbose('Found port for %s/%s = %s' %
(system, service, servicePort))
result['%s/%s' % (system, service)] = servicePort
else:
self.log.warn('No port found for %s' % spath)
else:
self.log.warn('%s is not defined in /DIRAC/Setups/%s' %
(system, setup))
if printOutput:
print self.pPrint.pformat(result)
return S_OK(result)
#############################################################################
def getProxy(self, userDN, userGroup, validity=43200, limited=False):
"""Retrieves a proxy with default 12hr validity and stores
this in a file in the local directory by default.
Example usage:
>>> print diracAdmin.getProxy()
{'OK': True, 'Value': }
:return: S_OK,S_ERROR
"""
return gProxyManager.downloadProxy(userDN,
userGroup,
limited=limited,
requiredTimeLeft=validity)
#############################################################################
def getVOMSProxy(self,
userDN,
userGroup,
vomsAttr=False,
validity=43200,
limited=False):
"""Retrieves a proxy with default 12hr validity and VOMS extensions and stores
this in a file in the local directory by default.
Example usage:
>>> print diracAdmin.getVOMSProxy()
{'OK': True, 'Value': }
:return: S_OK,S_ERROR
"""
return gProxyManager.downloadVOMSProxy(userDN,
userGroup,
limited=limited,
requiredVOMSAttribute=vomsAttr,
requiredTimeLeft=validity)
#############################################################################
def getPilotProxy(self, userDN, userGroup, validity=43200):
"""Retrieves a pilot proxy with default 12hr validity and stores
this in a file in the local directory by default.
Example usage:
>>> print diracAdmin.getVOMSProxy()
{'OK': True, 'Value': }
:return: S_OK,S_ERROR
"""
return gProxyManager.getPilotProxyFromDIRACGroup(
userDN, userGroup, requiredTimeLeft=validity)
#############################################################################
def resetJob(self, jobID):
"""Reset a job or list of jobs in the WMS. This operation resets the reschedule
counter for a job or list of jobs and allows them to run as new.
Example::
>>> print dirac.reset(12345)
{'OK': True, 'Value': [12345]}
:param job: JobID
:type job: integer or list of integers
:return: S_OK,S_ERROR
"""
if isinstance(jobID, basestring):
try:
jobID = int(jobID)
except Exception as x:
return self._errorReport(
str(x),
'Expected integer or convertible integer for existing jobID'
)
elif isinstance(jobID, list):
try:
jobID = [int(job) for job in jobID]
except Exception as x:
return self._errorReport(
str(x),
'Expected integer or convertible integer for existing jobIDs'
)
jobManager = RPCClient('WorkloadManagement/JobManager',
useCertificates=False)
result = jobManager.resetJob(jobID)
return result
#############################################################################
def getJobPilotOutput(self, jobID, directory=''):
"""Retrieve the pilot output for an existing job in the WMS.
The output will be retrieved in a local directory unless
otherwise specified.
>>> print dirac.getJobPilotOutput(12345)
{'OK': True, StdOut:'',StdError:''}
:param job: JobID
:type job: integer or string
:return: S_OK,S_ERROR
"""
if not directory:
directory = self.currentDir
if not os.path.exists(directory):
return self._errorReport('Directory %s does not exist' % directory)
wmsAdmin = RPCClient('WorkloadManagement/WMSAdministrator')
result = wmsAdmin.getJobPilotOutput(jobID)
if not result['OK']:
return result
outputPath = '%s/pilot_%s' % (directory, jobID)
if os.path.exists(outputPath):
self.log.info('Remove %s and retry to continue' % outputPath)
return S_ERROR('Remove %s and retry to continue' % outputPath)
if not os.path.exists(outputPath):
self.log.verbose('Creating directory %s' % outputPath)
os.mkdir(outputPath)
outputs = result['Value']
if outputs.has_key('StdOut'):
stdout = '%s/std.out' % (outputPath)
with open(stdout, 'w') as fopen:
fopen.write(outputs['StdOut'])
self.log.verbose('Standard output written to %s' % (stdout))
else:
self.log.warn('No standard output returned')
if outputs.has_key('StdError'):
stderr = '%s/std.err' % (outputPath)
with open(stderr, 'w') as fopen:
fopen.write(outputs['StdError'])
self.log.verbose('Standard error written to %s' % (stderr))
else:
self.log.warn('No standard error returned')
self.log.always('Outputs retrieved in %s' % outputPath)
return result
#############################################################################
def getPilotOutput(self, gridReference, directory=''):
"""Retrieve the pilot output (std.out and std.err) for an existing job in the WMS.
>>> print dirac.getJobPilotOutput(12345)
{'OK': True, 'Value': {}}
:param job: JobID
:type job: integer or string
:return: S_OK,S_ERROR
"""
if not isinstance(gridReference, basestring):
return self._errorReport('Expected string for pilot reference')
if not directory:
directory = self.currentDir
if not os.path.exists(directory):
return self._errorReport('Directory %s does not exist' % directory)
wmsAdmin = RPCClient('WorkloadManagement/WMSAdministrator')
result = wmsAdmin.getPilotOutput(gridReference)
if not result['OK']:
return result
gridReferenceSmall = gridReference.split('/')[-1]
if not gridReferenceSmall:
gridReferenceSmall = 'reference'
outputPath = '%s/pilot_%s' % (directory, gridReferenceSmall)
if os.path.exists(outputPath):
self.log.info('Remove %s and retry to continue' % outputPath)
return S_ERROR('Remove %s and retry to continue' % outputPath)
if not os.path.exists(outputPath):
self.log.verbose('Creating directory %s' % outputPath)
os.mkdir(outputPath)
outputs = result['Value']
if outputs.has_key('StdOut'):
stdout = '%s/std.out' % (outputPath)
with open(stdout, 'w') as fopen:
fopen.write(outputs['StdOut'])
self.log.info('Standard output written to %s' % (stdout))
else:
self.log.warn('No standard output returned')
if outputs.has_key('StdErr'):
stderr = '%s/std.err' % (outputPath)
with open(stderr, 'w') as fopen:
fopen.write(outputs['StdErr'])
self.log.info('Standard error written to %s' % (stderr))
else:
self.log.warn('No standard error returned')
self.log.always('Outputs retrieved in %s' % outputPath)
return result
#############################################################################
def getPilotInfo(self, gridReference):
"""Retrieve info relative to a pilot reference
>>> print dirac.getPilotInfo(12345)
{'OK': True, 'Value': {}}
:param gridReference: Pilot Job Reference
:type gridReference: string
:return: S_OK,S_ERROR
"""
if not isinstance(gridReference, basestring):
return self._errorReport('Expected string for pilot reference')
wmsAdmin = RPCClient('WorkloadManagement/WMSAdministrator')
result = wmsAdmin.getPilotInfo(gridReference)
return result
#############################################################################
def killPilot(self, gridReference):
"""Kill the pilot specified
>>> print dirac.getPilotInfo(12345)
{'OK': True, 'Value': {}}
:param gridReference: Pilot Job Reference
:return: S_OK,S_ERROR
"""
if not isinstance(gridReference, basestring):
return self._errorReport('Expected string for pilot reference')
wmsAdmin = RPCClient('WorkloadManagement/WMSAdministrator')
result = wmsAdmin.killPilot(gridReference)
return result
#############################################################################
def getPilotLoggingInfo(self, gridReference):
"""Retrieve the pilot logging info for an existing job in the WMS.
>>> print dirac.getPilotLoggingInfo(12345)
{'OK': True, 'Value': {"The output of the command"}}
:param gridReference: Gridp pilot job reference Id
:type gridReference: string
:return: S_OK,S_ERROR
"""
if type(gridReference) not in types.StringTypes:
return self._errorReport('Expected string for pilot reference')
wmsAdmin = RPCClient('WorkloadManagement/WMSAdministrator')
return wmsAdmin.getPilotLoggingInfo(gridReference)
#############################################################################
def getJobPilots(self, jobID):
"""Extract the list of submitted pilots and their status for a given
jobID from the WMS. Useful information is printed to the screen.
>>> print dirac.getJobPilots()
{'OK': True, 'Value': {PilotID:{StatusDict}}}
:param job: JobID
:type job: integer or string
:return: S_OK,S_ERROR
"""
if isinstance(jobID, basestring):
try:
jobID = int(jobID)
except Exception as x:
return self._errorReport(
str(x), 'Expected integer or string for existing jobID')
wmsAdmin = RPCClient('WorkloadManagement/WMSAdministrator')
result = wmsAdmin.getPilots(jobID)
if result['OK']:
print self.pPrint.pformat(result['Value'])
return result
#############################################################################
def getPilotSummary(self, startDate='', endDate=''):
"""Retrieve the pilot output for an existing job in the WMS. Summary is
printed at INFO level, full dictionary of results also returned.
>>> print dirac.getPilotSummary()
{'OK': True, 'Value': {CE:{Status:Count}}}
:param job: JobID
:type job: integer or string
:return: S_OK,S_ERROR
"""
wmsAdmin = RPCClient('WorkloadManagement/WMSAdministrator')
result = wmsAdmin.getPilotSummary(startDate, endDate)
if not result['OK']:
return result
ceDict = result['Value']
headers = 'CE'.ljust(28)
i = 0
for ce, summary in ceDict.iteritems():
states = summary.keys()
if len(states) > i:
i = len(states)
for i in xrange(i):
headers += 'Status'.ljust(12) + 'Count'.ljust(12)
print headers
for ce, summary in ceDict.iteritems():
line = ce.ljust(28)
states = summary.keys()
states.sort()
for state in states:
count = str(summary[state])
line += state.ljust(12) + count.ljust(12)
print line
return result
#############################################################################
def selectRequests(self,
jobID=None,
requestID=None,
requestName=None,
requestType=None,
status=None,
operation=None,
ownerDN=None,
ownerGroup=None,
requestStart=0,
limit=100,
printOutput=False):
"""Select requests from the request management system. A few notes on the selection criteria:
- jobID is the WMS JobID for the request (if applicable)
- requestID is assigned during submission of the request
- requestName is the corresponding XML file name
- requestType e.g. 'transfer'
- status e.g. Done
- operation e.g. replicateAndRegister
- requestStart e.g. the first request to consider (start from 0 by default)
- limit e.g. selection limit (default 100)
>>> dirac.selectRequests(jobID='4894')
{'OK': True, 'Value': [[<Requests>]]}
"""
options = {
'RequestID': requestID,
'RequestName': requestName,
'JobID': jobID,
'OwnerDN': ownerDN,
'OwnerGroup': ownerGroup,
'RequestType': requestType,
'Status': status,
'Operation': operation
}
conditions = {}
for key, value in options.iteritems():
if value:
try:
conditions[key] = str(value)
except Exception as x:
return self._errorReport(
str(x), 'Expected string for %s field' % key)
try:
requestStart = int(requestStart)
limit = int(limit)
except Exception as x:
return self._errorReport(str(x),
'Expected integer for %s field' % limit)
self.log.verbose('Will select requests with the following conditions')
self.log.verbose(self.pPrint.pformat(conditions))
requestClient = RPCClient("RequestManagement/centralURL")
result = requestClient.getRequestSummaryWeb(conditions, [],
requestStart, limit)
if not result['OK']:
self.log.warn(result['Message'])
return result
requestIDs = result['Value']
conds = []
for key, value in conditions.iteritems():
if value:
conds.append('%s = %s' % (key, value))
self.log.verbose(
'%s request(s) selected with conditions %s and limit %s' %
(len(requestIDs['Records']), ', '.join(conds), limit))
if printOutput:
requests = []
if len(requestIDs['Records']) > limit:
requestList = requestIDs['Records']
requests = requestList[:limit]
else:
requests = requestIDs['Records']
print '%s request(s) selected with conditions %s and limit %s' % (
len(requestIDs['Records']), ', '.join(conds), limit)
print requestIDs['ParameterNames']
for request in requests:
print request
if not requestIDs:
return S_ERROR('No requests selected for conditions: %s' %
conditions)
else:
return result
#############################################################################
def getRequestSummary(self, printOutput=False):
"""
Get a summary of the requests in the request DB.
"""
requestClient = RPCClient("RequestManagement/centralURL", timeout=120)
result = requestClient.getDBSummary()
if not result['OK']:
self.log.warn(result['Message'])
return result
if printOutput:
print self.pPrint.pformat(result['Value'])
return result
#############################################################################
def getExternalPackageVersions(self):
"""
Simple function that attempts to obtain the external versions for
the local DIRAC installation (frequently needed for debugging purposes).
"""
gLogger.info(
'DIRAC version v%dr%d build %d' %
(DIRAC.majorVersion, DIRAC.minorVersion, DIRAC.patchLevel))
try:
import lcg_util
infoStr = 'Using lcg_util from: \n%s' % lcg_util.__file__
gLogger.info(infoStr)
infoStr = "The version of lcg_utils is %s" % lcg_util.lcg_util_version(
)
gLogger.info(infoStr)
except Exception as x:
errStr = "SRM2Storage.__init__: Failed to import lcg_util: %s" % (
x)
gLogger.exception(errStr)
try:
import gfalthr as gfal
infoStr = "Using gfalthr from: \n%s" % gfal.__file__
gLogger.info(infoStr)
infoStr = "The version of gfalthr is %s" % gfal.gfal_version()
gLogger.info(infoStr)
except Exception as x:
errStr = "SRM2Storage.__init__: Failed to import gfalthr: %s." % (
x)
gLogger.warn(errStr)
try:
import gfal
infoStr = "Using gfal from: %s" % gfal.__file__
gLogger.info(infoStr)
infoStr = "The version of gfal is %s" % gfal.gfal_version()
gLogger.info(infoStr)
except Exception as x:
errStr = "SRM2Storage.__init__: Failed to import gfal: %s" % (
x)
gLogger.exception(errStr)
defaultProtocols = gConfig.getValue(
'/Resources/StorageElements/DefaultProtocols', [])
gLogger.info('Default list of protocols are: %s' %
(', '.join(defaultProtocols)))
return S_OK()
#############################################################################
def getSiteProtocols(self, site, printOutput=False):
"""
Allows to check the defined protocols for each site SE.
"""
result = self.__checkSiteIsValid(site)
if not result['OK']:
return result
siteSection = '/Resources/Sites/%s/%s/SE' % (site.split('.')[0], site)
siteSEs = gConfig.getValue(siteSection, [])
if not siteSEs:
return S_ERROR('No SEs found for site %s in section %s' %
(site, siteSection))
defaultProtocols = gConfig.getValue(
'/Resources/StorageElements/DefaultProtocols', [])
self.log.verbose('Default list of protocols are'
', '.join(defaultProtocols))
seInfo = {}
siteSEs.sort()
for se in siteSEs:
sections = gConfig.getSections('/Resources/StorageElements/%s/' %
(se))
if not sections['OK']:
return sections
for section in sections['Value']:
if gConfig.getValue(
'/Resources/StorageElements/%s/%s/ProtocolName' %
(se, section), '') == 'SRM2':
path = '/Resources/StorageElements/%s/%s/ProtocolsList' % (
se, section)
seProtocols = gConfig.getValue(path, [])
if not seProtocols:
seProtocols = defaultProtocols
seInfo[se] = seProtocols
if printOutput:
print '\nSummary of protocols for StorageElements at site %s' % site
print '\nStorageElement'.ljust(30) + 'ProtocolsList'.ljust(
30) + '\n'
for se, protocols in seInfo.iteritems():
print se.ljust(30) + ', '.join(protocols).ljust(30)
return S_OK(seInfo)
#############################################################################
def setSiteProtocols(self, site, protocolsList, printOutput=False):
"""
Allows to set the defined protocols for each SE for a given site.
"""
result = self.__checkSiteIsValid(site)
if not result['OK']:
return result
siteSection = '/Resources/Sites/%s/%s/SE' % (site.split('.')[0], site)
siteSEs = gConfig.getValue(siteSection, [])
if not siteSEs:
return S_ERROR('No SEs found for site %s in section %s' %
(site, siteSection))
defaultProtocols = gConfig.getValue(
'/Resources/StorageElements/DefaultProtocols', [])
self.log.verbose('Default list of protocols are',
', '.join(defaultProtocols))
for protocol in protocolsList:
if not protocol in defaultProtocols:
return S_ERROR(
'Requested to set protocol %s in list but %s is not '
'in default list of protocols:\n%s' %
(protocol, protocol, ', '.join(defaultProtocols)))
modifiedCS = False
result = promptUser(
'Do you want to add the following default protocols:'
' %s for SE(s):\n%s' %
(', '.join(protocolsList), ', '.join(siteSEs)))
if not result['OK']:
return result
if result['Value'].lower() != 'y':
self.log.always('No protocols will be added')
return S_OK()
for se in siteSEs:
sections = gConfig.getSections('/Resources/StorageElements/%s/' %
(se))
if not sections['OK']:
return sections
for section in sections['Value']:
if gConfig.getValue(
'/Resources/StorageElements/%s/%s/ProtocolName' %
(se, section), '') == 'SRM2':
path = '/Resources/StorageElements/%s/%s/ProtocolsList' % (
se, section)
self.log.verbose('Setting %s to %s' %
(path, ', '.join(protocolsList)))
result = self.csSetOption(path, ', '.join(protocolsList))
if not result['OK']:
return result
modifiedCS = True
if modifiedCS:
result = self.csCommitChanges(False)
if not result['OK']:
return S_ERROR('CS Commit failed with message = %s' %
(result['Message']))
else:
if printOutput:
print 'Successfully committed changes to CS'
else:
if printOutput:
print 'No modifications to CS required'
return S_OK()
#############################################################################
def csSetOption(self, optionPath, optionValue):
"""
Function to modify an existing value in the CS.
"""
return self.csAPI.setOption(optionPath, optionValue)
#############################################################################
def csSetOptionComment(self, optionPath, comment):
"""
Function to modify an existing value in the CS.
"""
return self.csAPI.setOptionComment(optionPath, comment)
#############################################################################
def csModifyValue(self, optionPath, newValue):
"""
Function to modify an existing value in the CS.
"""
return self.csAPI.modifyValue(optionPath, newValue)
#############################################################################
def csRegisterUser(self, username, properties):
"""
Registers a user in the CS.
- username: Username of the user (easy;)
- properties: Dict containing:
- DN
- groups : list/tuple of groups the user belongs to
- <others> : More properties of the user, like mail
"""
return self.csAPI.addUser(username, properties)
#############################################################################
def csDeleteUser(self, user):
"""
Deletes a user from the CS. Can take a list of users
"""
return self.csAPI.deleteUsers(user)
#############################################################################
def csModifyUser(self, username, properties, createIfNonExistant=False):
"""
Modify a user in the CS. Takes the same params as in addUser and
applies the changes
"""
return self.csAPI.modifyUser(username, properties, createIfNonExistant)
#############################################################################
def csListUsers(self, group=False):
"""
Lists the users in the CS. If no group is specified return all users.
"""
return self.csAPI.listUsers(group)
#############################################################################
def csDescribeUsers(self, mask=False):
"""
List users and their properties in the CS.
If a mask is given, only users in the mask will be returned
"""
return self.csAPI.describeUsers(mask)
#############################################################################
def csModifyGroup(self, groupname, properties, createIfNonExistant=False):
"""
Modify a user in the CS. Takes the same params as in addGroup and applies
the changes
"""
return self.csAPI.modifyGroup(groupname, properties,
createIfNonExistant)
#############################################################################
def csListHosts(self):
"""
Lists the hosts in the CS
"""
return self.csAPI.listHosts()
#############################################################################
def csDescribeHosts(self, mask=False):
"""
Gets extended info for the hosts in the CS
"""
return self.csAPI.describeHosts(mask)
#############################################################################
def csModifyHost(self, hostname, properties, createIfNonExistant=False):
"""
Modify a host in the CS. Takes the same params as in addHost and applies
the changes
"""
return self.csAPI.modifyHost(hostname, properties, createIfNonExistant)
#############################################################################
def csListGroups(self):
"""
Lists groups in the CS
"""
return self.csAPI.listGroups()
#############################################################################
def csDescribeGroups(self, mask=False):
"""
List groups and their properties in the CS.
If a mask is given, only groups in the mask will be returned
"""
return self.csAPI.describeGroups(mask)
#############################################################################
def csSyncUsersWithCFG(self, usersCFG):
"""
Synchronize users in cfg with its contents
"""
return self.csAPI.syncUsersWithCFG(usersCFG)
#############################################################################
def csCommitChanges(self, sortUsers=True):
"""
Commit the changes in the CS
"""
return self.csAPI.commitChanges(sortUsers=False)
#############################################################################
def sendMail(self,
address,
subject,
body,
fromAddress=None,
localAttempt=True,
html=False):
"""
Send mail to specified address with body.
"""
notification = NotificationClient()
return notification.sendMail(address, subject, body, fromAddress,
localAttempt, html)
#############################################################################
def sendSMS(self, userName, body, fromAddress=None):
"""
Send mail to specified address with body.
"""
if len(body) > 160:
return S_ERROR('Exceeded maximum SMS length of 160 characters')
notification = NotificationClient()
return notification.sendSMS(userName, body, fromAddress)
#############################################################################
def getBDIISite(self, site, host=None):
"""
Get information about site from BDII at host
"""
return ldapSite(site, host=host)
#############################################################################
def getBDIICluster(self, ce, host=None):
"""
Get information about ce from BDII at host
"""
return ldapCluster(ce, host=host)
#############################################################################
def getBDIICE(self, ce, host=None):
"""
Get information about ce from BDII at host
"""
return ldapCE(ce, host=host)
#############################################################################
def getBDIIService(self, ce, host=None):
"""
Get information about ce from BDII at host
"""
return ldapService(ce, host=host)
#############################################################################
def getBDIICEState(self, ce, useVO=voName, host=None):
"""
Get information about ce state from BDII at host
"""
return ldapCEState(ce, useVO, host=host)
#############################################################################
def getBDIICEVOView(self, ce, useVO=voName, host=None):
"""
Get information about ce voview from BDII at host
"""
return ldapCEVOView(ce, useVO, host=host)
#############################################################################
def getBDIISE(self, site, useVO=voName, host=None):
"""
Get information about SA from BDII at host
"""
return ldapSE(site, useVO, host=host)
def __syncCSWithVOMS(self):
self.__adminMsgs = {'Errors': [], 'Info': []}
#Get DIRAC VOMS Mapping
self.log.info("Getting DIRAC VOMS mapping")
mappingSection = '/Registry/VOMS/Mapping'
ret = gConfig.getOptionsDict(mappingSection)
if not ret['OK']:
self.log.fatal('No VOMS to DIRAC Group Mapping Available')
return ret
vomsMapping = ret['Value']
self.log.info("There are %s registered voms mappings in DIRAC" %
len(vomsMapping))
#Get VOMS VO name
self.log.info("Getting VOMS VO name")
result = self.vomsSrv.admGetVOName()
if not ret['OK']:
self.log.fatal('Could not retrieve VOMS VO name')
voNameInVOMS = result['Value']
self.log.info("VOMS VO Name is %s" % voNameInVOMS)
#Get VOMS roles
self.log.info("Getting the list of registered roles in VOMS")
result = self.vomsSrv.admListRoles()
if not ret['OK']:
self.log.fatal('Could not retrieve registered roles in VOMS')
rolesInVOMS = result['Value']
self.log.info("There are %s registered roles in VOMS" %
len(rolesInVOMS))
print rolesInVOMS
rolesInVOMS.append('')
#Map VOMS roles
vomsRoles = {}
for role in rolesInVOMS:
if role:
role = "%s/%s" % (voNameInVOMS, role)
else:
role = voNameInVOMS
groupsForRole = []
for group in vomsMapping:
if vomsMapping[group] == role:
groupsForRole.append(group)
if groupsForRole:
vomsRoles[role] = {'Groups': groupsForRole, 'Users': []}
self.log.info("DIRAC valid VOMS roles are:\n\t",
"\n\t ".join(vomsRoles.keys()))
#Get DIRAC users
self.log.info("Getting the list of registered users in DIRAC")
csapi = CSAPI()
ret = csapi.listUsers()
if not ret['OK']:
self.log.fatal('Could not retrieve current list of Users')
return ret
currentUsers = ret['Value']
ret = csapi.describeUsers(currentUsers)
if not ret['OK']:
self.log.fatal('Could not retrieve current User description')
return ret
currentUsers = ret['Value']
self.__adminMsgs['Info'].append(
"There are %s registered users in DIRAC" % len(currentUsers))
self.log.info("There are %s registered users in DIRAC" %
len(currentUsers))
#Get VOMS user entries
self.log.info("Getting the list of registered user entries in VOMS")
result = self.vomsSrv.admListMembers()
if not ret['OK']:
self.log.fatal(
'Could not retrieve registered user entries in VOMS')
usersInVOMS = result['Value']
self.__adminMsgs['Info'].append(
"There are %s registered user entries in VOMS" % len(usersInVOMS))
self.log.info("There are %s registered user entries in VOMS" %
len(usersInVOMS))
#Consolidate users by nickname
usersData = {}
newUserNames = []
knownUserNames = []
obsoleteUserNames = []
self.log.info("Retrieving usernames...")
usersInVOMS.sort()
for iUPos in range(len(usersInVOMS)):
userName = ''
user = usersInVOMS[iUPos]
for oldUser in currentUsers:
if user['DN'].strip() in List.fromChar(
currentUsers[oldUser]['DN']):
userName = oldUser
if not userName:
result = self.vomsSrv.attGetUserNickname(
user['DN'], user['CA'])
if result['OK']:
userName = result['Value']
else:
self.__adminMsgs['Errors'].append(
"Could not retrieve nickname for DN %s" % user['DN'])
self.log.error("Could not get nickname for DN", user['DN'])
userName = user['mail'][:user['mail'].find('@')]
if not userName:
self.log.error("Empty nickname for DN", user['DN'])
self.__adminMsgs['Errors'].append("Empty nickname for DN %s" %
user['DN'])
continue
self.log.info(
" (%02d%%) Found username %s : %s " %
((iUPos * 100 / len(usersInVOMS)), userName, user['DN']))
if userName not in usersData:
usersData[userName] = {
'DN': [],
'CA': [],
'Email': [],
'Groups': ['user']
}
for key in ('DN', 'CA', 'mail'):
value = user[key]
if value:
if key == "mail":
List.appendUnique(usersData[userName]['Email'], value)
else:
usersData[userName][key].append(value.strip())
if userName not in currentUsers:
List.appendUnique(newUserNames, userName)
else:
List.appendUnique(knownUserNames, userName)
self.log.info("Finished retrieving usernames")
if newUserNames:
self.log.info("There are %s new users" % len(newUserNames))
else:
self.log.info("There are no new users")
#Get the list of users for each group
result = csapi.listGroups()
if not result['OK']:
self.log.error("Could not get the list of groups in DIRAC",
result['Message'])
return result
staticGroups = result['Value']
vomsGroups = []
self.log.info("Mapping users in VOMS to groups")
for vomsRole in vomsRoles:
self.log.info(" Getting users for role %s" % vomsRole)
groupsForRole = vomsRoles[vomsRole]['Groups']
vomsMap = vomsRole.split("Role=")
for g in groupsForRole:
if g in staticGroups:
staticGroups.pop(staticGroups.index(g))
else:
vomsGroups.append(g)
if len(vomsMap) == 1:
# no Role
users = usersInVOMS
else:
vomsGroup = "Role=".join(vomsMap[:-1])
if vomsGroup[-1] == "/":
vomsGroup = vomsGroup[:-1]
vomsRole = "Role=%s" % vomsMap[-1]
result = self.vomsSrv.admListUsersWithRole(vomsGroup, vomsRole)
if not result['OK']:
errorMsg = "Could not get list of users for VOMS %s" % (
vomsMapping[group])
self.__adminMsgs['Errors'].append(errorMsg)
self.log.error(errorMsg, result['Message'])
return result
users = result['Value']
numUsersInGroup = 0
for vomsUser in users:
for userName in usersData:
if vomsUser['DN'] in usersData[userName]['DN']:
numUsersInGroup += 1
usersData[userName]['Groups'].extend(groupsForRole)
infoMsg = "There are %s users in group(s) %s for VOMS Role %s" % (
numUsersInGroup, ",".join(groupsForRole), vomsRole)
self.__adminMsgs['Info'].append(infoMsg)
self.log.info(" %s" % infoMsg)
self.log.info("Checking static groups")
staticUsers = []
for group in staticGroups:
self.log.info(" Checking static group %s" % group)
numUsersInGroup = 0
result = csapi.listUsers(group)
if not result['OK']:
self.log.error(
"Could not get the list of users in DIRAC group %s" %
group, result['Message'])
return result
for userName in result['Value']:
if userName in usersData:
numUsersInGroup += 1
usersData[userName]['Groups'].append(group)
else:
if group not in vomsGroups and userName not in staticUsers:
staticUsers.append(userName)
infoMsg = "There are %s users in group %s" % (numUsersInGroup,
group)
self.__adminMsgs['Info'].append(infoMsg)
self.log.info(" %s" % infoMsg)
if staticUsers:
infoMsg = "There are %s static users: %s" % (
len(staticUsers), ', '.join(staticUsers))
self.__adminMsgs['Info'].append(infoMsg)
self.log.info("%s" % infoMsg)
for user in currentUsers:
if user not in usersData and user not in staticUsers:
self.log.info('User %s is no longer valid' % user)
obsoleteUserNames.append(user)
#Do the CS Sync
self.log.info("Updating CS...")
ret = csapi.downloadCSData()
if not ret['OK']:
self.log.fatal('Can not update from CS', ret['Message'])
return ret
usersWithMoreThanOneDN = {}
for user in usersData:
csUserData = dict(usersData[user])
if len(csUserData['DN']) > 1:
usersWithMoreThanOneDN[user] = csUserData['DN']
result = csapi.describeUsers([user])
if result['OK']:
if result['Value']:
prevUser = result['Value'][user]
prevDNs = List.fromChar(prevUser['DN'])
newDNs = csUserData['DN']
for DN in newDNs:
if DN not in prevDNs:
self.__adminMsgs['Info'].append(
"User %s has new DN %s" % (user, DN))
for DN in prevDNs:
if DN not in newDNs:
self.__adminMsgs['Info'].append(
"User %s has lost a DN %s" % (user, DN))
else:
newDNs = csUserData['DN']
for DN in newDNs:
self.__adminMsgs['Info'].append(
"New user %s has new DN %s" % (user, DN))
for k in ('DN', 'CA', 'Email'):
csUserData[k] = ", ".join(csUserData[k])
result = csapi.modifyUser(user,
csUserData,
createIfNonExistant=True)
if not result['OK']:
self.__adminMsgs['Error'].append("Cannot modify user %s: %s" %
(user, result['Message']))
self.log.error("Cannot modify user", user)
if usersWithMoreThanOneDN:
self.__adminMsgs['Info'].append("\nUsers with more than one DN:")
for uwmtod in sorted(usersWithMoreThanOneDN):
self.__adminMsgs['Info'].append(" %s" % uwmtod)
self.__adminMsgs['Info'].append(" + DN list:")
for DN in usersWithMoreThanOneDN[uwmtod]:
self.__adminMsgs['Info'].append(" - %s" % DN)
if obsoleteUserNames:
self.__adminMsgs['Info'].append("\nObsolete users:")
address = self.am_getOption('MailTo', '*****@*****.**')
fromAddress = self.am_getOption('mailFrom', '*****@*****.**')
subject = 'Obsolete LFC Users found'
body = 'Delete entries into LFC: \n'
for obsoleteUser in obsoleteUserNames:
self.log.info(subject, ", ".join(obsoleteUserNames))
body += 'for ' + obsoleteUser + '\n'
self.__adminMsgs['Info'].append(" %s" % obsoleteUser)
self.log.info("Deleting %s users" % len(obsoleteUserNames))
NotificationClient().sendMail(address,
'UsersAndGroupsAgent: %s' % subject,
body, fromAddress)
csapi.deleteUsers(obsoleteUserNames)
if newUserNames:
self.__adminMsgs['Info'].append("\nNew users:")
for newUser in newUserNames:
self.__adminMsgs['Info'].append(" %s" % newUser)
self.__adminMsgs['Info'].append(" + DN list:")
for DN in usersData[newUser]['DN']:
self.__adminMsgs['Info'].append(" - %s" % DN)
self.__adminMsgs['Info'].append(" + EMail: %s" %
usersData[newUser]['Email'])
result = csapi.commitChanges()
if not result['OK']:
self.log.error("Could not commit configuration changes",
result['Message'])
return result
self.log.info("Configuration committed")
#LFC Check
if self.am_getOption("LFCCheckEnabled", True):
result = self.checkLFCRegisteredUsers(usersData)
if not result['OK']:
return result
return S_OK()
class CE2CSAgent( AgentModule ):
def __init__( self, agentName, baseAgentName, properties ):
AgentModule.__init__( self, agentName, baseAgentName, properties )
self.addressTo = ''
self.addressFrom = ''
self.voName = ''
self.csAPI = CSAPI()
self.subject = "CE2CSAgent"
self.alternativeBDIIs = []
def initialize( self ):
# TODO: Have no default and if no mail is found then use the diracAdmin group
# and resolve all associated mail addresses.
self.addressTo = self.am_getOption( 'MailTo', self.addressTo )
self.addressFrom = self.am_getOption( 'MailFrom', self.addressFrom )
# create a list of alternative bdii urls
self.alternativeBDIIs = self.am_getOption( 'AlternativeBDIIs', [] )
# check if the bdii url is appended by a port number, if not append the default 2170
for index, url in enumerate( self.alternativeBDIIs ):
if not url.split( ':' )[-1].isdigit():
self.alternativeBDIIs[index] += ':2170'
if self.addressTo and self.addressFrom:
self.log.info( "MailTo", self.addressTo )
self.log.info( "MailFrom", self.addressFrom )
if self.alternativeBDIIs :
self.log.info( "AlternativeBDII URLs:", self.alternativeBDIIs )
self.subject = "CE2CSAgent"
# This sets the Default Proxy to used as that defined under
# /Operations/Shifter/SAMManager
# the shifterProxy option in the Configuration can be used to change this default.
self.am_setOption( 'shifterProxy', 'SAMManager' )
self.voName = self.am_getOption( 'VirtualOrganization', self.voName )
if not self.voName:
self.voName = getVO()
if not self.voName:
self.log.fatal( "VO option not defined for agent" )
return S_ERROR()
return S_OK()
def execute( self ):
self.log.info( "Start Execution" )
result = getProxyInfo()
if not result[ 'OK' ]:
return result
infoDict = result[ 'Value' ]
self.log.info( formatProxyInfoAsString( infoDict ) )
self.__lookForCE()
self.__infoFromCE()
self.log.info( "End Execution" )
return S_OK()
def __checkAlternativeBDIISite( self, fun, *args ):
if self.alternativeBDIIs:
self.log.warn( "Trying to use alternative bdii sites" )
for site in self.alternativeBDIIs :
self.log.info( "Trying to contact alternative bdii ", site )
if len( args ) == 1 :
result = fun( args[0], host = site )
elif len( args ) == 2 :
result = fun( args[0], vo = args[1], host = site )
if not result['OK'] :
self.log.error ( "Problem contacting alternative bddii", result['Message'] )
elif result['OK'] :
return result
self.log.warn( "Also checking alternative BDII sites failed" )
return result
def __lookForCE( self ):
knownces = self.am_getOption( 'BannedCEs', [] )
result = gConfig.getSections( '/Resources/Sites' )
if not result['OK']:
return
grids = result['Value']
for grid in grids:
result = gConfig.getSections( '/Resources/Sites/%s' % grid )
if not result['OK']:
return
sites = result['Value']
for site in sites:
opt = gConfig.getOptionsDict( '/Resources/Sites/%s/%s' % ( grid, site ) )['Value']
ces = List.fromChar( opt.get( 'CE', '' ) )
knownces += ces
response = ldapCEState( '', vo = self.voName )
if not response['OK']:
self.log.error( "Error during BDII request", response['Message'] )
response = self.__checkAlternativeBDIISite( ldapCEState, '', self.voName )
return response
newces = {}
for queue in response['Value']:
try:
queuename = queue['GlueCEUniqueID']
except:
continue
cename = queuename.split( ":" )[0]
if not cename in knownces:
newces[cename] = None
self.log.debug( "newce", cename )
body = ""
possibleNewSites = []
for ce in newces.iterkeys():
response = ldapCluster( ce )
if not response['OK']:
self.log.warn( "Error during BDII request", response['Message'] )
response = self.__checkAlternativeBDIISite( ldapCluster, ce )
continue
clusters = response['Value']
if len( clusters ) != 1:
self.log.warn( "Error in cluster length", " CE %s Length %d" % ( ce, len( clusters ) ) )
if len( clusters ) == 0:
continue
cluster = clusters[0]
fkey = cluster.get( 'GlueForeignKey', [] )
if type( fkey ) == type( '' ):
fkey = [fkey]
nameBDII = None
for entry in fkey:
if entry.count( 'GlueSiteUniqueID' ):
nameBDII = entry.split( '=' )[1]
break
if not nameBDII:
continue
cestring = "CE: %s, GOCDB Name: %s" % ( ce, nameBDII )
self.log.info( cestring )
response = ldapCE( ce )
if not response['OK']:
self.log.warn( "Error during BDII request", response['Message'] )
response = self.__checkAlternativeBDIISite( ldapCE, ce )
continue
ceinfos = response['Value']
if len( ceinfos ):
ceinfo = ceinfos[0]
systemName = ceinfo.get( 'GlueHostOperatingSystemName', 'Unknown' )
systemVersion = ceinfo.get( 'GlueHostOperatingSystemVersion', 'Unknown' )
systemRelease = ceinfo.get( 'GlueHostOperatingSystemRelease', 'Unknown' )
else:
systemName = "Unknown"
systemVersion = "Unknown"
systemRelease = "Unknown"
osstring = "SystemName: %s, SystemVersion: %s, SystemRelease: %s" % ( systemName, systemVersion, systemRelease )
self.log.info( osstring )
response = ldapCEState( ce, vo = self.voName )
if not response['OK']:
self.log.warn( "Error during BDII request", response['Message'] )
response = self.__checkAlternativeBDIISite( ldapCEState, ce, self.voName )
continue
newcestring = "\n\n%s\n%s" % ( cestring, osstring )
usefull = False
cestates = response['Value']
for cestate in cestates:
queuename = cestate.get( 'GlueCEUniqueID', 'UnknownName' )
queuestatus = cestate.get( 'GlueCEStateStatus', 'UnknownStatus' )
queuestring = "%s %s" % ( queuename, queuestatus )
self.log.info( queuestring )
newcestring += "\n%s" % queuestring
if queuestatus.count( 'Production' ):
usefull = True
if usefull:
body += newcestring
possibleNewSites.append( 'dirac-admin-add-site DIRACSiteName %s %s' % ( nameBDII, ce ) )
if body:
body = "We are glade to inform You about new CE(s) possibly suitable for %s:\n" % self.voName + body
body += "\n\nTo suppress information about CE add its name to BannedCEs list."
for possibleNewSite in possibleNewSites:
body = "%s\n%s" % ( body, possibleNewSite )
self.log.info( body )
if self.addressTo and self.addressFrom:
notification = NotificationClient()
result = notification.sendMail( self.addressTo, self.subject, body, self.addressFrom, localAttempt = False )
return S_OK()
def __infoFromCE( self ):
sitesSection = cfgPath( 'Resources', 'Sites' )
result = gConfig.getSections( sitesSection )
if not result['OK']:
return
grids = result['Value']
for grid in grids:
gridSection = cfgPath( sitesSection, grid )
result = gConfig.getSections( gridSection )
if not result['OK']:
return
sites = result['Value']
changed = False
body = ""
for site in sites:
# if site[-2:]!='ru':
# continue
siteSection = cfgPath( gridSection, site )
opt = gConfig.getOptionsDict( siteSection )['Value']
name = opt.get( 'Name', '' )
if name:
coor = opt.get( 'Coordinates', 'Unknown' )
mail = opt.get( 'Mail', 'Unknown' )
result = ldapSite( name )
if not result['OK']:
self.log.warn( "BDII site %s: %s" % ( name, result['Message'] ) )
result = self.__checkAlternativeBDIISite( ldapSite, name )
if result['OK']:
bdiisites = result['Value']
if len( bdiisites ) == 0:
self.log.warn( name, "Error in bdii: leng = 0" )
else:
if not len( bdiisites ) == 1:
self.log.warn( name, "Warning in bdii: leng = %d" % len( bdiisites ) )
bdiisite = bdiisites[0]
try:
longitude = bdiisite['GlueSiteLongitude']
latitude = bdiisite['GlueSiteLatitude']
newcoor = "%s:%s" % ( longitude, latitude )
except:
self.log.warn( "Error in bdii coor" )
newcoor = "Unknown"
try:
newmail = bdiisite['GlueSiteSysAdminContact'].split( ":" )[-1].strip()
except:
self.log.warn( "Error in bdii mail" )
newmail = "Unknown"
self.log.debug( "%s %s %s" % ( name, newcoor, newmail ) )
if newcoor != coor:
self.log.info( "%s" % ( name ), "%s -> %s" % ( coor, newcoor ) )
if coor == 'Unknown':
self.csAPI.setOption( cfgPath( siteSection, 'Coordinates' ), newcoor )
else:
self.csAPI.modifyValue( cfgPath( siteSection, 'Coordinates' ), newcoor )
changed = True
if newmail != mail:
self.log.info( "%s" % ( name ), "%s -> %s" % ( mail, newmail ) )
if mail == 'Unknown':
self.csAPI.setOption( cfgPath( siteSection, 'Mail' ), newmail )
else:
self.csAPI.modifyValue( cfgPath( siteSection, 'Coordinates' ), newmail )
changed = True
celist = List.fromChar( opt.get( 'CE', '' ) )
if not celist:
self.log.warn( site, 'Empty site list' )
continue
# result = gConfig.getSections( cfgPath( siteSection,'CEs' )
# if not result['OK']:
# self.log.debug( "Section CEs:", result['Message'] )
for ce in celist:
ceSection = cfgPath( siteSection, 'CEs', ce )
result = gConfig.getOptionsDict( ceSection )
if not result['OK']:
self.log.debug( "Section CE", result['Message'] )
wnTmpDir = 'Unknown'
arch = 'Unknown'
os = 'Unknown'
si00 = 'Unknown'
pilot = 'Unknown'
cetype = 'Unknown'
else:
ceopt = result['Value']
wnTmpDir = ceopt.get( 'wnTmpDir', 'Unknown' )
arch = ceopt.get( 'architecture', 'Unknown' )
os = ceopt.get( 'OS', 'Unknown' )
si00 = ceopt.get( 'SI00', 'Unknown' )
pilot = ceopt.get( 'Pilot', 'Unknown' )
cetype = ceopt.get( 'CEType', 'Unknown' )
result = ldapCE( ce )
if not result['OK']:
self.log.warn( 'Error in bdii for %s' % ce, result['Message'] )
result = self.__checkAlternativeBDIISite( ldapCE, ce )
continue
try:
bdiice = result['Value'][0]
except:
self.log.warn( 'Error in bdii for %s' % ce, result )
bdiice = None
if bdiice:
try:
newwnTmpDir = bdiice['GlueSubClusterWNTmpDir']
except:
newwnTmpDir = 'Unknown'
if wnTmpDir != newwnTmpDir and newwnTmpDir != 'Unknown':
section = cfgPath( ceSection, 'wnTmpDir' )
self.log.info( section, " -> ".join( ( wnTmpDir, newwnTmpDir ) ) )
if wnTmpDir == 'Unknown':
self.csAPI.setOption( section, newwnTmpDir )
else:
self.csAPI.modifyValue( section, newwnTmpDir )
changed = True
try:
newarch = bdiice['GlueHostArchitecturePlatformType']
except:
newarch = 'Unknown'
if arch != newarch and newarch != 'Unknown':
section = cfgPath( ceSection, 'architecture' )
self.log.info( section, " -> ".join( ( arch, newarch ) ) )
if arch == 'Unknown':
self.csAPI.setOption( section, newarch )
else:
self.csAPI.modifyValue( section, newarch )
changed = True
try:
newos = '_'.join( ( bdiice['GlueHostOperatingSystemName'],
bdiice['GlueHostOperatingSystemVersion'],
bdiice['GlueHostOperatingSystemRelease'] ) )
except:
newos = 'Unknown'
if os != newos and newos != 'Unknown':
section = cfgPath( ceSection, 'OS' )
self.log.info( section, " -> ".join( ( os, newos ) ) )
if os == 'Unknown':
self.csAPI.setOption( section, newos )
else:
self.csAPI.modifyValue( section, newos )
changed = True
body = body + "OS was changed %s -> %s for %s at %s\n" % ( os, newos, ce, site )
try:
newsi00 = bdiice['GlueHostBenchmarkSI00']
except:
newsi00 = 'Unknown'
if si00 != newsi00 and newsi00 != 'Unknown':
section = cfgPath( ceSection, 'SI00' )
self.log.info( section, " -> ".join( ( si00, newsi00 ) ) )
if si00 == 'Unknown':
self.csAPI.setOption( section, newsi00 )
else:
self.csAPI.modifyValue( section, newsi00 )
changed = True
try:
rte = bdiice['GlueHostApplicationSoftwareRunTimeEnvironment']
if self.voName == 'lhcb':
if 'VO-lhcb-pilot' in rte:
newpilot = 'True'
else:
newpilot = 'False'
else:
newpilot = 'Unknown'
except:
newpilot = 'Unknown'
if pilot != newpilot and newpilot != 'Unknown':
section = cfgPath( ceSection, 'Pilot' )
self.log.info( section, " -> ".join( ( pilot, newpilot ) ) )
if pilot == 'Unknown':
self.csAPI.setOption( section, newpilot )
else:
self.csAPI.modifyValue( section, newpilot )
changed = True
result = ldapService( ce )
if not result['OK'] :
result = self.__checkAlternativeBDIISite( ldapService, ce )
if result['OK']:
services = result['Value']
newcetype = 'LCG'
for service in services:
if service['GlueServiceType'].count( 'CREAM' ):
newcetype = "CREAM"
else:
newcetype = 'Unknown'
if cetype != newcetype and newcetype != 'Unknown':
section = cfgPath( ceSection, 'CEType' )
self.log.info( section, " -> ".join( ( cetype, newcetype ) ) )
if cetype == 'Unknown':
self.csAPI.setOption( section, newcetype )
else:
self.csAPI.modifyValue( section, newcetype )
changed = True
result = ldapCEState( ce, vo = self.voName ) #getBDIICEVOView
if not result['OK']:
self.log.warn( 'Error in bdii for queue %s' % ce, result['Message'] )
result = self.__checkAlternativeBDIISite( ldapCEState, ce, self.voName )
continue
try:
queues = result['Value']
except:
self.log.warn( 'Error in bdii for queue %s' % ce, result['Massage'] )
continue
for queue in queues:
try:
queueName = queue['GlueCEUniqueID'].split( '/' )[-1]
except:
self.log.warn( 'error in queuename ', queue )
continue
try:
newmaxCPUTime = queue['GlueCEPolicyMaxCPUTime']
except:
newmaxCPUTime = None
newsi00 = None
try:
caps = queue['GlueCECapability']
if type( caps ) == type( '' ):
caps = [caps]
for cap in caps:
if cap.count( 'CPUScalingReferenceSI00' ):
newsi00 = cap.split( '=' )[-1]
except:
newsi00 = None
queueSection = cfgPath( ceSection, 'Queues', queueName )
result = gConfig.getOptionsDict( queueSection )
if not result['OK']:
self.log.warn( "Section Queues", result['Message'] )
maxCPUTime = 'Unknown'
si00 = 'Unknown'
else:
queueopt = result['Value']
maxCPUTime = queueopt.get( 'maxCPUTime', 'Unknown' )
si00 = queueopt.get( 'SI00', 'Unknown' )
if newmaxCPUTime and ( maxCPUTime != newmaxCPUTime ):
section = cfgPath( queueSection, 'maxCPUTime' )
self.log.info( section, " -> ".join( ( maxCPUTime, newmaxCPUTime ) ) )
if maxCPUTime == 'Unknown':
self.csAPI.setOption( section, newmaxCPUTime )
else:
self.csAPI.modifyValue( section, newmaxCPUTime )
changed = True
if newsi00 and ( si00 != newsi00 ):
section = cfgPath( queueSection, 'SI00' )
self.log.info( section, " -> ".join( ( si00, newsi00 ) ) )
if si00 == 'Unknown':
self.csAPI.setOption( section, newsi00 )
else:
self.csAPI.modifyValue( section, newsi00 )
changed = True
if changed:
self.log.info( body )
if body and self.addressTo and self.addressFrom:
notification = NotificationClient()
result = notification.sendMail( self.addressTo, self.subject, body, self.addressFrom, localAttempt = False )
return self.csAPI.commitChanges( sortUsers = False )
else:
self.log.info( "No changes found" )
return S_OK()
class DiracAdmin(API):
""" Administrative functionalities
"""
#############################################################################
def __init__(self):
"""Internal initialization of the DIRAC Admin API.
"""
super(DiracAdmin, self).__init__()
self.csAPI = CSAPI()
self.dbg = False
if gConfig.getValue(self.section + '/LogLevel', 'DEBUG') == 'DEBUG':
self.dbg = True
self.scratchDir = gConfig.getValue(self.section + '/ScratchDir',
'/tmp')
self.currentDir = os.getcwd()
self.rssFlag = ResourceStatus().rssFlag
self.sitestatus = SiteStatus()
self._siteSet = set(getSites().get('Value', []))
#############################################################################
def uploadProxy(self):
"""Upload a proxy to the DIRAC WMS. This method
Example usage:
>>> print diracAdmin.uploadProxy('dteam_pilot')
{'OK': True, 'Value': 0L}
:return: S_OK,S_ERROR
:param permanent: Indefinitely update proxy
:type permanent: boolean
"""
return gProxyManager.uploadProxy()
#############################################################################
def setProxyPersistency(self, userDN, userGroup, persistent=True):
"""Set the persistence of a proxy in the Proxy Manager
Example usage:
>>> gLogger.notice(diracAdmin.setProxyPersistency( 'some DN', 'dirac group', True ))
{'OK': True }
:param userDN: User DN
:type userDN: string
:param userGroup: DIRAC Group
:type userGroup: string
:param persistent: Persistent flag
:type persistent: boolean
:return: S_OK,S_ERROR
"""
return gProxyManager.setPersistency(userDN, userGroup, persistent)
#############################################################################
def checkProxyUploaded(self, userDN, userGroup, requiredTime):
"""Set the persistence of a proxy in the Proxy Manager
Example usage:
>>> gLogger.notice(diracAdmin.setProxyPersistency( 'some DN', 'dirac group', True ))
{'OK': True, 'Value' : True/False }
:param userDN: User DN
:type userDN: string
:param userGroup: DIRAC Group
:type userGroup: string
:param requiredTime: Required life time of the uploaded proxy
:type requiredTime: boolean
:return: S_OK,S_ERROR
"""
return gProxyManager.userHasProxy(userDN, userGroup, requiredTime)
#############################################################################
def getSiteMask(self, printOutput=False, status='Active'):
"""Retrieve current site mask from WMS Administrator service.
Example usage:
>>> gLogger.notice(diracAdmin.getSiteMask())
{'OK': True, 'Value': 0L}
:return: S_OK,S_ERROR
"""
result = self.sitestatus.getSites(siteState=status)
if result['OK']:
sites = result['Value']
if printOutput:
sites.sort()
for site in sites:
gLogger.notice(site)
return result
#############################################################################
def getBannedSites(self, printOutput=False):
"""Retrieve current list of banned and probing sites.
Example usage:
>>> gLogger.notice(diracAdmin.getBannedSites())
{'OK': True, 'Value': []}
:return: S_OK,S_ERROR
"""
bannedSites = self.sitestatus.getSites(siteState='Banned')
if not bannedSites['OK']:
return bannedSites
probingSites = self.sitestatus.getSites(siteState='Probing')
if not probingSites['OK']:
return probingSites
mergedList = sorted(bannedSites['Value'] + probingSites['Value'])
if printOutput:
gLogger.notice('\n'.join(mergedList))
return S_OK(mergedList)
#############################################################################
def getSiteSection(self, site, printOutput=False):
"""Simple utility to get the list of CEs for DIRAC site name.
Example usage:
>>> gLogger.notice(diracAdmin.getSiteSection('LCG.CERN.ch'))
{'OK': True, 'Value':}
:return: S_OK,S_ERROR
"""
gridType = site.split('.')[0]
if not gConfig.getSections('/Resources/Sites/%s' % (gridType))['OK']:
return S_ERROR('/Resources/Sites/%s is not a valid site section' %
(gridType))
result = gConfig.getOptionsDict('/Resources/Sites/%s/%s' %
(gridType, site))
if printOutput and result['OK']:
gLogger.notice(self.pPrint.pformat(result['Value']))
return result
#############################################################################
def allowSite(self, site, comment, printOutput=False):
"""Adds the site to the site mask.
Example usage:
>>> gLogger.notice(diracAdmin.allowSite())
{'OK': True, 'Value': }
:return: S_OK,S_ERROR
"""
result = self.__checkSiteIsValid(site)
if not result['OK']:
return result
result = self.getSiteMask(status='Active')
if not result['OK']:
return result
siteMask = result['Value']
if site in siteMask:
if printOutput:
gLogger.notice('Site %s is already Active' % site)
return S_OK('Site %s is already Active' % site)
if self.rssFlag:
result = self.sitestatus.setSiteStatus(site, 'Active', comment)
else:
result = WMSAdministratorClient().allowSite(site, comment)
if not result['OK']:
return result
if printOutput:
gLogger.notice('Site %s status is set to Active' % site)
return result
#############################################################################
def getSiteMaskLogging(self, site=None, printOutput=False):
"""Retrieves site mask logging information.
Example usage:
>>> gLogger.notice(diracAdmin.getSiteMaskLogging('LCG.AUVER.fr'))
{'OK': True, 'Value': }
:return: S_OK,S_ERROR
"""
result = self.__checkSiteIsValid(site)
if not result['OK']:
return result
if self.rssFlag:
result = ResourceStatusClient().selectStatusElement('Site',
'History',
name=site)
else:
result = WMSAdministratorClient().getSiteMaskLogging(site)
if not result['OK']:
return result
if printOutput:
if site:
gLogger.notice('\nSite Mask Logging Info for %s\n' % site)
else:
gLogger.notice('\nAll Site Mask Logging Info\n')
sitesLogging = result['Value']
if isinstance(sitesLogging, dict):
for siteName, tupleList in sitesLogging.items(
): # can be an iterator
if not siteName:
gLogger.notice('\n===> %s\n' % siteName)
for tup in tupleList:
stup = str(tup[0]).ljust(8) + str(tup[1]).ljust(20)
stup += '( ' + str(tup[2]).ljust(len(str(
tup[2]))) + ' ) "' + str(tup[3]) + '"'
gLogger.notice(stup)
gLogger.notice(' ')
elif isinstance(sitesLogging, list):
sitesLoggingList = [(sl[1], sl[3], sl[4])
for sl in sitesLogging]
for siteLog in sitesLoggingList:
gLogger.notice(siteLog)
return S_OK()
#############################################################################
def banSite(self, site, comment, printOutput=False):
"""Removes the site from the site mask.
Example usage:
>>> gLogger.notice(diracAdmin.banSite())
{'OK': True, 'Value': }
:return: S_OK,S_ERROR
"""
result = self.__checkSiteIsValid(site)
if not result['OK']:
return result
mask = self.getSiteMask(status='Banned')
if not mask['OK']:
return mask
siteMask = mask['Value']
if site in siteMask:
if printOutput:
gLogger.notice('Site %s is already Banned' % site)
return S_OK('Site %s is already Banned' % site)
if self.rssFlag:
result = self.sitestatus.setSiteStatus(site, 'Banned', comment)
else:
result = WMSAdministratorClient().banSite(site, comment)
if not result['OK']:
return result
if printOutput:
gLogger.notice('Site %s status is set to Banned' % site)
return result
#############################################################################
def __checkSiteIsValid(self, site):
"""Internal function to check that a site name is valid.
"""
if isinstance(site, (list, set, dict)):
site = set(site) - self._siteSet
if not site:
return S_OK()
elif site in self._siteSet:
return S_OK()
return S_ERROR('Specified site %s is not in list of defined sites' %
str(site))
#############################################################################
def getServicePorts(self, setup='', printOutput=False):
"""Checks the service ports for the specified setup. If not given this is
taken from the current installation (/DIRAC/Setup)
Example usage:
>>> gLogger.notice(diracAdmin.getServicePorts())
{'OK': True, 'Value':''}
:return: S_OK,S_ERROR
"""
if not setup:
setup = gConfig.getValue('/DIRAC/Setup', '')
setupList = gConfig.getSections('/DIRAC/Setups', [])
if not setupList['OK']:
return S_ERROR('Could not get /DIRAC/Setups sections')
setupList = setupList['Value']
if setup not in setupList:
return S_ERROR('Setup %s is not in allowed list: %s' %
(setup, ', '.join(setupList)))
serviceSetups = gConfig.getOptionsDict('/DIRAC/Setups/%s' % setup)
if not serviceSetups['OK']:
return S_ERROR('Could not get /DIRAC/Setups/%s options' % setup)
serviceSetups = serviceSetups['Value'] # dict
systemList = gConfig.getSections('/Systems')
if not systemList['OK']:
return S_ERROR('Could not get Systems sections')
systemList = systemList['Value']
result = {}
for system in systemList:
if system in serviceSetups:
path = '/Systems/%s/%s/Services' % (system,
serviceSetups[system])
servicesList = gConfig.getSections(path)
if not servicesList['OK']:
self.log.warn('Could not get sections in %s' % path)
else:
servicesList = servicesList['Value']
if not servicesList:
servicesList = []
self.log.verbose('System: %s ServicesList: %s' %
(system, ', '.join(servicesList)))
for service in servicesList:
spath = '%s/%s/Port' % (path, service)
servicePort = gConfig.getValue(spath, 0)
if servicePort:
self.log.verbose('Found port for %s/%s = %s' %
(system, service, servicePort))
result['%s/%s' % (system, service)] = servicePort
else:
self.log.warn('No port found for %s' % spath)
else:
self.log.warn('%s is not defined in /DIRAC/Setups/%s' %
(system, setup))
if printOutput:
gLogger.notice(self.pPrint.pformat(result))
return S_OK(result)
#############################################################################
def getProxy(self, userDN, userGroup, validity=43200, limited=False):
"""Retrieves a proxy with default 12hr validity and stores
this in a file in the local directory by default.
Example usage:
>>> gLogger.notice(diracAdmin.getProxy())
{'OK': True, 'Value': }
:return: S_OK,S_ERROR
"""
return gProxyManager.downloadProxy(userDN,
userGroup,
limited=limited,
requiredTimeLeft=validity)
#############################################################################
def getVOMSProxy(self,
userDN,
userGroup,
vomsAttr=False,
validity=43200,
limited=False):
"""Retrieves a proxy with default 12hr validity and VOMS extensions and stores
this in a file in the local directory by default.
Example usage:
>>> gLogger.notice(diracAdmin.getVOMSProxy())
{'OK': True, 'Value': }
:return: S_OK,S_ERROR
"""
return gProxyManager.downloadVOMSProxy(userDN,
userGroup,
limited=limited,
requiredVOMSAttribute=vomsAttr,
requiredTimeLeft=validity)
#############################################################################
def getPilotProxy(self, userDN, userGroup, validity=43200):
"""Retrieves a pilot proxy with default 12hr validity and stores
this in a file in the local directory by default.
Example usage:
>>> gLogger.notice(diracAdmin.getVOMSProxy())
{'OK': True, 'Value': }
:return: S_OK,S_ERROR
"""
return gProxyManager.getPilotProxyFromDIRACGroup(
userDN, userGroup, requiredTimeLeft=validity)
#############################################################################
def resetJob(self, jobID):
"""Reset a job or list of jobs in the WMS. This operation resets the reschedule
counter for a job or list of jobs and allows them to run as new.
Example::
>>> gLogger.notice(dirac.reset(12345))
{'OK': True, 'Value': [12345]}
:param job: JobID
:type job: integer or list of integers
:return: S_OK,S_ERROR
"""
if isinstance(jobID, six.string_types):
try:
jobID = int(jobID)
except Exception as x:
return self._errorReport(
str(x),
'Expected integer or convertible integer for existing jobID'
)
elif isinstance(jobID, list):
try:
jobID = [int(job) for job in jobID]
except Exception as x:
return self._errorReport(
str(x),
'Expected integer or convertible integer for existing jobIDs'
)
result = JobManagerClient(useCertificates=False).resetJob(jobID)
return result
#############################################################################
def getJobPilotOutput(self, jobID, directory=''):
"""Retrieve the pilot output for an existing job in the WMS.
The output will be retrieved in a local directory unless
otherwise specified.
>>> gLogger.notice(dirac.getJobPilotOutput(12345))
{'OK': True, StdOut:'',StdError:''}
:param job: JobID
:type job: integer or string
:return: S_OK,S_ERROR
"""
if not directory:
directory = self.currentDir
if not os.path.exists(directory):
return self._errorReport('Directory %s does not exist' % directory)
result = WMSAdministratorClient().getJobPilotOutput(jobID)
if not result['OK']:
return result
outputPath = '%s/pilot_%s' % (directory, jobID)
if os.path.exists(outputPath):
self.log.info('Remove %s and retry to continue' % outputPath)
return S_ERROR('Remove %s and retry to continue' % outputPath)
if not os.path.exists(outputPath):
self.log.verbose('Creating directory %s' % outputPath)
os.mkdir(outputPath)
outputs = result['Value']
if 'StdOut' in outputs:
stdout = '%s/std.out' % (outputPath)
with open(stdout, 'w') as fopen:
fopen.write(outputs['StdOut'])
self.log.verbose('Standard output written to %s' % (stdout))
else:
self.log.warn('No standard output returned')
if 'StdError' in outputs:
stderr = '%s/std.err' % (outputPath)
with open(stderr, 'w') as fopen:
fopen.write(outputs['StdError'])
self.log.verbose('Standard error written to %s' % (stderr))
else:
self.log.warn('No standard error returned')
self.log.always('Outputs retrieved in %s' % outputPath)
return result
#############################################################################
def getPilotOutput(self, gridReference, directory=''):
"""Retrieve the pilot output (std.out and std.err) for an existing job in the WMS.
>>> gLogger.notice(dirac.getJobPilotOutput(12345))
{'OK': True, 'Value': {}}
:param job: JobID
:type job: integer or string
:return: S_OK,S_ERROR
"""
if not isinstance(gridReference, six.string_types):
return self._errorReport('Expected string for pilot reference')
if not directory:
directory = self.currentDir
if not os.path.exists(directory):
return self._errorReport('Directory %s does not exist' % directory)
result = PilotManagerClient().getPilotOutput(gridReference)
if not result['OK']:
return result
gridReferenceSmall = gridReference.split('/')[-1]
if not gridReferenceSmall:
gridReferenceSmall = 'reference'
outputPath = '%s/pilot_%s' % (directory, gridReferenceSmall)
if os.path.exists(outputPath):
self.log.info('Remove %s and retry to continue' % outputPath)
return S_ERROR('Remove %s and retry to continue' % outputPath)
if not os.path.exists(outputPath):
self.log.verbose('Creating directory %s' % outputPath)
os.mkdir(outputPath)
outputs = result['Value']
if 'StdOut' in outputs:
stdout = '%s/std.out' % (outputPath)
with open(stdout, 'w') as fopen:
fopen.write(outputs['StdOut'])
self.log.info('Standard output written to %s' % (stdout))
else:
self.log.warn('No standard output returned')
if 'StdErr' in outputs:
stderr = '%s/std.err' % (outputPath)
with open(stderr, 'w') as fopen:
fopen.write(outputs['StdErr'])
self.log.info('Standard error written to %s' % (stderr))
else:
self.log.warn('No standard error returned')
self.log.always('Outputs retrieved in %s' % outputPath)
return result
#############################################################################
def getPilotInfo(self, gridReference):
"""Retrieve info relative to a pilot reference
>>> gLogger.notice(dirac.getPilotInfo(12345))
{'OK': True, 'Value': {}}
:param gridReference: Pilot Job Reference
:type gridReference: string
:return: S_OK,S_ERROR
"""
if not isinstance(gridReference, six.string_types):
return self._errorReport('Expected string for pilot reference')
result = PilotManagerClient().getPilotInfo(gridReference)
return result
#############################################################################
def killPilot(self, gridReference):
"""Kill the pilot specified
>>> gLogger.notice(dirac.getPilotInfo(12345))
{'OK': True, 'Value': {}}
:param gridReference: Pilot Job Reference
:return: S_OK,S_ERROR
"""
if not isinstance(gridReference, six.string_types):
return self._errorReport('Expected string for pilot reference')
result = PilotManagerClient().killPilot(gridReference)
return result
#############################################################################
def getPilotLoggingInfo(self, gridReference):
"""Retrieve the pilot logging info for an existing job in the WMS.
>>> gLogger.notice(dirac.getPilotLoggingInfo(12345))
{'OK': True, 'Value': {"The output of the command"}}
:param gridReference: Gridp pilot job reference Id
:type gridReference: string
:return: S_OK,S_ERROR
"""
if not isinstance(gridReference, six.string_types):
return self._errorReport('Expected string for pilot reference')
return PilotManagerClient().getPilotLoggingInfo(gridReference)
#############################################################################
def getJobPilots(self, jobID):
"""Extract the list of submitted pilots and their status for a given
jobID from the WMS. Useful information is printed to the screen.
>>> gLogger.notice(dirac.getJobPilots())
{'OK': True, 'Value': {PilotID:{StatusDict}}}
:param job: JobID
:type job: integer or string
:return: S_OK,S_ERROR
"""
if isinstance(jobID, six.string_types):
try:
jobID = int(jobID)
except Exception as x:
return self._errorReport(
str(x), 'Expected integer or string for existing jobID')
result = PilotManagerClient().getPilots(jobID)
if result['OK']:
gLogger.notice(self.pPrint.pformat(result['Value']))
return result
#############################################################################
def getPilotSummary(self, startDate='', endDate=''):
"""Retrieve the pilot output for an existing job in the WMS. Summary is
printed at INFO level, full dictionary of results also returned.
>>> gLogger.notice(dirac.getPilotSummary())
{'OK': True, 'Value': {CE:{Status:Count}}}
:param job: JobID
:type job: integer or string
:return: S_OK,S_ERROR
"""
result = PilotManagerClient().getPilotSummary(startDate, endDate)
if not result['OK']:
return result
ceDict = result['Value']
headers = 'CE'.ljust(28)
i = 0
for ce, summary in ceDict.iteritems():
states = summary.keys()
if len(states) > i:
i = len(states)
for i in xrange(i):
headers += 'Status'.ljust(12) + 'Count'.ljust(12)
gLogger.notice(headers)
for ce, summary in ceDict.iteritems():
line = ce.ljust(28)
states = sorted(summary)
for state in states:
count = str(summary[state])
line += state.ljust(12) + count.ljust(12)
gLogger.notice(line)
return result
#############################################################################
def setSiteProtocols(self, site, protocolsList, printOutput=False):
"""
Allows to set the defined protocols for each SE for a given site.
"""
result = self.__checkSiteIsValid(site)
if not result['OK']:
return result
siteSection = '/Resources/Sites/%s/%s/SE' % (site.split('.')[0], site)
siteSEs = gConfig.getValue(siteSection, [])
if not siteSEs:
return S_ERROR('No SEs found for site %s in section %s' %
(site, siteSection))
defaultProtocols = gConfig.getValue(
'/Resources/StorageElements/DefaultProtocols', [])
self.log.verbose('Default list of protocols are',
', '.join(defaultProtocols))
for protocol in protocolsList:
if protocol not in defaultProtocols:
return S_ERROR(
'Requested to set protocol %s in list but %s is not '
'in default list of protocols:\n%s' %
(protocol, protocol, ', '.join(defaultProtocols)))
modifiedCS = False
result = promptUser(
'Do you want to add the following default protocols:'
' %s for SE(s):\n%s' %
(', '.join(protocolsList), ', '.join(siteSEs)))
if not result['OK']:
return result
if result['Value'].lower() != 'y':
self.log.always('No protocols will be added')
return S_OK()
for se in siteSEs:
sections = gConfig.getSections('/Resources/StorageElements/%s/' %
(se))
if not sections['OK']:
return sections
for section in sections['Value']:
if gConfig.getValue(
'/Resources/StorageElements/%s/%s/ProtocolName' %
(se, section), '') == 'SRM2':
path = '/Resources/StorageElements/%s/%s/ProtocolsList' % (
se, section)
self.log.verbose('Setting %s to %s' %
(path, ', '.join(protocolsList)))
result = self.csSetOption(path, ', '.join(protocolsList))
if not result['OK']:
return result
modifiedCS = True
if modifiedCS:
result = self.csCommitChanges(False)
if not result['OK']:
return S_ERROR('CS Commit failed with message = %s' %
(result['Message']))
else:
if printOutput:
gLogger.notice('Successfully committed changes to CS')
else:
if printOutput:
gLogger.notice('No modifications to CS required')
return S_OK()
#############################################################################
def csSetOption(self, optionPath, optionValue):
"""
Function to modify an existing value in the CS.
"""
return self.csAPI.setOption(optionPath, optionValue)
#############################################################################
def csSetOptionComment(self, optionPath, comment):
"""
Function to modify an existing value in the CS.
"""
return self.csAPI.setOptionComment(optionPath, comment)
#############################################################################
def csModifyValue(self, optionPath, newValue):
"""
Function to modify an existing value in the CS.
"""
return self.csAPI.modifyValue(optionPath, newValue)
#############################################################################
def csRegisterUser(self, username, properties):
"""
Registers a user in the CS.
- username: Username of the user (easy;)
- properties: Dict containing:
- DN
- groups : list/tuple of groups the user belongs to
- <others> : More properties of the user, like mail
"""
return self.csAPI.addUser(username, properties)
#############################################################################
def csDeleteUser(self, user):
"""
Deletes a user from the CS. Can take a list of users
"""
return self.csAPI.deleteUsers(user)
#############################################################################
def csModifyUser(self, username, properties, createIfNonExistant=False):
"""
Modify a user in the CS. Takes the same params as in addUser and
applies the changes
"""
return self.csAPI.modifyUser(username, properties, createIfNonExistant)
#############################################################################
def csListUsers(self, group=False):
"""
Lists the users in the CS. If no group is specified return all users.
"""
return self.csAPI.listUsers(group)
#############################################################################
def csDescribeUsers(self, mask=False):
"""
List users and their properties in the CS.
If a mask is given, only users in the mask will be returned
"""
return self.csAPI.describeUsers(mask)
#############################################################################
def csModifyGroup(self, groupname, properties, createIfNonExistant=False):
"""
Modify a user in the CS. Takes the same params as in addGroup and applies
the changes
"""
return self.csAPI.modifyGroup(groupname, properties,
createIfNonExistant)
#############################################################################
def csListHosts(self):
"""
Lists the hosts in the CS
"""
return self.csAPI.listHosts()
#############################################################################
def csDescribeHosts(self, mask=False):
"""
Gets extended info for the hosts in the CS
"""
return self.csAPI.describeHosts(mask)
#############################################################################
def csModifyHost(self, hostname, properties, createIfNonExistant=False):
"""
Modify a host in the CS. Takes the same params as in addHost and applies
the changes
"""
return self.csAPI.modifyHost(hostname, properties, createIfNonExistant)
#############################################################################
def csListGroups(self):
"""
Lists groups in the CS
"""
return self.csAPI.listGroups()
#############################################################################
def csDescribeGroups(self, mask=False):
"""
List groups and their properties in the CS.
If a mask is given, only groups in the mask will be returned
"""
return self.csAPI.describeGroups(mask)
#############################################################################
def csSyncUsersWithCFG(self, usersCFG):
"""
Synchronize users in cfg with its contents
"""
return self.csAPI.syncUsersWithCFG(usersCFG)
#############################################################################
def csCommitChanges(self, sortUsers=True):
"""
Commit the changes in the CS
"""
return self.csAPI.commitChanges(sortUsers=False)
#############################################################################
def sendMail(self,
address,
subject,
body,
fromAddress=None,
localAttempt=True,
html=False):
"""
Send mail to specified address with body.
"""
notification = NotificationClient()
return notification.sendMail(address, subject, body, fromAddress,
localAttempt, html)
#############################################################################
def sendSMS(self, userName, body, fromAddress=None):
"""
Send mail to specified address with body.
"""
if len(body) > 160:
return S_ERROR('Exceeded maximum SMS length of 160 characters')
notification = NotificationClient()
return notification.sendSMS(userName, body, fromAddress)
#############################################################################
def getBDIISite(self, site, host=None):
"""
Get information about site from BDII at host
"""
return ldapSite(site, host=host)
#############################################################################
def getBDIICluster(self, ce, host=None):
"""
Get information about ce from BDII at host
"""
return ldapCluster(ce, host=host)
#############################################################################
def getBDIICE(self, ce, host=None):
"""
Get information about ce from BDII at host
"""
return ldapCE(ce, host=host)
#############################################################################
def getBDIIService(self, ce, host=None):
"""
Get information about ce from BDII at host
"""
return ldapService(ce, host=host)
#############################################################################
def getBDIICEState(self, ce, useVO=voName, host=None):
"""
Get information about ce state from BDII at host
"""
return ldapCEState(ce, useVO, host=host)
#############################################################################
def getBDIICEVOView(self, ce, useVO=voName, host=None):
"""
Get information about ce voview from BDII at host
"""
return ldapCEVOView(ce, useVO, host=host)
change = False
if newSite:
gLogger.notice("Adding new site to CS: %s" % diracSiteName)
csAPI.setOption("%s/Name" % cfgBase, gridSiteName)
gLogger.notice("Adding CEs: %s" % ",".join(ces))
csAPI.setOption("%s/CE" % cfgBase, ",".join(ces))
change = True
else:
cesCS = set(gConfig.getValue("%s/CE" % cfgBase, []))
ces = set(ces)
newCEs = ces - cesCS
if newCEs:
cesCS = cesCS.union(ces)
gLogger.notice("Adding CEs %s" % ",".join(newCEs))
cesCS = cesCS.union(ces)
csAPI.modifyValue("%s/CE" % cfgBase, ",".join(cesCS))
change = True
if change:
res = csAPI.commitChanges()
if not res["OK"]:
gLogger.error("Failed to commit changes to CS", res["Message"])
DIRACExit(-1)
else:
if newSite:
gLogger.notice(
"Successfully added site %s to the CS with name %s and CEs: %s"
% (diracSiteName, gridSiteName, ",".join(ces))
)
else:
gLogger.notice("Successfully added new CEs to site %s: %s" % (diracSiteName, ",".join(newCEs)))
class VOMS2CSAgent( AgentModule ):
def __init__( self, *args, **kwargs ):
""" Defines default parameters
"""
super(VOMS2CSAgent, self).__init__( *args, **kwargs )
self.__voDict = {}
self.__adminMsgs = {}
self.csapi = None
self.voChanged = False
self.dryRun = False
self.autoAddUsers = False
self.autoModifyUsers = False
self.autoDeleteUsers = False
self.detailedReport = True
self.makeFCEntry = False
def initialize( self ):
""" Initialize the default parameters
"""
voNames = self.am_getOption( 'VO', [] )
if not voNames[0].lower() == "none":
if voNames[0].lower() == "any":
voNames = []
result = getVOMSVOs( voNames )
if not result['OK']:
return result
self.__voDict = result['Value']
self.log.notice( "VOs: %s" % self.__voDict.keys() )
self.csapi = CSAPI()
self.dryRun = self.am_getOption( 'DryRun', self.dryRun )
self.autoAddUsers = self.am_getOption( 'AutoAddUsers', self.autoAddUsers )
self.autoModifyUsers = self.am_getOption( 'AutoModifyUsers', self.autoModifyUsers )
self.autoDeleteUsers = self.am_getOption( 'AutoDeleteUsers', self.autoDeleteUsers )
self.detailedReport = self.am_getOption( 'DetailedReport', self.detailedReport )
self.makeFCEntry = self.am_getOption( 'MakeHomeDirectory', self.makeFCEntry )
return S_OK()
def execute( self ):
self.__adminMsgs = {}
self.csapi.downloadCSData()
for vo in self.__voDict:
self.voChanged = False
voAdminUser = getVOOption( vo, "VOAdmin")
voAdminMail = None
if voAdminUser:
voAdminMail = getUserOption( voAdminUser, "Email")
voAdminGroup = getVOOption( vo, "VOAdminGroup", getVOOption( vo, "DefaultGroup" ) )
self.log.info( 'Performing VOMS sync for VO %s with credentials %s@%s' % ( vo, voAdminUser, voAdminGroup ) )
result = self.__syncCSWithVOMS( vo, proxyUserName = voAdminUser, proxyUserGroup = voAdminGroup ) #pylint: disable=unexpected-keyword-arg
if not result['OK']:
self.log.error( 'Failed to perform VOMS to CS synchronization:', 'VO %s: %s' % ( vo, result["Message"] ) )
continue
resultDict = result['Value']
newUsers = resultDict.get( "NewUsers", [] )
modUsers = resultDict.get( "ModifiedUsers", [] )
delUsers = resultDict.get( "DeletedUsers", [] )
self.log.info( "Run results: new users %d, modified users %d, deleted users %d" % \
( len( newUsers ), len( modUsers ), len( delUsers ) ) )
if self.csapi.csModified:
# We have accumulated all the changes, commit them now
self.log.info( "There are changes to the CS for vo %s ready to be committed" % vo )
if self.dryRun:
self.log.info( "Dry Run: CS won't be updated" )
self.csapi.showDiff()
else:
result = self.csapi.commitChanges()
if not result[ 'OK' ]:
self.log.error( "Could not commit configuration changes", result[ 'Message' ] )
return result
self.log.notice( "Configuration committed for VO %s" % vo )
else:
self.log.info( "No changes to the CS for VO %s recorded at this cycle" % vo )
# Add user home directory in the file catalog
if self.makeFCEntry and newUsers:
self.log.info( "Creating home directories for users %s" % str( newUsers ) )
result = self.__addHomeDirectory( vo, newUsers, proxyUserName = voAdminUser, proxyUserGroup = voAdminGroup ) #pylint: disable=unexpected-keyword-arg
if not result['OK']:
self.log.error( 'Failed to create user home directories:', 'VO %s: %s' % ( vo, result["Message"] ) )
else:
for user in result['Value']['Failed']:
self.log.error( "Failed to create home directory", "user: %s, operation: %s" % \
( user, result['Value']['Failed'][user] ) )
self.__adminMsgs[ 'Errors' ].append( "Failed to create home directory for user %s: operation %s" % \
( user, result['Value']['Failed'][user] ) )
for user in result['Value']['Successful']:
self.__adminMsgs[ 'Info' ].append( "Created home directory for user %s" % user )
if self.voChanged or self.detailedReport:
mailMsg = ""
if self.__adminMsgs[ 'Errors' ]:
mailMsg += "\nErrors list:\n %s" % "\n ".join( self.__adminMsgs[ 'Errors' ] )
if self.__adminMsgs[ 'Info' ]:
mailMsg += "\nRun result:\n %s" % "\n ".join( self.__adminMsgs[ 'Info' ] )
if self.detailedReport:
result = self.getVOUserReport( vo )
if result['OK']:
mailMsg += '\n\n'
mailMsg += result['Value']
else:
mailMsg += 'Failed to produce a detailed user report'
mailMsg += result['Message']
NotificationClient().sendMail( self.am_getOption( 'MailTo', voAdminMail ),
"VOMS2CSAgent run log", mailMsg,
self.am_getOption( 'MailFrom', self.am_getOption( 'mailFrom', "DIRAC system" ) ) )
return S_OK()
def getVOUserData( self, vo, refreshFlag = False ):
""" Get a report for users of a given VO
:param str vo: VO name
:return: S_OK/S_ERROR, Value = user description dictionary
"""
if refreshFlag:
gConfig.forceRefresh()
# Get DIRAC users
diracUsers = getUsersInVO( vo )
if not diracUsers:
return S_ERROR( "No VO users found for %s" % vo )
if refreshFlag:
result = self.csapi.downloadCSData()
if not result['OK']:
return result
result = self.csapi.describeUsers( diracUsers )
if not result['OK']:
self.log.error( 'Could not retrieve CS User description' )
return result
def getVOUserReport( self, vo ):
"""
:param str vo: VO name
:return: report string
"""
result = self.getVOUserData( vo, refreshFlag = True )
if not result['OK']:
return result
userDict = result['Value']
# Get DIRAC group vs VOMS Role Mappings
result = getVOMSRoleGroupMapping( vo )
if not result['OK']:
return result
diracVOMSMapping = result['Value']['DIRACVOMS']
records = []
groupDict = defaultdict( int )
multiDNUsers = {}
suspendedUsers = []
for user in userDict:
for group in userDict[user]['Groups']:
groupDict[group] += 1
dnList = fromChar( userDict[user]['DN'] )
if len( dnList ) > 1:
multiDNUsers[user] = dnList
if userDict[user].get( 'Status', 'Active' ) == 'Suspended':
suspendedUsers.append( user )
for group in diracVOMSMapping:
records.append( ( group, str( groupDict[group] ), diracVOMSMapping.get( group, '' ) ) )
fields = [ 'Group', 'Number of users', 'VOMS Role' ]
output = printTable( fields, records, sortField = 'Group', printOut = False, numbering = False )
if multiDNUsers:
output += '\nUsers with multiple DNs:\n'
for user in multiDNUsers:
output += ' %s:\n' % user
for dn in multiDNUsers[user]:
output += ' %s\n' % dn
if suspendedUsers:
output += '\n%d suspended users:\n' % len( suspendedUsers )
output += ' %s' % ','.join( suspendedUsers )
return S_OK( output )
@executeWithUserProxy
def __syncCSWithVOMS( self, vo ):
self.__adminMsgs = { 'Errors' : [], 'Info' : [] }
resultDict = defaultdict( list )
# Get DIRAC group vs VOMS Role Mappings
result = getVOMSRoleGroupMapping( vo )
if not result['OK']:
return result
vomsDIRACMapping = result['Value']['VOMSDIRAC']
diracVOMSMapping = result['Value']['DIRACVOMS']
noVOMSGroups = result['Value']['NoVOMS']
noSyncVOMSGroups = result['Value']['NoSyncVOMS']
vomsSrv = VOMSService( vo )
# Get VOMS VO name
result = vomsSrv.admGetVOName()
if not result['OK']:
self.log.error( 'Could not retrieve VOMS VO name', "for %s" % vo )
return result
vomsVOName = result[ 'Value' ].lstrip( '/' )
self.log.verbose( "VOMS VO Name for %s is %s" % ( vo, vomsVOName ) )
# Get VOMS users
result = vomsSrv.getUsers()
if not result['OK']:
self.log.error( 'Could not retrieve user information from VOMS', result['Message'] )
return result
vomsUserDict = result[ 'Value' ]
message = "There are %s user entries in VOMS for VO %s" % ( len( vomsUserDict ), vomsVOName )
self.__adminMsgs[ 'Info' ].append( message )
self.log.info( message )
# Get DIRAC users
result = self.getVOUserData( vo )
if not result['OK']:
return result
diracUserDict = result['Value']
self.__adminMsgs[ 'Info' ].append( "There are %s registered users in DIRAC for VO %s" % ( len( diracUserDict ), vo ) )
self.log.info( "There are %s registered users in DIRAC VO %s" % ( len( diracUserDict ), vo ) )
# Find new and obsoleted user DNs
existingDNs = []
obsoletedDNs = []
newDNs = []
for user in diracUserDict:
dn = diracUserDict[user]['DN']
# We can have users with more than one DN registered
dnList = fromChar( dn )
existingDNs.extend( dnList )
for dn in dnList:
if dn not in vomsUserDict:
obsoletedDNs.append( dn )
for dn in vomsUserDict:
if dn not in existingDNs:
newDNs.append( dn )
allDiracUsers = getAllUsers()
nonVOUserDict = {}
nonVOUsers = list( set( allDiracUsers ) - set( diracUserDict.keys() ) )
if nonVOUsers:
result = self.csapi.describeUsers( nonVOUsers )
if not result['OK']:
self.log.error( 'Could not retrieve CS User description' )
return result
nonVOUserDict = result['Value']
# Process users
defaultVOGroup = getVOOption( vo, "DefaultGroup", "%s_user" % vo )
newAddedUserDict = {}
for dn in vomsUserDict:
nickName = ''
newDNForExistingUser = ''
diracName = ''
if dn in existingDNs:
for user in diracUserDict:
if dn == diracUserDict[user]['DN']:
diracName = user
if dn in newDNs:
# Find if the DN is already registered in the DIRAC CS
for user in nonVOUserDict:
if dn == nonVOUserDict[user]['DN']:
diracName = user
# Check the nickName in the same VO to see if the user is already registered
# with another DN
result = vomsSrv.attGetUserNickname( dn, vomsUserDict[dn]['CA'] )
if result['OK']:
nickName = result['Value']
if nickName in diracUserDict or nickName in newAddedUserDict:
diracName = nickName
# This is a flag for adding the new DN to an already existing user
newDNForExistingUser = dn
# We have a real new user
if not diracName:
if nickName:
newDiracName = nickName
else:
newDiracName = getUserName( dn, vomsUserDict[dn]['mail'], vo )
# If the chosen user name exists already, append a distinguishing suffix
ind = 1
trialName = newDiracName
while newDiracName in allDiracUsers:
# We have a user with the same name but with a different DN
newDiracName = "%s_%d" % ( trialName, ind )
ind += 1
# We now have everything to add the new user
userDict = { "DN": dn, "CA": vomsUserDict[dn]['CA'], "Email": vomsUserDict[dn]['mail'] }
groupsWithRole = []
for role in vomsUserDict[dn]['Roles']:
fullRole = "/%s/%s" % ( vomsVOName, role )
groupList = vomsDIRACMapping.get( fullRole, [] )
for group in groupList:
if group not in noSyncVOMSGroups:
groupsWithRole.append( group )
userDict['Groups'] = list( set( groupsWithRole + [defaultVOGroup] ) )
message = "\n Added new user %s:\n" % newDiracName
for key in userDict:
message += " %s: %s\n" % ( key, str( userDict[key] ) )
self.__adminMsgs[ 'Info' ].append( message )
self.voChanged = True
if self.autoAddUsers:
self.log.info( "Adding new user %s: %s" % ( newDiracName, str( userDict ) ) )
result = self.csapi.modifyUser( newDiracName, userDict, createIfNonExistant = True )
if not result['OK']:
self.log.warn( 'Failed adding new user %s' % newDiracName )
resultDict['NewUsers'].append( newDiracName )
newAddedUserDict[newDiracName] = userDict
continue
# We have an already existing user
modified = False
userDict = { "DN": dn, "CA": vomsUserDict[dn]['CA'], "Email": vomsUserDict[dn]['mail'] }
if newDNForExistingUser:
userDict['DN'] = ','.join( [ dn, diracUserDict[diracName]['DN'] ] )
modified = True
existingGroups = diracUserDict.get( diracName, {} ).get( 'Groups', [] )
nonVOGroups = list( set( existingGroups ) - set( diracVOMSMapping.keys() ) )
groupsWithRole = []
for role in vomsUserDict[dn]['Roles']:
fullRole = "/%s/%s" % ( vomsVOName, role )
groupList = vomsDIRACMapping.get( fullRole, [] )
for group in groupList:
if group not in noSyncVOMSGroups:
groupsWithRole.append( group )
keepGroups = nonVOGroups + groupsWithRole + [defaultVOGroup]
for group in existingGroups:
if group in nonVOGroups:
continue
role = diracVOMSMapping.get( group, '' )
# Among already existing groups for the user keep those without a special VOMS Role
# because this membership is done by hand in the CS
if not "Role" in role:
keepGroups.append( group )
# Keep existing groups with no VOMS attribute if any
if group in noVOMSGroups:
keepGroups.append( group )
# Keep groups for which syncronization with VOMS is forbidden
if group in noSyncVOMSGroups:
keepGroups.append( group )
userDict['Groups'] = list( set( keepGroups ) )
# Merge together groups for the same user but different DNs
if diracName in newAddedUserDict:
otherGroups = newAddedUserDict[diracName].get( 'Groups', [] )
userDict['Groups'] = list( set( keepGroups + otherGroups ) )
modified = True
# Check if something changed before asking CSAPI to modify
if diracName in diracUserDict:
message = "\n Modified user %s:\n" % diracName
modMsg = ''
for key in userDict:
if key == "Groups":
addedGroups = set( userDict[key] ) - set( diracUserDict.get( diracName, {} ).get( key, [] ) )
removedGroups = set( diracUserDict.get( diracName, {} ).get( key, [] ) ) - set( userDict[key] )
if addedGroups:
modMsg += " Added to group(s) %s\n" % ','.join( addedGroups )
if removedGroups:
modMsg += " Removed from group(s) %s\n" % ','.join( removedGroups )
else:
oldValue = str( diracUserDict.get( diracName, {} ).get( key, '' ) )
if str( userDict[key] ) != oldValue:
modMsg += " %s: %s -> %s\n" % ( key, oldValue, str( userDict[key] ) )
if modMsg:
self.__adminMsgs[ 'Info' ].append( message + modMsg )
modified = True
if self.autoModifyUsers and modified:
result = self.csapi.modifyUser( diracName, userDict )
if result['OK'] and result['Value']:
self.log.info( "Modified user %s: %s" % ( diracName, str( userDict ) ) )
self.voChanged = True
resultDict['ModifiedUsers'].append( diracName )
# Check if there are potentially obsoleted users
oldUsers = set()
for user in diracUserDict:
dnSet = set( fromChar( diracUserDict[user]['DN'] ) )
if not dnSet.intersection( set( vomsUserDict.keys() ) ) and user not in nonVOUserDict:
for group in diracUserDict[user]['Groups']:
if group not in noVOMSGroups:
oldUsers.add( user )
# Check for obsoleted DNs
for user in diracUserDict:
dnSet = set( fromChar( diracUserDict[user]['DN'] ) )
for dn in dnSet:
if dn in obsoletedDNs and user not in oldUsers:
self.log.verbose( "Modified user %s: dropped DN %s" % ( user, dn ) )
if self.autoModifyUsers:
userDict = diracUserDict[user]
modDNSet = dnSet - set( [dn] )
if modDNSet:
userDict['DN'] = ','.join( modDNSet )
result = self.csapi.modifyUser( user, userDict )
if result['OK'] and result['Value']:
self.log.info( "Modified user %s: dropped DN %s" % ( user, dn ) )
self.__adminMsgs[ 'Info' ].append( "Modified user %s: dropped DN %s" % ( user, dn ) )
self.voChanged = True
resultDict['ModifiedUsers'].append( diracName )
else:
oldUsers.add( user )
if oldUsers:
self.voChanged = True
if self.autoDeleteUsers:
self.log.info( 'The following users will be deleted: %s' % str( oldUsers ) )
result = self.csapi.deleteUsers( oldUsers )
if result['OK']:
self.__adminMsgs[ 'Info' ].append( 'The following users are deleted from CS:\n %s\n' % str( oldUsers ) )
resultDict['DeletedUsers'] = oldUsers
else:
self.__adminMsgs[ 'Errors' ].append( 'Error in deleting users from CS:\n %s' % str( oldUsers ) )
self.log.error( 'Error while user deletion from CS', result )
else:
self.__adminMsgs[ 'Info' ].append( 'The following users to be checked for deletion:\n %s' % str( oldUsers ) )
self.log.info( 'The following users to be checked for deletion: %s' % str( oldUsers ) )
return S_OK( resultDict )
@executeWithUserProxy
def __addHomeDirectory( self, vo, newUsers ):
fc = FileCatalog( vo = vo )
defaultVOGroup = getVOOption( vo, "DefaultGroup", "%s_user" % vo )
failed = {}
successful = {}
for user in newUsers:
result = fc.addUser( user )
if not result['OK']:
failed[user] = "addUser"
continue
dirName = '/%s/user/%s/%s' % ( vo, user[0], user )
result = fc.createDirectory( dirName )
if not result['OK']:
failed[user] = "createDirectory"
continue
result = fc.changePathOwner( { dirName: user }, recursive = True )
if not result['OK']:
failed[user] = "changePathOwner"
continue
result = fc.changePathGroup( { dirName: defaultVOGroup }, recursive = True )
if not result['OK']:
failed[user] = "changePathGroup"
continue
successful[user] = True
return S_OK( { "Successful": successful, "Failed": failed } )
if not res['OK']:
gLogger.error(
"The provided site (%s) does not have an associated catalog." %
site)
continue
res = csAPI.setOption("%s/%s/Status" % (storageCFGBase, site), "InActive")
if not res['OK']:
gLogger.error("Failed to update %s catalog status to InActive" % site)
else:
gLogger.debug("Successfully updated %s catalog status to InActive" %
site)
banned.append(site)
if not banned:
gLogger.error("Failed to ban any catalog mirrors")
DIRAC.exit(-1)
res = csAPI.commitChanges()
if not res['OK']:
gLogger.error("Failed to commit changes to CS", res['Message'])
DIRAC.exit(-1)
subject = '%d catalog instance(s) banned for use' % len(banned)
address = gConfig.getValue('/Operations/EMail/Production', '*****@*****.**')
body = 'The catalog mirrors at the following sites were banned'
for site in banned:
body = "%s\n%s" % (body, site)
NotificationClient().sendMail(address, subject, body, '*****@*****.**' % userName)
DIRAC.exit(0)
siteName = result['Value']
gLogger.notice( 'Processing CondDB endpoint at site %s' % siteName )
csapi.copySection( '/Resources/CondDB/%s' % site,
'%s/Sites/%s/DBServer/CondDB' % (RESOURCES_NEW_SECTION,siteName) )
csapi.setOptionComment( '%s/Sites/%s/DBServer' % (RESOURCES_NEW_SECTION,siteName),
'Database server resource' )
csapi.setOptionComment( '%s/Sites/%s/DBServer/CondDB' % (RESOURCES_NEW_SECTION,siteName),
'Conditions database' )
if __name__ == '__main__':
if computingFlag:
result = convertSites()
if storageFlag:
result = convertSEs()
if catalogFlag:
result = convertCatalogs()
if transferFlag:
result = convertTransfers()
if dbFlag:
result = convertDBServers()
csapi.commitChanges()
print csapi.getCurrentCFG()['Value'].serialize()
class VOMS2CSAgent( AgentModule ):
def initialize( self ):
self.__voDict = {}
voNames = self.am_getOption( 'VO', [] )
if not voNames[0].lower() == "none":
if voNames[0].lower() == "any":
voNames = []
result = getVOMSVOs( voNames )
if not result['OK']:
return result
self.__voDict = result['Value']
self.__adminMsgs = {}
self.csapi = CSAPI()
self.voChanged = False
self.log.notice( "VOs: %s" % self.__voDict.keys() )
self.autoAddUsers = self.am_getOption( 'AutoAddUsers', False )
self.autoModifyUsers = self.am_getOption( 'AutoModifyUsers', False )
self.autoSuspendUsers = self.am_getOption( 'AutoSuspendUsers', False )
return S_OK()
def execute( self ):
self.__adminMsgs = {}
self.csapi.downloadCSData()
for vo in self.__voDict:
self.voChanged = False
voAdminUser = getVOOption( vo, "VOAdmin")
voAdminMail = None
if voAdminUser:
voAdminMail = getUserOption( voAdminUser, "Email")
voAdminGroup = getVOOption( vo, "VOAdminGroup", getVOOption( vo, "DefaultGroup" ) )
self.log.info( 'Performing VOMS sync for VO %s with credentials %s@%s' % ( vo, voAdminUser, voAdminGroup ) )
result = self.__syncCSWithVOMS( vo, proxyUserName = voAdminUser, proxyUserGroup = voAdminGroup ) #pylint: disable=unexpected-keyword-arg
if not result['OK']:
self.log.error( 'Failed to perform VOMS to CS synchronization:', 'VO %s: %s' % ( vo, result["Message"] ) )
continue
if self.voChanged:
mailMsg = ""
if self.__adminMsgs[ 'Errors' ]:
mailMsg += "\nErrors list:\n %s" % "\n ".join( self.__adminMsgs[ 'Errors' ] )
if self.__adminMsgs[ 'Info' ]:
mailMsg += "\nRun result:\n %s" % "\n ".join( self.__adminMsgs[ 'Info' ] )
NotificationClient().sendMail( self.am_getOption( 'MailTo', voAdminMail ),
"VOMS2CSAgent run log", mailMsg,
self.am_getOption( 'mailFrom', "DIRAC system" ) )
# We have accumulated all the changes, commit them now
result = self.csapi.commitChanges()
if not result[ 'OK' ]:
self.log.error( "Could not commit configuration changes", result[ 'Message' ] )
return result
self.log.info( "Configuration committed" )
return S_OK()
@executeWithUserProxy
def __syncCSWithVOMS( self, vo ):
self.__adminMsgs = { 'Errors' : [], 'Info' : [] }
# Get DIRAC group vs VOMS Role Mappings
result = getVOMSRoleGroupMapping( vo )
if not result['OK']:
return result
vomsDIRACMapping = result['Value']['VOMSDIRAC']
diracVOMSMapping = result['Value']['DIRACVOMS']
noVOMSGroups = result['Value']['NoVOMS']
vomsSrv = VOMSService( vo )
# Get VOMS VO name
result = vomsSrv.admGetVOName()
if not result['OK']:
self.log.error( 'Could not retrieve VOMS VO name', "for %s" % vo )
return result
vomsVOName = result[ 'Value' ].lstrip( '/' )
self.log.verbose( "VOMS VO Name for %s is %s" % ( vo, vomsVOName ) )
# Get VOMS users
result = vomsSrv.getUsers()
if not result['OK']:
self.log.error( 'Could not retrieve user information from VOMS', result['Message'] )
return result
vomsUserDict = result[ 'Value' ]
message = "There are %s registered users in VOMS VO %s" % ( len( vomsUserDict ), vomsVOName )
self.__adminMsgs[ 'Info' ].append( message )
self.log.info( message )
# Get DIRAC users
diracUsers = getUsersInVO( vo )
if not diracUsers:
return S_ERROR( "No VO users found for %s" % vo )
result = self.csapi.describeUsers( diracUsers )
if not result['OK']:
self.log.error( 'Could not retrieve CS User description' )
return result
diracUserDict = result['Value']
self.__adminMsgs[ 'Info' ].append( "There are %s registered users in DIRAC for VO %s" % ( len( diracUserDict ), vo ) )
self.log.info( "There are %s registered users in DIRAC VO %s" % ( len( diracUserDict ), vo ) )
# Find new and obsoleted user DNs
existingDNs = []
obsoletedDNs = []
newDNs = []
for user in diracUserDict:
dn = diracUserDict[user]['DN']
existingDNs.append( dn )
if dn not in vomsUserDict:
obsoletedDNs.append( dn )
for dn in vomsUserDict:
if dn not in existingDNs:
newDNs.append( dn )
allDiracUsers = getAllUsers()
nonVOusers = list( set( allDiracUsers ) - set(diracUsers) )
result = self.csapi.describeUsers( nonVOusers )
if not result['OK']:
self.log.error( 'Could not retrieve CS User description' )
return result
nonVOUserDict = result['Value']
# Process users
defaultVOGroup = getVOOption( vo, "DefaultGroup", "%s_user" % vo )
for dn in vomsUserDict:
if dn in newDNs:
# Find if the DN is already registered in the DIRAC CS
diracName = ''
for user in nonVOUserDict:
if dn == nonVOUserDict[user]['DN']:
diracName = user
# We have a real new user
if not diracName:
nickName = ''
result = vomsSrv.attGetUserNickname( dn, vomsUserDict[dn]['CA'] )
if result['OK']:
nickName = result['Value']
if nickName:
newDiracName = nickName
else:
newDiracName = getUserName( dn, vomsUserDict[dn]['mail'] )
ind = 1
trialName = newDiracName
while newDiracName in allDiracUsers:
# We have a user with the same name but with a different DN
newDiracName = "%s_%d" % ( trialName, ind )
ind += 1
# We now have everything to add the new user
userDict = { "DN": dn, "CA": vomsUserDict[dn]['CA'], "Email": vomsUserDict[dn]['mail'] }
groupsWithRole = []
for role in vomsUserDict[dn]['Roles']:
fullRole = "/%s/%s" % ( vomsVOName, role )
group = vomsDIRACMapping.get( fullRole )
if group:
groupsWithRole.append( group )
userDict['Groups'] = list( set( groupsWithRole + [defaultVOGroup] ) )
self.__adminMsgs[ 'Info' ].append( "Adding new user %s: %s" % ( newDiracName, str( userDict ) ) )
self.voChanged = True
if self.autoAddUsers:
self.log.info( "Adding new user %s: %s" % ( newDiracName, str( userDict ) ) )
result = self.csapi.modifyUser( newDiracName, userDict, createIfNonExistant = True )
if not result['OK']:
self.log.warn( 'Failed adding new user %s' % newDiracName )
continue
# We have an already existing user
userDict = { "DN": dn, "CA": vomsUserDict[dn]['CA'], "Email": vomsUserDict[dn]['mail'] }
nonVOGroups = nonVOUserDict.get( diracName, {} ).get( 'Groups', [] )
existingGroups = diracUserDict.get( diracName, {} ).get( 'Groups', [] )
groupsWithRole = []
for role in vomsUserDict[dn]['Roles']:
fullRole = "/%s/%s" % ( vomsVOName, role )
group = vomsDIRACMapping.get( fullRole )
if group:
groupsWithRole.append( group )
keepGroups = nonVOGroups + groupsWithRole + [defaultVOGroup]
for group in existingGroups:
role = diracVOMSMapping[group]
# Among already existing groups for the user keep those without a special VOMS Role
# because this membership is done by hand in the CS
if not "Role" in role:
keepGroups.append( group )
# Keep existing groups with no VOMS attribute if any
if group in noVOMSGroups:
keepGroups.append( group )
userDict['Groups'] = keepGroups
if self.autoModifyUsers:
result = self.csapi.modifyUser( diracName, userDict )
if result['OK'] and result['Value']:
self.voChanged = True
# Check if there are potentially obsoleted users
oldUsers = set()
for user in diracUserDict:
dn = diracUserDict[user]['DN']
if not dn in vomsUserDict and not user in nonVOUserDict:
for group in diracUserDict[user]['Groups']:
if not group in noVOMSGroups:
oldUsers.add( user )
if oldUsers:
self.voChanged = True
self.__adminMsgs[ 'Info' ].append( 'The following users to be checked for deletion: %s' % str( oldUsers ) )
self.log.info( 'The following users to be checked for deletion: %s' % str( oldUsers ) )
return S_OK()
class DiracAdmin( API ):
""" Administrative functionalities
"""
#############################################################################
def __init__( self ):
"""Internal initialization of the DIRAC Admin API.
"""
super( DiracAdmin, self ).__init__()
self.csAPI = CSAPI()
self.dbg = False
if gConfig.getValue( self.section + '/LogLevel', 'DEBUG' ) == 'DEBUG':
self.dbg = True
self.scratchDir = gConfig.getValue( self.section + '/ScratchDir', '/tmp' )
self.currentDir = os.getcwd()
#############################################################################
def uploadProxy( self, group ):
"""Upload a proxy to the DIRAC WMS. This method
Example usage:
>>> print diracAdmin.uploadProxy('lhcb_pilot')
{'OK': True, 'Value': 0L}
:param group: DIRAC Group
:type job: string
:return: S_OK,S_ERROR
:param permanent: Indefinitely update proxy
:type permanent: boolean
"""
return gProxyManager.uploadProxy( diracGroup = group )
#############################################################################
def setProxyPersistency( self, userDN, userGroup, persistent = True ):
"""Set the persistence of a proxy in the Proxy Manager
Example usage:
>>> print diracAdmin.setProxyPersistency( 'some DN', 'dirac group', True )
{'OK': True }
:param userDN: User DN
:type userDN: string
:param userGroup: DIRAC Group
:type userGroup: string
:param persistent: Persistent flag
:type persistent: boolean
:return: S_OK,S_ERROR
"""
return gProxyManager.setPersistency( userDN, userGroup, persistent )
#############################################################################
def checkProxyUploaded( self, userDN, userGroup, requiredTime ):
"""Set the persistence of a proxy in the Proxy Manager
Example usage:
>>> print diracAdmin.setProxyPersistency( 'some DN', 'dirac group', True )
{'OK': True, 'Value' : True/False }
:param userDN: User DN
:type userDN: string
:param userGroup: DIRAC Group
:type userGroup: string
:param requiredTime: Required life time of the uploaded proxy
:type requiredTime: boolean
:return: S_OK,S_ERROR
"""
return gProxyManager.userHasProxy( userDN, userGroup, requiredTime )
#############################################################################
def getSiteMask( self, printOutput = False ):
"""Retrieve current site mask from WMS Administrator service.
Example usage:
>>> print diracAdmin.getSiteMask()
{'OK': True, 'Value': 0L}
:return: S_OK,S_ERROR
"""
wmsAdmin = RPCClient( 'WorkloadManagement/WMSAdministrator' )
result = wmsAdmin.getSiteMask()
if result['OK']:
sites = result['Value']
if printOutput:
sites.sort()
for site in sites:
print site
return result
#############################################################################
def getBannedSites( self, gridType = [], printOutput = False ):
"""Retrieve current list of banned sites.
Example usage:
>>> print diracAdmin.getBannedSites()
{'OK': True, 'Value': []}
:return: S_OK,S_ERROR
"""
wmsAdmin = RPCClient( 'WorkloadManagement/WMSAdministrator' )
bannedSites = []
totalList = []
result = wmsAdmin.getSiteMask()
if not result['OK']:
self.log.warn( result['Message'] )
return result
sites = result['Value']
if not gridType:
result = gConfig.getSections( '/Resources/Sites' )
if not result['OK']:
return result
gridType = result['Value']
for grid in gridType:
result = gConfig.getSections( '/Resources/Sites/%s' % grid )
if not result['OK']:
return result
totalList += result['Value']
for site in totalList:
if not site in sites:
bannedSites.append( site )
bannedSites.sort()
if printOutput:
print '\n'.join( bannedSites )
return S_OK( bannedSites )
#############################################################################
def getSiteSection( self, site, printOutput = False ):
"""Simple utility to get the list of CEs for DIRAC site name.
Example usage:
>>> print diracAdmin.getSiteSection('LCG.CERN.ch')
{'OK': True, 'Value':}
:return: S_OK,S_ERROR
"""
gridType = site.split( '.' )[0]
if not gConfig.getSections( '/Resources/Sites/%s' % ( gridType ) )['OK']:
return S_ERROR( '/Resources/Sites/%s is not a valid site section' % ( gridType ) )
result = gConfig.getOptionsDict( '/Resources/Sites/%s/%s' % ( gridType, site ) )
if printOutput and result['OK']:
print self.pPrint.pformat( result['Value'] )
return result
#############################################################################
def addSiteInMask( self, site, comment, printOutput = False ):
"""Adds the site to the site mask.
Example usage:
>>> print diracAdmin.addSiteInMask()
{'OK': True, 'Value': }
:return: S_OK,S_ERROR
"""
result = self.__checkSiteIsValid( site )
if not result['OK']:
return result
mask = self.getSiteMask()
if not mask['OK']:
return mask
siteMask = mask['Value']
if site in siteMask:
return S_ERROR( 'Site %s already in mask of allowed sites' % site )
wmsAdmin = RPCClient( 'WorkloadManagement/WMSAdministrator' )
result = wmsAdmin.allowSite( site, comment )
if not result['OK']:
return result
if printOutput:
print 'Allowing %s in site mask' % site
return result
#############################################################################
def getSiteMaskLogging( self, site = None, printOutput = False ):
"""Retrieves site mask logging information.
Example usage:
>>> print diracAdmin.getSiteMaskLogging('LCG.AUVER.fr')
{'OK': True, 'Value': }
:return: S_OK,S_ERROR
"""
result = self.__checkSiteIsValid( site )
if not result['OK']:
return result
wmsAdmin = RPCClient( 'WorkloadManagement/WMSAdministrator' )
result = wmsAdmin.getSiteMaskLogging( site )
if not result['OK']:
return result
if site:
if not result['Value'].has_key( site ):
return S_ERROR( 'Site mask information not available for %s' % ( site ) )
if printOutput:
if site:
print '\nSite Mask Logging Info for %s\n' % site
else:
print '\nAll Site Mask Logging Info\n'
siteDict = result['Value']
for site, tupleList in siteDict.iteritems():
if not site:
print '\n===> %s\n' % site
for tup in tupleList:
print str( tup[0] ).ljust( 8 ) + str( tup[1] ).ljust( 20 ) + \
'( ' + str( tup[2] ).ljust( len( str( tup[2] ) ) ) + ' ) "' + str( tup[3] ) + '"'
print ' '
return result
#############################################################################
def banSiteFromMask( self, site, comment, printOutput = False ):
"""Removes the site from the site mask.
Example usage:
>>> print diracAdmin.banSiteFromMask()
{'OK': True, 'Value': }
:return: S_OK,S_ERROR
"""
result = self.__checkSiteIsValid( site )
if not result['OK']:
return result
mask = self.getSiteMask()
if not mask['OK']:
return mask
siteMask = mask['Value']
if not site in siteMask:
return S_ERROR( 'Site %s is already banned' % site )
wmsAdmin = RPCClient( 'WorkloadManagement/WMSAdministrator' )
result = wmsAdmin.banSite( site, comment )
if not result['OK']:
return result
if printOutput:
print 'Removing %s from site mask' % site
return result
#############################################################################
@classmethod
def __checkSiteIsValid( self, site ):
"""Internal function to check that a site name is valid.
"""
sites = getSiteCEMapping()
if not sites['OK']:
return S_ERROR( 'Could not get site CE mapping' )
siteList = sites['Value'].keys()
if not site in siteList:
return S_ERROR( 'Specified site %s is not in list of defined sites' % site )
return S_OK( '%s is valid' % site )
#############################################################################
def clearMask( self ):
"""Removes all sites from the site mask. Should be used with care.
Example usage:
>>> print diracAdmin.clearMask()
{'OK': True, 'Value':''}
:return: S_OK,S_ERROR
"""
wmsAdmin = RPCClient( 'WorkloadManagement/WMSAdministrator' )
result = wmsAdmin.clearMask()
return result
#############################################################################
def getServicePorts( self, setup = '', printOutput = False ):
"""Checks the service ports for the specified setup. If not given this is
taken from the current installation (/DIRAC/Setup)
Example usage:
>>> print diracAdmin.getServicePorts()
{'OK': True, 'Value':''}
:return: S_OK,S_ERROR
"""
if not setup:
setup = gConfig.getValue( '/DIRAC/Setup', '' )
setupList = gConfig.getSections( '/DIRAC/Setups', [] )
if not setupList['OK']:
return S_ERROR( 'Could not get /DIRAC/Setups sections' )
setupList = setupList['Value']
if not setup in setupList:
return S_ERROR( 'Setup %s is not in allowed list: %s' % ( setup, ', '.join( setupList ) ) )
serviceSetups = gConfig.getOptionsDict( '/DIRAC/Setups/%s' % setup )
if not serviceSetups['OK']:
return S_ERROR( 'Could not get /DIRAC/Setups/%s options' % setup )
serviceSetups = serviceSetups['Value'] # dict
systemList = gConfig.getSections( '/Systems' )
if not systemList['OK']:
return S_ERROR( 'Could not get Systems sections' )
systemList = systemList['Value']
result = {}
for system in systemList:
if serviceSetups.has_key( system ):
path = '/Systems/%s/%s/Services' % ( system, serviceSetups[system] )
servicesList = gConfig.getSections( path )
if not servicesList['OK']:
self.log.warn( 'Could not get sections in %s' % path )
else:
servicesList = servicesList['Value']
if not servicesList:
servicesList = []
self.log.verbose( 'System: %s ServicesList: %s' % ( system, ', '.join( servicesList ) ) )
for service in servicesList:
spath = '%s/%s/Port' % ( path, service )
servicePort = gConfig.getValue( spath, 0 )
if servicePort:
self.log.verbose( 'Found port for %s/%s = %s' % ( system, service, servicePort ) )
result['%s/%s' % ( system, service )] = servicePort
else:
self.log.warn( 'No port found for %s' % spath )
else:
self.log.warn( '%s is not defined in /DIRAC/Setups/%s' % ( system, setup ) )
if printOutput:
print self.pPrint.pformat( result )
return S_OK( result )
#############################################################################
def getProxy( self, userDN, userGroup, validity = 43200, limited = False ):
"""Retrieves a proxy with default 12hr validity and stores
this in a file in the local directory by default.
Example usage:
>>> print diracAdmin.getProxy()
{'OK': True, 'Value': }
:return: S_OK,S_ERROR
"""
return gProxyManager.downloadProxy( userDN, userGroup, limited = limited,
requiredTimeLeft = validity )
#############################################################################
def getVOMSProxy( self, userDN, userGroup, vomsAttr = False, validity = 43200, limited = False ):
"""Retrieves a proxy with default 12hr validity and VOMS extensions and stores
this in a file in the local directory by default.
Example usage:
>>> print diracAdmin.getVOMSProxy()
{'OK': True, 'Value': }
:return: S_OK,S_ERROR
"""
return gProxyManager.downloadVOMSProxy( userDN, userGroup, limited = limited,
requiredVOMSAttribute = vomsAttr,
requiredTimeLeft = validity )
#############################################################################
def getPilotProxy( self, userDN, userGroup, validity = 43200 ):
"""Retrieves a pilot proxy with default 12hr validity and stores
this in a file in the local directory by default.
Example usage:
>>> print diracAdmin.getVOMSProxy()
{'OK': True, 'Value': }
:return: S_OK,S_ERROR
"""
return gProxyManager.getPilotProxyFromDIRACGroup( userDN, userGroup, requiredTimeLeft = validity )
#############################################################################
def resetJob( self, jobID ):
"""Reset a job or list of jobs in the WMS. This operation resets the reschedule
counter for a job or list of jobs and allows them to run as new.
Example::
>>> print dirac.reset(12345)
{'OK': True, 'Value': [12345]}
:param job: JobID
:type job: integer or list of integers
:return: S_OK,S_ERROR
"""
if isinstance( jobID, basestring ):
try:
jobID = int( jobID )
except Exception as x:
return self._errorReport( str( x ), 'Expected integer or convertible integer for existing jobID' )
elif isinstance( jobID, list ):
try:
jobID = [int( job ) for job in jobID]
except Exception as x:
return self._errorReport( str( x ), 'Expected integer or convertible integer for existing jobIDs' )
jobManager = RPCClient( 'WorkloadManagement/JobManager', useCertificates = False )
result = jobManager.resetJob( jobID )
return result
#############################################################################
def getJobPilotOutput( self, jobID, directory = '' ):
"""Retrieve the pilot output for an existing job in the WMS.
The output will be retrieved in a local directory unless
otherwise specified.
>>> print dirac.getJobPilotOutput(12345)
{'OK': True, StdOut:'',StdError:''}
:param job: JobID
:type job: integer or string
:return: S_OK,S_ERROR
"""
if not directory:
directory = self.currentDir
if not os.path.exists( directory ):
return self._errorReport( 'Directory %s does not exist' % directory )
wmsAdmin = RPCClient( 'WorkloadManagement/WMSAdministrator' )
result = wmsAdmin.getJobPilotOutput( jobID )
if not result['OK']:
return result
outputPath = '%s/pilot_%s' % ( directory, jobID )
if os.path.exists( outputPath ):
self.log.info( 'Remove %s and retry to continue' % outputPath )
return S_ERROR( 'Remove %s and retry to continue' % outputPath )
if not os.path.exists( outputPath ):
self.log.verbose( 'Creating directory %s' % outputPath )
os.mkdir( outputPath )
outputs = result['Value']
if outputs.has_key( 'StdOut' ):
stdout = '%s/std.out' % ( outputPath )
with open( stdout, 'w' ) as fopen:
fopen.write( outputs['StdOut'] )
self.log.verbose( 'Standard output written to %s' % ( stdout ) )
else:
self.log.warn( 'No standard output returned' )
if outputs.has_key( 'StdError' ):
stderr = '%s/std.err' % ( outputPath )
with open( stderr, 'w' ) as fopen:
fopen.write( outputs['StdError'] )
self.log.verbose( 'Standard error written to %s' % ( stderr ) )
else:
self.log.warn( 'No standard error returned' )
self.log.always( 'Outputs retrieved in %s' % outputPath )
return result
#############################################################################
def getPilotOutput( self, gridReference, directory = '' ):
"""Retrieve the pilot output (std.out and std.err) for an existing job in the WMS.
>>> print dirac.getJobPilotOutput(12345)
{'OK': True, 'Value': {}}
:param job: JobID
:type job: integer or string
:return: S_OK,S_ERROR
"""
if not isinstance( gridReference, basestring ):
return self._errorReport( 'Expected string for pilot reference' )
if not directory:
directory = self.currentDir
if not os.path.exists( directory ):
return self._errorReport( 'Directory %s does not exist' % directory )
wmsAdmin = RPCClient( 'WorkloadManagement/WMSAdministrator' )
result = wmsAdmin.getPilotOutput( gridReference )
if not result['OK']:
return result
gridReferenceSmall = gridReference.split( '/' )[-1]
if not gridReferenceSmall:
gridReferenceSmall = 'reference'
outputPath = '%s/pilot_%s' % ( directory, gridReferenceSmall )
if os.path.exists( outputPath ):
self.log.info( 'Remove %s and retry to continue' % outputPath )
return S_ERROR( 'Remove %s and retry to continue' % outputPath )
if not os.path.exists( outputPath ):
self.log.verbose( 'Creating directory %s' % outputPath )
os.mkdir( outputPath )
outputs = result['Value']
if outputs.has_key( 'StdOut' ):
stdout = '%s/std.out' % ( outputPath )
with open( stdout, 'w' ) as fopen:
fopen.write( outputs['StdOut'] )
self.log.info( 'Standard output written to %s' % ( stdout ) )
else:
self.log.warn( 'No standard output returned' )
if outputs.has_key( 'StdErr' ):
stderr = '%s/std.err' % ( outputPath )
with open( stderr, 'w' ) as fopen:
fopen.write( outputs['StdErr'] )
self.log.info( 'Standard error written to %s' % ( stderr ) )
else:
self.log.warn( 'No standard error returned' )
self.log.always( 'Outputs retrieved in %s' % outputPath )
return result
#############################################################################
def getPilotInfo( self, gridReference ):
"""Retrieve info relative to a pilot reference
>>> print dirac.getPilotInfo(12345)
{'OK': True, 'Value': {}}
:param gridReference: Pilot Job Reference
:type gridReference: string
:return: S_OK,S_ERROR
"""
if not isinstance( gridReference, basestring ):
return self._errorReport( 'Expected string for pilot reference' )
wmsAdmin = RPCClient( 'WorkloadManagement/WMSAdministrator' )
result = wmsAdmin.getPilotInfo( gridReference )
return result
#############################################################################
def killPilot( self, gridReference ):
"""Kill the pilot specified
>>> print dirac.getPilotInfo(12345)
{'OK': True, 'Value': {}}
:param gridReference: Pilot Job Reference
:return: S_OK,S_ERROR
"""
if not isinstance( gridReference, basestring ):
return self._errorReport( 'Expected string for pilot reference' )
wmsAdmin = RPCClient( 'WorkloadManagement/WMSAdministrator' )
result = wmsAdmin.killPilot( gridReference )
return result
#############################################################################
def getPilotLoggingInfo( self, gridReference ):
"""Retrieve the pilot logging info for an existing job in the WMS.
>>> print dirac.getPilotLoggingInfo(12345)
{'OK': True, 'Value': {"The output of the command"}}
:param gridReference: Gridp pilot job reference Id
:type gridReference: string
:return: S_OK,S_ERROR
"""
if type( gridReference ) not in types.StringTypes:
return self._errorReport( 'Expected string for pilot reference' )
wmsAdmin = RPCClient( 'WorkloadManagement/WMSAdministrator' )
return wmsAdmin.getPilotLoggingInfo( gridReference )
#############################################################################
def getJobPilots( self, jobID ):
"""Extract the list of submitted pilots and their status for a given
jobID from the WMS. Useful information is printed to the screen.
>>> print dirac.getJobPilots()
{'OK': True, 'Value': {PilotID:{StatusDict}}}
:param job: JobID
:type job: integer or string
:return: S_OK,S_ERROR
"""
if isinstance( jobID, basestring ):
try:
jobID = int( jobID )
except Exception as x:
return self._errorReport( str( x ), 'Expected integer or string for existing jobID' )
wmsAdmin = RPCClient( 'WorkloadManagement/WMSAdministrator' )
result = wmsAdmin.getPilots( jobID )
if result['OK']:
print self.pPrint.pformat( result['Value'] )
return result
#############################################################################
def getPilotSummary( self, startDate = '', endDate = '' ):
"""Retrieve the pilot output for an existing job in the WMS. Summary is
printed at INFO level, full dictionary of results also returned.
>>> print dirac.getPilotSummary()
{'OK': True, 'Value': {CE:{Status:Count}}}
:param job: JobID
:type job: integer or string
:return: S_OK,S_ERROR
"""
wmsAdmin = RPCClient( 'WorkloadManagement/WMSAdministrator' )
result = wmsAdmin.getPilotSummary( startDate, endDate )
if not result['OK']:
return result
ceDict = result['Value']
headers = 'CE'.ljust( 28 )
i = 0
for ce, summary in ceDict.iteritems():
states = summary.keys()
if len( states ) > i:
i = len( states )
for i in xrange( i ):
headers += 'Status'.ljust( 12 ) + 'Count'.ljust( 12 )
print headers
for ce, summary in ceDict.iteritems():
line = ce.ljust( 28 )
states = summary.keys()
states.sort()
for state in states:
count = str( summary[state] )
line += state.ljust( 12 ) + count.ljust( 12 )
print line
return result
#############################################################################
def selectRequests( self, jobID = None, requestID = None, requestName = None,
requestType = None, status = None, operation = None, ownerDN = None,
ownerGroup = None, requestStart = 0, limit = 100, printOutput = False ):
"""Select requests from the request management system. A few notes on the selection criteria:
- jobID is the WMS JobID for the request (if applicable)
- requestID is assigned during submission of the request
- requestName is the corresponding XML file name
- requestType e.g. 'transfer'
- status e.g. Done
- operation e.g. replicateAndRegister
- requestStart e.g. the first request to consider (start from 0 by default)
- limit e.g. selection limit (default 100)
>>> dirac.selectRequests(jobID='4894')
{'OK': True, 'Value': [[<Requests>]]}
"""
options = {'RequestID':requestID, 'RequestName':requestName, 'JobID':jobID, 'OwnerDN':ownerDN,
'OwnerGroup':ownerGroup, 'RequestType':requestType, 'Status':status, 'Operation':operation}
conditions = {}
for key, value in options.iteritems():
if value:
try:
conditions[key] = str( value )
except Exception as x:
return self._errorReport( str( x ), 'Expected string for %s field' % key )
try:
requestStart = int( requestStart )
limit = int( limit )
except Exception as x:
return self._errorReport( str( x ), 'Expected integer for %s field' % limit )
self.log.verbose( 'Will select requests with the following conditions' )
self.log.verbose( self.pPrint.pformat( conditions ) )
requestClient = RPCClient( "RequestManagement/centralURL" )
result = requestClient.getRequestSummaryWeb( conditions, [], requestStart, limit )
if not result['OK']:
self.log.warn( result['Message'] )
return result
requestIDs = result['Value']
conds = []
for key, value in conditions.iteritems():
if value:
conds.append( '%s = %s' % ( key, value ) )
self.log.verbose( '%s request(s) selected with conditions %s and limit %s' % ( len( requestIDs['Records'] ),
', '.join( conds ), limit ) )
if printOutput:
requests = []
if len( requestIDs['Records'] ) > limit:
requestList = requestIDs['Records']
requests = requestList[:limit]
else:
requests = requestIDs['Records']
print '%s request(s) selected with conditions %s and limit %s' % ( len( requestIDs['Records'] ),
', '.join( conds ), limit )
print requestIDs['ParameterNames']
for request in requests:
print request
if not requestIDs:
return S_ERROR( 'No requests selected for conditions: %s' % conditions )
else:
return result
#############################################################################
def getRequestSummary( self, printOutput = False ):
"""
Get a summary of the requests in the request DB.
"""
requestClient = RPCClient( "RequestManagement/centralURL", timeout = 120 )
result = requestClient.getDBSummary()
if not result['OK']:
self.log.warn( result['Message'] )
return result
if printOutput:
print self.pPrint.pformat( result['Value'] )
return result
#############################################################################
def getExternalPackageVersions( self ):
"""
Simple function that attempts to obtain the external versions for
the local DIRAC installation (frequently needed for debugging purposes).
"""
gLogger.info( 'DIRAC version v%dr%d build %d' % ( DIRAC.majorVersion, DIRAC.minorVersion, DIRAC.patchLevel ) )
try:
import lcg_util
infoStr = 'Using lcg_util from: \n%s' % lcg_util.__file__
gLogger.info( infoStr )
infoStr = "The version of lcg_utils is %s" % lcg_util.lcg_util_version()
gLogger.info( infoStr )
except Exception as x:
errStr = "SRM2Storage.__init__: Failed to import lcg_util: %s" % ( x )
gLogger.exception( errStr )
try:
import gfalthr as gfal
infoStr = "Using gfalthr from: \n%s" % gfal.__file__
gLogger.info( infoStr )
infoStr = "The version of gfalthr is %s" % gfal.gfal_version()
gLogger.info( infoStr )
except Exception as x:
errStr = "SRM2Storage.__init__: Failed to import gfalthr: %s." % ( x )
gLogger.warn( errStr )
try:
import gfal
infoStr = "Using gfal from: %s" % gfal.__file__
gLogger.info( infoStr )
infoStr = "The version of gfal is %s" % gfal.gfal_version()
gLogger.info( infoStr )
except Exception as x:
errStr = "SRM2Storage.__init__: Failed to import gfal: %s" % ( x )
gLogger.exception( errStr )
defaultProtocols = gConfig.getValue( '/Resources/StorageElements/DefaultProtocols', [] )
gLogger.info( 'Default list of protocols are: %s' % ( ', '.join( defaultProtocols ) ) )
return S_OK()
#############################################################################
def getSiteProtocols( self, site, printOutput = False ):
"""
Allows to check the defined protocols for each site SE.
"""
result = self.__checkSiteIsValid( site )
if not result['OK']:
return result
siteSection = '/Resources/Sites/%s/%s/SE' % ( site.split( '.' )[0], site )
siteSEs = gConfig.getValue( siteSection, [] )
if not siteSEs:
return S_ERROR( 'No SEs found for site %s in section %s' % ( site, siteSection ) )
defaultProtocols = gConfig.getValue( '/Resources/StorageElements/DefaultProtocols', [] )
self.log.verbose( 'Default list of protocols are' ', '.join( defaultProtocols ) )
seInfo = {}
siteSEs.sort()
for se in siteSEs:
sections = gConfig.getSections( '/Resources/StorageElements/%s/' % ( se ) )
if not sections['OK']:
return sections
for section in sections['Value']:
if gConfig.getValue( '/Resources/StorageElements/%s/%s/ProtocolName' % ( se, section ), '' ) == 'SRM2':
path = '/Resources/StorageElements/%s/%s/ProtocolsList' % ( se, section )
seProtocols = gConfig.getValue( path, [] )
if not seProtocols:
seProtocols = defaultProtocols
seInfo[se] = seProtocols
if printOutput:
print '\nSummary of protocols for StorageElements at site %s' % site
print '\nStorageElement'.ljust( 30 ) + 'ProtocolsList'.ljust( 30 ) + '\n'
for se, protocols in seInfo.iteritems():
print se.ljust( 30 ) + ', '.join( protocols ).ljust( 30 )
return S_OK( seInfo )
#############################################################################
def setSiteProtocols( self, site, protocolsList, printOutput = False ):
"""
Allows to set the defined protocols for each SE for a given site.
"""
result = self.__checkSiteIsValid( site )
if not result['OK']:
return result
siteSection = '/Resources/Sites/%s/%s/SE' % ( site.split( '.' )[0], site )
siteSEs = gConfig.getValue( siteSection, [] )
if not siteSEs:
return S_ERROR( 'No SEs found for site %s in section %s' % ( site, siteSection ) )
defaultProtocols = gConfig.getValue( '/Resources/StorageElements/DefaultProtocols', [] )
self.log.verbose( 'Default list of protocols are', ', '.join( defaultProtocols ) )
for protocol in protocolsList:
if not protocol in defaultProtocols:
return S_ERROR( 'Requested to set protocol %s in list but %s is not '
'in default list of protocols:\n%s' % ( protocol, protocol, ', '.join( defaultProtocols ) ) )
modifiedCS = False
result = promptUser( 'Do you want to add the following default protocols:'
' %s for SE(s):\n%s' % ( ', '.join( protocolsList ), ', '.join( siteSEs ) ) )
if not result['OK']:
return result
if result['Value'].lower() != 'y':
self.log.always( 'No protocols will be added' )
return S_OK()
for se in siteSEs:
sections = gConfig.getSections( '/Resources/StorageElements/%s/' % ( se ) )
if not sections['OK']:
return sections
for section in sections['Value']:
if gConfig.getValue( '/Resources/StorageElements/%s/%s/ProtocolName' % ( se, section ), '' ) == 'SRM2':
path = '/Resources/StorageElements/%s/%s/ProtocolsList' % ( se, section )
self.log.verbose( 'Setting %s to %s' % ( path, ', '.join( protocolsList ) ) )
result = self.csSetOption( path, ', '.join( protocolsList ) )
if not result['OK']:
return result
modifiedCS = True
if modifiedCS:
result = self.csCommitChanges( False )
if not result[ 'OK' ]:
return S_ERROR( 'CS Commit failed with message = %s' % ( result[ 'Message' ] ) )
else:
if printOutput:
print 'Successfully committed changes to CS'
else:
if printOutput:
print 'No modifications to CS required'
return S_OK()
#############################################################################
def csSetOption( self, optionPath, optionValue ):
"""
Function to modify an existing value in the CS.
"""
return self.csAPI.setOption( optionPath, optionValue )
#############################################################################
def csSetOptionComment( self, optionPath, comment ):
"""
Function to modify an existing value in the CS.
"""
return self.csAPI.setOptionComment( optionPath, comment )
#############################################################################
def csModifyValue( self, optionPath, newValue ):
"""
Function to modify an existing value in the CS.
"""
return self.csAPI.modifyValue( optionPath, newValue )
#############################################################################
def csRegisterUser( self, username, properties ):
"""
Registers a user in the CS.
- username: Username of the user (easy;)
- properties: Dict containing:
- DN
- groups : list/tuple of groups the user belongs to
- <others> : More properties of the user, like mail
"""
return self.csAPI.addUser( username, properties )
#############################################################################
def csDeleteUser( self, user ):
"""
Deletes a user from the CS. Can take a list of users
"""
return self.csAPI.deleteUsers( user )
#############################################################################
def csModifyUser( self, username, properties, createIfNonExistant = False ):
"""
Modify a user in the CS. Takes the same params as in addUser and
applies the changes
"""
return self.csAPI.modifyUser( username, properties, createIfNonExistant )
#############################################################################
def csListUsers( self, group = False ):
"""
Lists the users in the CS. If no group is specified return all users.
"""
return self.csAPI.listUsers( group )
#############################################################################
def csDescribeUsers( self, mask = False ):
"""
List users and their properties in the CS.
If a mask is given, only users in the mask will be returned
"""
return self.csAPI.describeUsers( mask )
#############################################################################
def csModifyGroup( self, groupname, properties, createIfNonExistant = False ):
"""
Modify a user in the CS. Takes the same params as in addGroup and applies
the changes
"""
return self.csAPI.modifyGroup( groupname, properties, createIfNonExistant )
#############################################################################
def csListHosts( self ):
"""
Lists the hosts in the CS
"""
return self.csAPI.listHosts()
#############################################################################
def csDescribeHosts( self, mask = False ):
"""
Gets extended info for the hosts in the CS
"""
return self.csAPI.describeHosts( mask )
#############################################################################
def csModifyHost( self, hostname, properties, createIfNonExistant = False ):
"""
Modify a host in the CS. Takes the same params as in addHost and applies
the changes
"""
return self.csAPI.modifyHost( hostname, properties, createIfNonExistant )
#############################################################################
def csListGroups( self ):
"""
Lists groups in the CS
"""
return self.csAPI.listGroups()
#############################################################################
def csDescribeGroups( self, mask = False ):
"""
List groups and their properties in the CS.
If a mask is given, only groups in the mask will be returned
"""
return self.csAPI.describeGroups( mask )
#############################################################################
def csSyncUsersWithCFG( self, usersCFG ):
"""
Synchronize users in cfg with its contents
"""
return self.csAPI.syncUsersWithCFG( usersCFG )
#############################################################################
def csCommitChanges( self, sortUsers = True ):
"""
Commit the changes in the CS
"""
return self.csAPI.commitChanges( sortUsers = False )
#############################################################################
def sendMail( self, address, subject, body, fromAddress = None, localAttempt = True, html = False ):
"""
Send mail to specified address with body.
"""
notification = NotificationClient()
return notification.sendMail( address, subject, body, fromAddress, localAttempt, html )
#############################################################################
def sendSMS( self, userName, body, fromAddress = None ):
"""
Send mail to specified address with body.
"""
if len( body ) > 160:
return S_ERROR( 'Exceeded maximum SMS length of 160 characters' )
notification = NotificationClient()
return notification.sendSMS( userName, body, fromAddress )
#############################################################################
def getBDIISite( self, site, host = None ):
"""
Get information about site from BDII at host
"""
return ldapSite( site, host = host )
#############################################################################
def getBDIICluster( self, ce, host = None ):
"""
Get information about ce from BDII at host
"""
return ldapCluster( ce, host = host )
#############################################################################
def getBDIICE( self, ce, host = None ):
"""
Get information about ce from BDII at host
"""
return ldapCE( ce, host = host )
#############################################################################
def getBDIIService( self, ce, host = None ):
"""
Get information about ce from BDII at host
"""
return ldapService( ce, host = host )
#############################################################################
def getBDIICEState( self, ce, useVO = voName, host = None ):
"""
Get information about ce state from BDII at host
"""
return ldapCEState( ce, useVO, host = host )
#############################################################################
def getBDIICEVOView( self, ce, useVO = voName, host = None ):
"""
Get information about ce voview from BDII at host
"""
return ldapCEVOView( ce, useVO, host = host )
#############################################################################
def getBDIISE( self, site, useVO = voName, host = None ):
"""
Get information about SA from BDII at host
"""
return ldapSE( site, useVO, host = host )
class VOMS2CSAgent(AgentModule):
def __init__(self, *args, **kwargs):
""" Defines default parameters
"""
super(VOMS2CSAgent, self).__init__(*args, **kwargs)
self.__voDict = {}
self.__adminMsgs = {}
self.csapi = None
self.voChanged = False
self.dryRun = False
self.autoAddUsers = False
self.autoModifyUsers = False
def initialize(self):
""" Initialize the default parameters
"""
voNames = self.am_getOption('VO', [])
if not voNames[0].lower() == "none":
if voNames[0].lower() == "any":
voNames = []
result = getVOMSVOs(voNames)
if not result['OK']:
return result
self.__voDict = result['Value']
self.log.notice("VOs: %s" % self.__voDict.keys())
self.csapi = CSAPI()
self.dryRun = self.am_getOption('DryRun', self.dryRun)
self.autoAddUsers = self.am_getOption('AutoAddUsers',
self.autoAddUsers)
self.autoModifyUsers = self.am_getOption('AutoModifyUsers',
self.autoModifyUsers)
return S_OK()
def execute(self):
self.__adminMsgs = {}
self.csapi.downloadCSData()
for vo in self.__voDict:
self.voChanged = False
voAdminUser = getVOOption(vo, "VOAdmin")
voAdminMail = None
if voAdminUser:
voAdminMail = getUserOption(voAdminUser, "Email")
voAdminGroup = getVOOption(vo, "VOAdminGroup",
getVOOption(vo, "DefaultGroup"))
self.log.info(
'Performing VOMS sync for VO %s with credentials %s@%s' %
(vo, voAdminUser, voAdminGroup))
result = self.__syncCSWithVOMS(vo,
proxyUserName=voAdminUser,
proxyUserGroup=voAdminGroup) #pylint: disable=unexpected-keyword-arg
if not result['OK']:
self.log.error('Failed to perform VOMS to CS synchronization:',
'VO %s: %s' % (vo, result["Message"]))
continue
if self.voChanged:
mailMsg = ""
if self.__adminMsgs['Errors']:
mailMsg += "\nErrors list:\n %s" % "\n ".join(
self.__adminMsgs['Errors'])
if self.__adminMsgs['Info']:
mailMsg += "\nRun result:\n %s" % "\n ".join(
self.__adminMsgs['Info'])
NotificationClient().sendMail(
self.am_getOption('MailTo', voAdminMail),
"VOMS2CSAgent run log", mailMsg,
self.am_getOption('mailFrom', "DIRAC system"))
if self.csapi.csModified:
# We have accumulated all the changes, commit them now
self.log.info("There are changes to the CS ready to be committed")
if self.dryRun:
self.log.info("Dry Run: CS won't be updated")
self.csapi.showDiff()
else:
result = self.csapi.commitChanges()
if not result['OK']:
self.log.error("Could not commit configuration changes",
result['Message'])
return result
self.log.notice("Configuration committed")
else:
self.log.info("No changes to the CS recorded at this cycle")
return S_OK()
@executeWithUserProxy
def __syncCSWithVOMS(self, vo):
self.__adminMsgs = {'Errors': [], 'Info': []}
# Get DIRAC group vs VOMS Role Mappings
result = getVOMSRoleGroupMapping(vo)
if not result['OK']:
return result
vomsDIRACMapping = result['Value']['VOMSDIRAC']
diracVOMSMapping = result['Value']['DIRACVOMS']
noVOMSGroups = result['Value']['NoVOMS']
vomsSrv = VOMSService(vo)
# Get VOMS VO name
result = vomsSrv.admGetVOName()
if not result['OK']:
self.log.error('Could not retrieve VOMS VO name', "for %s" % vo)
return result
vomsVOName = result['Value'].lstrip('/')
self.log.verbose("VOMS VO Name for %s is %s" % (vo, vomsVOName))
# Get VOMS users
result = vomsSrv.getUsers()
if not result['OK']:
self.log.error('Could not retrieve user information from VOMS',
result['Message'])
return result
vomsUserDict = result['Value']
message = "There are %s registered users in VOMS VO %s" % (
len(vomsUserDict), vomsVOName)
self.__adminMsgs['Info'].append(message)
self.log.info(message)
# Get DIRAC users
diracUsers = getUsersInVO(vo)
if not diracUsers:
return S_ERROR("No VO users found for %s" % vo)
result = self.csapi.describeUsers(diracUsers)
if not result['OK']:
self.log.error('Could not retrieve CS User description')
return result
diracUserDict = result['Value']
self.__adminMsgs['Info'].append(
"There are %s registered users in DIRAC for VO %s" %
(len(diracUserDict), vo))
self.log.info("There are %s registered users in DIRAC VO %s" %
(len(diracUserDict), vo))
# Find new and obsoleted user DNs
existingDNs = []
obsoletedDNs = []
newDNs = []
for user in diracUserDict:
dn = diracUserDict[user]['DN']
existingDNs.append(dn)
if dn not in vomsUserDict:
obsoletedDNs.append(dn)
for dn in vomsUserDict:
if dn not in existingDNs:
newDNs.append(dn)
allDiracUsers = getAllUsers()
nonVOusers = list(set(allDiracUsers) - set(diracUsers))
result = self.csapi.describeUsers(nonVOusers)
if not result['OK']:
self.log.error('Could not retrieve CS User description')
return result
nonVOUserDict = result['Value']
# Process users
defaultVOGroup = getVOOption(vo, "DefaultGroup", "%s_user" % vo)
for dn in vomsUserDict:
if dn in newDNs:
# Find if the DN is already registered in the DIRAC CS
diracName = ''
for user in nonVOUserDict:
if dn == nonVOUserDict[user]['DN']:
diracName = user
# We have a real new user
if not diracName:
nickName = ''
result = vomsSrv.attGetUserNickname(
dn, vomsUserDict[dn]['CA'])
if result['OK']:
nickName = result['Value']
if nickName:
newDiracName = nickName
else:
newDiracName = getUserName(dn,
vomsUserDict[dn]['mail'])
ind = 1
trialName = newDiracName
while newDiracName in allDiracUsers:
# We have a user with the same name but with a different DN
newDiracName = "%s_%d" % (trialName, ind)
ind += 1
# We now have everything to add the new user
userDict = {
"DN": dn,
"CA": vomsUserDict[dn]['CA'],
"Email": vomsUserDict[dn]['mail']
}
groupsWithRole = []
for role in vomsUserDict[dn]['Roles']:
fullRole = "/%s/%s" % (vomsVOName, role)
group = vomsDIRACMapping.get(fullRole)
if group:
groupsWithRole.extend(group)
userDict['Groups'] = list(
set(groupsWithRole + [defaultVOGroup]))
self.__adminMsgs['Info'].append(
"Adding new user %s: %s" %
(newDiracName, str(userDict)))
self.voChanged = True
if self.autoAddUsers:
self.log.info("Adding new user %s: %s" %
(newDiracName, str(userDict)))
result = self.csapi.modifyUser(
newDiracName, userDict, createIfNonExistant=True)
if not result['OK']:
self.log.warn('Failed adding new user %s' %
newDiracName)
continue
# We have an already existing user
userDict = {
"DN": dn,
"CA": vomsUserDict[dn]['CA'],
"Email": vomsUserDict[dn]['mail']
}
nonVOGroups = nonVOUserDict.get(diracName,
{}).get('Groups', [])
existingGroups = diracUserDict.get(diracName,
{}).get('Groups', [])
groupsWithRole = []
for role in vomsUserDict[dn]['Roles']:
fullRole = "/%s/%s" % (vomsVOName, role)
group = vomsDIRACMapping.get(fullRole)
if group:
groupsWithRole.extend(group)
keepGroups = nonVOGroups + groupsWithRole + [defaultVOGroup]
for group in existingGroups:
role = diracVOMSMapping[group]
# Among already existing groups for the user keep those without a special VOMS Role
# because this membership is done by hand in the CS
if not "Role" in role:
keepGroups.append(group)
# Keep existing groups with no VOMS attribute if any
if group in noVOMSGroups:
keepGroups.append(group)
userDict['Groups'] = keepGroups
if self.autoModifyUsers:
result = self.csapi.modifyUser(diracName, userDict)
if result['OK'] and result['Value']:
self.voChanged = True
# Check if there are potentially obsoleted users
oldUsers = set()
for user in diracUserDict:
dn = diracUserDict[user]['DN']
if not dn in vomsUserDict and not user in nonVOUserDict:
for group in diracUserDict[user]['Groups']:
if not group in noVOMSGroups:
oldUsers.add(user)
if oldUsers:
self.voChanged = True
self.__adminMsgs['Info'].append(
'The following users to be checked for deletion: %s' %
str(oldUsers))
self.log.info(
'The following users to be checked for deletion: %s' %
str(oldUsers))
return S_OK()
for site in sites:
result = getSiteName(site)
siteName = result['Value']
gLogger.notice('Processing CondDB endpoint at site %s' % siteName)
csapi.copySection(
'/Resources/CondDB/%s' % site,
'%s/Sites/%s/DBServer/CondDB' % (RESOURCES_NEW_SECTION, siteName))
csapi.setOptionComment(
'%s/Sites/%s/DBServer' % (RESOURCES_NEW_SECTION, siteName),
'Database server resource')
csapi.setOptionComment(
'%s/Sites/%s/DBServer/CondDB' % (RESOURCES_NEW_SECTION, siteName),
'Conditions database')
if __name__ == '__main__':
if computingFlag:
result = convertSites()
if storageFlag:
result = convertSEs()
if catalogFlag:
result = convertCatalogs()
if transferFlag:
result = convertTransfers()
if dbFlag:
result = convertDBServers()
csapi.commitChanges()
print csapi.getCurrentCFG()['Value'].serialize()