class DiracAdmin(API):
""" Administrative functionalities
"""
#############################################################################
def __init__(self):
"""Internal initialization of the DIRAC Admin API.
"""
super(DiracAdmin, self).__init__()
self.csAPI = CSAPI()
self.dbg = False
if gConfig.getValue(self.section + '/LogLevel', 'DEBUG') == 'DEBUG':
self.dbg = True
self.scratchDir = gConfig.getValue(self.section + '/ScratchDir',
'/tmp')
self.currentDir = os.getcwd()
self.rssFlag = ResourceStatus().rssFlag
self.sitestatus = SiteStatus()
#############################################################################
def uploadProxy(self, group):
"""Upload a proxy to the DIRAC WMS. This method
Example usage:
>>> print diracAdmin.uploadProxy('lhcb_pilot')
{'OK': True, 'Value': 0L}
:param group: DIRAC Group
:type job: string
:return: S_OK,S_ERROR
:param permanent: Indefinitely update proxy
:type permanent: boolean
"""
return gProxyManager.uploadProxy(diracGroup=group)
#############################################################################
def setProxyPersistency(self, userDN, userGroup, persistent=True):
"""Set the persistence of a proxy in the Proxy Manager
Example usage:
>>> print diracAdmin.setProxyPersistency( 'some DN', 'dirac group', True )
{'OK': True }
:param userDN: User DN
:type userDN: string
:param userGroup: DIRAC Group
:type userGroup: string
:param persistent: Persistent flag
:type persistent: boolean
:return: S_OK,S_ERROR
"""
return gProxyManager.setPersistency(userDN, userGroup, persistent)
#############################################################################
def checkProxyUploaded(self, userDN, userGroup, requiredTime):
"""Set the persistence of a proxy in the Proxy Manager
Example usage:
>>> print diracAdmin.setProxyPersistency( 'some DN', 'dirac group', True )
{'OK': True, 'Value' : True/False }
:param userDN: User DN
:type userDN: string
:param userGroup: DIRAC Group
:type userGroup: string
:param requiredTime: Required life time of the uploaded proxy
:type requiredTime: boolean
:return: S_OK,S_ERROR
"""
return gProxyManager.userHasProxy(userDN, userGroup, requiredTime)
#############################################################################
def getSiteMask(self, printOutput=False, status='Active'):
"""Retrieve current site mask from WMS Administrator service.
Example usage:
>>> print diracAdmin.getSiteMask()
{'OK': True, 'Value': 0L}
:return: S_OK,S_ERROR
"""
result = self.sitestatus.getSites(siteState=status)
if result['OK']:
sites = result['Value']
if printOutput:
sites.sort()
for site in sites:
print site
return result
#############################################################################
def getBannedSites(self, gridType=[], printOutput=False):
"""Retrieve current list of banned and probing sites.
Example usage:
>>> print diracAdmin.getBannedSites()
{'OK': True, 'Value': []}
:return: S_OK,S_ERROR
"""
bannedSites = self.sitestatus.getSites(siteState='Banned')
if not bannedSites['OK']:
return bannedSites
probingSites = self.sitestatus.getSites(siteState='Probing')
if not probingSites['OK']:
return probingSites
mergedList = bannedSites['Value'] + probingSites['Value']
mergedList.sort()
if printOutput:
print '\n'.join(mergedList)
return S_OK(mergedList)
#############################################################################
def getSiteSection(self, site, printOutput=False):
"""Simple utility to get the list of CEs for DIRAC site name.
Example usage:
>>> print diracAdmin.getSiteSection('LCG.CERN.ch')
{'OK': True, 'Value':}
:return: S_OK,S_ERROR
"""
gridType = site.split('.')[0]
if not gConfig.getSections('/Resources/Sites/%s' % (gridType))['OK']:
return S_ERROR('/Resources/Sites/%s is not a valid site section' %
(gridType))
result = gConfig.getOptionsDict('/Resources/Sites/%s/%s' %
(gridType, site))
if printOutput and result['OK']:
print self.pPrint.pformat(result['Value'])
return result
#############################################################################
def allowSite(self, site, comment, printOutput=False):
"""Adds the site to the site mask.
Example usage:
>>> print diracAdmin.allowSite()
{'OK': True, 'Value': }
:return: S_OK,S_ERROR
"""
result = self.__checkSiteIsValid(site)
if not result['OK']:
return result
result = self.getSiteMask(status='Active')
if not result['OK']:
return result
siteMask = result['Value']
if site in siteMask:
if printOutput:
print 'Site %s is already Active' % site
return S_OK('Site %s is already Active' % site)
if self.rssFlag:
result = self.sitestatus.setSiteStatus(site, 'Active', comment)
else:
wmsAdmin = RPCClient('WorkloadManagement/WMSAdministrator')
result = wmsAdmin.allowSite(site, comment)
if not result['OK']:
return result
if printOutput:
print 'Site %s status is set to Active' % site
return result
#############################################################################
def getSiteMaskLogging(self, site=None, printOutput=False):
"""Retrieves site mask logging information.
Example usage:
>>> print diracAdmin.getSiteMaskLogging('LCG.AUVER.fr')
{'OK': True, 'Value': }
:return: S_OK,S_ERROR
"""
result = self.__checkSiteIsValid(site)
if not result['OK']:
return result
wmsAdmin = RPCClient('WorkloadManagement/WMSAdministrator')
result = wmsAdmin.getSiteMaskLogging(site)
if not result['OK']:
return result
if site:
if not result['Value'].has_key(site):
return S_ERROR('Site mask information not available for %s' %
(site))
if printOutput:
if site:
print '\nSite Mask Logging Info for %s\n' % site
else:
print '\nAll Site Mask Logging Info\n'
siteDict = result['Value']
for site, tupleList in siteDict.iteritems():
if not site:
print '\n===> %s\n' % site
for tup in tupleList:
print str( tup[0] ).ljust( 8 ) + str( tup[1] ).ljust( 20 ) + \
'( ' + str( tup[2] ).ljust( len( str( tup[2] ) ) ) + ' ) "' + str( tup[3] ) + '"'
print ' '
return result
#############################################################################
def banSite(self, site, comment, printOutput=False):
"""Removes the site from the site mask.
Example usage:
>>> print diracAdmin.banSite()
{'OK': True, 'Value': }
:return: S_OK,S_ERROR
"""
result = self.__checkSiteIsValid(site)
if not result['OK']:
return result
mask = self.getSiteMask(status='Banned')
if not mask['OK']:
return mask
siteMask = mask['Value']
if site in siteMask:
if printOutput:
print 'Site %s is already Banned' % site
return S_OK('Site %s is already Banned' % site)
if self.rssFlag:
result = self.sitestatus.setSiteStatus(site, 'Banned', comment)
else:
wmsAdmin = RPCClient('WorkloadManagement/WMSAdministrator')
result = wmsAdmin.banSite(site, comment)
if not result['OK']:
return result
if printOutput:
print 'Site %s status is set to Banned' % site
return result
#############################################################################
def __checkSiteIsValid(self, site):
"""Internal function to check that a site name is valid.
"""
sites = getSiteCEMapping()
if not sites['OK']:
return S_ERROR('Could not get site CE mapping')
siteList = sites['Value'].keys()
if not site in siteList:
return S_ERROR(
'Specified site %s is not in list of defined sites' % site)
return S_OK('%s is valid' % site)
#############################################################################
def clearMask(self):
"""Removes all sites from the site mask. Should be used with care.
Example usage:
>>> print diracAdmin.clearMask()
{'OK': True, 'Value':''}
:return: S_OK,S_ERROR
"""
wmsAdmin = RPCClient('WorkloadManagement/WMSAdministrator')
result = wmsAdmin.clearMask()
return result
#############################################################################
def getServicePorts(self, setup='', printOutput=False):
"""Checks the service ports for the specified setup. If not given this is
taken from the current installation (/DIRAC/Setup)
Example usage:
>>> print diracAdmin.getServicePorts()
{'OK': True, 'Value':''}
:return: S_OK,S_ERROR
"""
if not setup:
setup = gConfig.getValue('/DIRAC/Setup', '')
setupList = gConfig.getSections('/DIRAC/Setups', [])
if not setupList['OK']:
return S_ERROR('Could not get /DIRAC/Setups sections')
setupList = setupList['Value']
if not setup in setupList:
return S_ERROR('Setup %s is not in allowed list: %s' %
(setup, ', '.join(setupList)))
serviceSetups = gConfig.getOptionsDict('/DIRAC/Setups/%s' % setup)
if not serviceSetups['OK']:
return S_ERROR('Could not get /DIRAC/Setups/%s options' % setup)
serviceSetups = serviceSetups['Value'] # dict
systemList = gConfig.getSections('/Systems')
if not systemList['OK']:
return S_ERROR('Could not get Systems sections')
systemList = systemList['Value']
result = {}
for system in systemList:
if serviceSetups.has_key(system):
path = '/Systems/%s/%s/Services' % (system,
serviceSetups[system])
servicesList = gConfig.getSections(path)
if not servicesList['OK']:
self.log.warn('Could not get sections in %s' % path)
else:
servicesList = servicesList['Value']
if not servicesList:
servicesList = []
self.log.verbose('System: %s ServicesList: %s' %
(system, ', '.join(servicesList)))
for service in servicesList:
spath = '%s/%s/Port' % (path, service)
servicePort = gConfig.getValue(spath, 0)
if servicePort:
self.log.verbose('Found port for %s/%s = %s' %
(system, service, servicePort))
result['%s/%s' % (system, service)] = servicePort
else:
self.log.warn('No port found for %s' % spath)
else:
self.log.warn('%s is not defined in /DIRAC/Setups/%s' %
(system, setup))
if printOutput:
print self.pPrint.pformat(result)
return S_OK(result)
#############################################################################
def getProxy(self, userDN, userGroup, validity=43200, limited=False):
"""Retrieves a proxy with default 12hr validity and stores
this in a file in the local directory by default.
Example usage:
>>> print diracAdmin.getProxy()
{'OK': True, 'Value': }
:return: S_OK,S_ERROR
"""
return gProxyManager.downloadProxy(userDN,
userGroup,
limited=limited,
requiredTimeLeft=validity)
#############################################################################
def getVOMSProxy(self,
userDN,
userGroup,
vomsAttr=False,
validity=43200,
limited=False):
"""Retrieves a proxy with default 12hr validity and VOMS extensions and stores
this in a file in the local directory by default.
Example usage:
>>> print diracAdmin.getVOMSProxy()
{'OK': True, 'Value': }
:return: S_OK,S_ERROR
"""
return gProxyManager.downloadVOMSProxy(userDN,
userGroup,
limited=limited,
requiredVOMSAttribute=vomsAttr,
requiredTimeLeft=validity)
#############################################################################
def getPilotProxy(self, userDN, userGroup, validity=43200):
"""Retrieves a pilot proxy with default 12hr validity and stores
this in a file in the local directory by default.
Example usage:
>>> print diracAdmin.getVOMSProxy()
{'OK': True, 'Value': }
:return: S_OK,S_ERROR
"""
return gProxyManager.getPilotProxyFromDIRACGroup(
userDN, userGroup, requiredTimeLeft=validity)
#############################################################################
def resetJob(self, jobID):
"""Reset a job or list of jobs in the WMS. This operation resets the reschedule
counter for a job or list of jobs and allows them to run as new.
Example::
>>> print dirac.reset(12345)
{'OK': True, 'Value': [12345]}
:param job: JobID
:type job: integer or list of integers
:return: S_OK,S_ERROR
"""
if isinstance(jobID, basestring):
try:
jobID = int(jobID)
except Exception as x:
return self._errorReport(
str(x),
'Expected integer or convertible integer for existing jobID'
)
elif isinstance(jobID, list):
try:
jobID = [int(job) for job in jobID]
except Exception as x:
return self._errorReport(
str(x),
'Expected integer or convertible integer for existing jobIDs'
)
jobManager = RPCClient('WorkloadManagement/JobManager',
useCertificates=False)
result = jobManager.resetJob(jobID)
return result
#############################################################################
def getJobPilotOutput(self, jobID, directory=''):
"""Retrieve the pilot output for an existing job in the WMS.
The output will be retrieved in a local directory unless
otherwise specified.
>>> print dirac.getJobPilotOutput(12345)
{'OK': True, StdOut:'',StdError:''}
:param job: JobID
:type job: integer or string
:return: S_OK,S_ERROR
"""
if not directory:
directory = self.currentDir
if not os.path.exists(directory):
return self._errorReport('Directory %s does not exist' % directory)
wmsAdmin = RPCClient('WorkloadManagement/WMSAdministrator')
result = wmsAdmin.getJobPilotOutput(jobID)
if not result['OK']:
return result
outputPath = '%s/pilot_%s' % (directory, jobID)
if os.path.exists(outputPath):
self.log.info('Remove %s and retry to continue' % outputPath)
return S_ERROR('Remove %s and retry to continue' % outputPath)
if not os.path.exists(outputPath):
self.log.verbose('Creating directory %s' % outputPath)
os.mkdir(outputPath)
outputs = result['Value']
if outputs.has_key('StdOut'):
stdout = '%s/std.out' % (outputPath)
with open(stdout, 'w') as fopen:
fopen.write(outputs['StdOut'])
self.log.verbose('Standard output written to %s' % (stdout))
else:
self.log.warn('No standard output returned')
if outputs.has_key('StdError'):
stderr = '%s/std.err' % (outputPath)
with open(stderr, 'w') as fopen:
fopen.write(outputs['StdError'])
self.log.verbose('Standard error written to %s' % (stderr))
else:
self.log.warn('No standard error returned')
self.log.always('Outputs retrieved in %s' % outputPath)
return result
#############################################################################
def getPilotOutput(self, gridReference, directory=''):
"""Retrieve the pilot output (std.out and std.err) for an existing job in the WMS.
>>> print dirac.getJobPilotOutput(12345)
{'OK': True, 'Value': {}}
:param job: JobID
:type job: integer or string
:return: S_OK,S_ERROR
"""
if not isinstance(gridReference, basestring):
return self._errorReport('Expected string for pilot reference')
if not directory:
directory = self.currentDir
if not os.path.exists(directory):
return self._errorReport('Directory %s does not exist' % directory)
wmsAdmin = RPCClient('WorkloadManagement/WMSAdministrator')
result = wmsAdmin.getPilotOutput(gridReference)
if not result['OK']:
return result
gridReferenceSmall = gridReference.split('/')[-1]
if not gridReferenceSmall:
gridReferenceSmall = 'reference'
outputPath = '%s/pilot_%s' % (directory, gridReferenceSmall)
if os.path.exists(outputPath):
self.log.info('Remove %s and retry to continue' % outputPath)
return S_ERROR('Remove %s and retry to continue' % outputPath)
if not os.path.exists(outputPath):
self.log.verbose('Creating directory %s' % outputPath)
os.mkdir(outputPath)
outputs = result['Value']
if outputs.has_key('StdOut'):
stdout = '%s/std.out' % (outputPath)
with open(stdout, 'w') as fopen:
fopen.write(outputs['StdOut'])
self.log.info('Standard output written to %s' % (stdout))
else:
self.log.warn('No standard output returned')
if outputs.has_key('StdErr'):
stderr = '%s/std.err' % (outputPath)
with open(stderr, 'w') as fopen:
fopen.write(outputs['StdErr'])
self.log.info('Standard error written to %s' % (stderr))
else:
self.log.warn('No standard error returned')
self.log.always('Outputs retrieved in %s' % outputPath)
return result
#############################################################################
def getPilotInfo(self, gridReference):
"""Retrieve info relative to a pilot reference
>>> print dirac.getPilotInfo(12345)
{'OK': True, 'Value': {}}
:param gridReference: Pilot Job Reference
:type gridReference: string
:return: S_OK,S_ERROR
"""
if not isinstance(gridReference, basestring):
return self._errorReport('Expected string for pilot reference')
wmsAdmin = RPCClient('WorkloadManagement/WMSAdministrator')
result = wmsAdmin.getPilotInfo(gridReference)
return result
#############################################################################
def killPilot(self, gridReference):
"""Kill the pilot specified
>>> print dirac.getPilotInfo(12345)
{'OK': True, 'Value': {}}
:param gridReference: Pilot Job Reference
:return: S_OK,S_ERROR
"""
if not isinstance(gridReference, basestring):
return self._errorReport('Expected string for pilot reference')
wmsAdmin = RPCClient('WorkloadManagement/WMSAdministrator')
result = wmsAdmin.killPilot(gridReference)
return result
#############################################################################
def getPilotLoggingInfo(self, gridReference):
"""Retrieve the pilot logging info for an existing job in the WMS.
>>> print dirac.getPilotLoggingInfo(12345)
{'OK': True, 'Value': {"The output of the command"}}
:param gridReference: Gridp pilot job reference Id
:type gridReference: string
:return: S_OK,S_ERROR
"""
if type(gridReference) not in types.StringTypes:
return self._errorReport('Expected string for pilot reference')
wmsAdmin = RPCClient('WorkloadManagement/WMSAdministrator')
return wmsAdmin.getPilotLoggingInfo(gridReference)
#############################################################################
def getJobPilots(self, jobID):
"""Extract the list of submitted pilots and their status for a given
jobID from the WMS. Useful information is printed to the screen.
>>> print dirac.getJobPilots()
{'OK': True, 'Value': {PilotID:{StatusDict}}}
:param job: JobID
:type job: integer or string
:return: S_OK,S_ERROR
"""
if isinstance(jobID, basestring):
try:
jobID = int(jobID)
except Exception as x:
return self._errorReport(
str(x), 'Expected integer or string for existing jobID')
wmsAdmin = RPCClient('WorkloadManagement/WMSAdministrator')
result = wmsAdmin.getPilots(jobID)
if result['OK']:
print self.pPrint.pformat(result['Value'])
return result
#############################################################################
def getPilotSummary(self, startDate='', endDate=''):
"""Retrieve the pilot output for an existing job in the WMS. Summary is
printed at INFO level, full dictionary of results also returned.
>>> print dirac.getPilotSummary()
{'OK': True, 'Value': {CE:{Status:Count}}}
:param job: JobID
:type job: integer or string
:return: S_OK,S_ERROR
"""
wmsAdmin = RPCClient('WorkloadManagement/WMSAdministrator')
result = wmsAdmin.getPilotSummary(startDate, endDate)
if not result['OK']:
return result
ceDict = result['Value']
headers = 'CE'.ljust(28)
i = 0
for ce, summary in ceDict.iteritems():
states = summary.keys()
if len(states) > i:
i = len(states)
for i in xrange(i):
headers += 'Status'.ljust(12) + 'Count'.ljust(12)
print headers
for ce, summary in ceDict.iteritems():
line = ce.ljust(28)
states = summary.keys()
states.sort()
for state in states:
count = str(summary[state])
line += state.ljust(12) + count.ljust(12)
print line
return result
#############################################################################
def selectRequests(self,
jobID=None,
requestID=None,
requestName=None,
requestType=None,
status=None,
operation=None,
ownerDN=None,
ownerGroup=None,
requestStart=0,
limit=100,
printOutput=False):
"""Select requests from the request management system. A few notes on the selection criteria:
- jobID is the WMS JobID for the request (if applicable)
- requestID is assigned during submission of the request
- requestName is the corresponding XML file name
- requestType e.g. 'transfer'
- status e.g. Done
- operation e.g. replicateAndRegister
- requestStart e.g. the first request to consider (start from 0 by default)
- limit e.g. selection limit (default 100)
>>> dirac.selectRequests(jobID='4894')
{'OK': True, 'Value': [[<Requests>]]}
"""
options = {
'RequestID': requestID,
'RequestName': requestName,
'JobID': jobID,
'OwnerDN': ownerDN,
'OwnerGroup': ownerGroup,
'RequestType': requestType,
'Status': status,
'Operation': operation
}
conditions = {}
for key, value in options.iteritems():
if value:
try:
conditions[key] = str(value)
except Exception as x:
return self._errorReport(
str(x), 'Expected string for %s field' % key)
try:
requestStart = int(requestStart)
limit = int(limit)
except Exception as x:
return self._errorReport(str(x),
'Expected integer for %s field' % limit)
self.log.verbose('Will select requests with the following conditions')
self.log.verbose(self.pPrint.pformat(conditions))
requestClient = RPCClient("RequestManagement/centralURL")
result = requestClient.getRequestSummaryWeb(conditions, [],
requestStart, limit)
if not result['OK']:
self.log.warn(result['Message'])
return result
requestIDs = result['Value']
conds = []
for key, value in conditions.iteritems():
if value:
conds.append('%s = %s' % (key, value))
self.log.verbose(
'%s request(s) selected with conditions %s and limit %s' %
(len(requestIDs['Records']), ', '.join(conds), limit))
if printOutput:
requests = []
if len(requestIDs['Records']) > limit:
requestList = requestIDs['Records']
requests = requestList[:limit]
else:
requests = requestIDs['Records']
print '%s request(s) selected with conditions %s and limit %s' % (
len(requestIDs['Records']), ', '.join(conds), limit)
print requestIDs['ParameterNames']
for request in requests:
print request
if not requestIDs:
return S_ERROR('No requests selected for conditions: %s' %
conditions)
else:
return result
#############################################################################
def getRequestSummary(self, printOutput=False):
"""
Get a summary of the requests in the request DB.
"""
requestClient = RPCClient("RequestManagement/centralURL", timeout=120)
result = requestClient.getDBSummary()
if not result['OK']:
self.log.warn(result['Message'])
return result
if printOutput:
print self.pPrint.pformat(result['Value'])
return result
#############################################################################
def getExternalPackageVersions(self):
"""
Simple function that attempts to obtain the external versions for
the local DIRAC installation (frequently needed for debugging purposes).
"""
gLogger.info(
'DIRAC version v%dr%d build %d' %
(DIRAC.majorVersion, DIRAC.minorVersion, DIRAC.patchLevel))
try:
import lcg_util
infoStr = 'Using lcg_util from: \n%s' % lcg_util.__file__
gLogger.info(infoStr)
infoStr = "The version of lcg_utils is %s" % lcg_util.lcg_util_version(
)
gLogger.info(infoStr)
except Exception as x:
errStr = "SRM2Storage.__init__: Failed to import lcg_util: %s" % (
x)
gLogger.exception(errStr)
try:
import gfalthr as gfal
infoStr = "Using gfalthr from: \n%s" % gfal.__file__
gLogger.info(infoStr)
infoStr = "The version of gfalthr is %s" % gfal.gfal_version()
gLogger.info(infoStr)
except Exception as x:
errStr = "SRM2Storage.__init__: Failed to import gfalthr: %s." % (
x)
gLogger.warn(errStr)
try:
import gfal
infoStr = "Using gfal from: %s" % gfal.__file__
gLogger.info(infoStr)
infoStr = "The version of gfal is %s" % gfal.gfal_version()
gLogger.info(infoStr)
except Exception as x:
errStr = "SRM2Storage.__init__: Failed to import gfal: %s" % (
x)
gLogger.exception(errStr)
defaultProtocols = gConfig.getValue(
'/Resources/StorageElements/DefaultProtocols', [])
gLogger.info('Default list of protocols are: %s' %
(', '.join(defaultProtocols)))
return S_OK()
#############################################################################
def getSiteProtocols(self, site, printOutput=False):
"""
Allows to check the defined protocols for each site SE.
"""
result = self.__checkSiteIsValid(site)
if not result['OK']:
return result
siteSection = '/Resources/Sites/%s/%s/SE' % (site.split('.')[0], site)
siteSEs = gConfig.getValue(siteSection, [])
if not siteSEs:
return S_ERROR('No SEs found for site %s in section %s' %
(site, siteSection))
defaultProtocols = gConfig.getValue(
'/Resources/StorageElements/DefaultProtocols', [])
self.log.verbose('Default list of protocols are'
', '.join(defaultProtocols))
seInfo = {}
siteSEs.sort()
for se in siteSEs:
sections = gConfig.getSections('/Resources/StorageElements/%s/' %
(se))
if not sections['OK']:
return sections
for section in sections['Value']:
if gConfig.getValue(
'/Resources/StorageElements/%s/%s/ProtocolName' %
(se, section), '') == 'SRM2':
path = '/Resources/StorageElements/%s/%s/ProtocolsList' % (
se, section)
seProtocols = gConfig.getValue(path, [])
if not seProtocols:
seProtocols = defaultProtocols
seInfo[se] = seProtocols
if printOutput:
print '\nSummary of protocols for StorageElements at site %s' % site
print '\nStorageElement'.ljust(30) + 'ProtocolsList'.ljust(
30) + '\n'
for se, protocols in seInfo.iteritems():
print se.ljust(30) + ', '.join(protocols).ljust(30)
return S_OK(seInfo)
#############################################################################
def setSiteProtocols(self, site, protocolsList, printOutput=False):
"""
Allows to set the defined protocols for each SE for a given site.
"""
result = self.__checkSiteIsValid(site)
if not result['OK']:
return result
siteSection = '/Resources/Sites/%s/%s/SE' % (site.split('.')[0], site)
siteSEs = gConfig.getValue(siteSection, [])
if not siteSEs:
return S_ERROR('No SEs found for site %s in section %s' %
(site, siteSection))
defaultProtocols = gConfig.getValue(
'/Resources/StorageElements/DefaultProtocols', [])
self.log.verbose('Default list of protocols are',
', '.join(defaultProtocols))
for protocol in protocolsList:
if not protocol in defaultProtocols:
return S_ERROR(
'Requested to set protocol %s in list but %s is not '
'in default list of protocols:\n%s' %
(protocol, protocol, ', '.join(defaultProtocols)))
modifiedCS = False
result = promptUser(
'Do you want to add the following default protocols:'
' %s for SE(s):\n%s' %
(', '.join(protocolsList), ', '.join(siteSEs)))
if not result['OK']:
return result
if result['Value'].lower() != 'y':
self.log.always('No protocols will be added')
return S_OK()
for se in siteSEs:
sections = gConfig.getSections('/Resources/StorageElements/%s/' %
(se))
if not sections['OK']:
return sections
for section in sections['Value']:
if gConfig.getValue(
'/Resources/StorageElements/%s/%s/ProtocolName' %
(se, section), '') == 'SRM2':
path = '/Resources/StorageElements/%s/%s/ProtocolsList' % (
se, section)
self.log.verbose('Setting %s to %s' %
(path, ', '.join(protocolsList)))
result = self.csSetOption(path, ', '.join(protocolsList))
if not result['OK']:
return result
modifiedCS = True
if modifiedCS:
result = self.csCommitChanges(False)
if not result['OK']:
return S_ERROR('CS Commit failed with message = %s' %
(result['Message']))
else:
if printOutput:
print 'Successfully committed changes to CS'
else:
if printOutput:
print 'No modifications to CS required'
return S_OK()
#############################################################################
def csSetOption(self, optionPath, optionValue):
"""
Function to modify an existing value in the CS.
"""
return self.csAPI.setOption(optionPath, optionValue)
#############################################################################
def csSetOptionComment(self, optionPath, comment):
"""
Function to modify an existing value in the CS.
"""
return self.csAPI.setOptionComment(optionPath, comment)
#############################################################################
def csModifyValue(self, optionPath, newValue):
"""
Function to modify an existing value in the CS.
"""
return self.csAPI.modifyValue(optionPath, newValue)
#############################################################################
def csRegisterUser(self, username, properties):
"""
Registers a user in the CS.
- username: Username of the user (easy;)
- properties: Dict containing:
- DN
- groups : list/tuple of groups the user belongs to
- <others> : More properties of the user, like mail
"""
return self.csAPI.addUser(username, properties)
#############################################################################
def csDeleteUser(self, user):
"""
Deletes a user from the CS. Can take a list of users
"""
return self.csAPI.deleteUsers(user)
#############################################################################
def csModifyUser(self, username, properties, createIfNonExistant=False):
"""
Modify a user in the CS. Takes the same params as in addUser and
applies the changes
"""
return self.csAPI.modifyUser(username, properties, createIfNonExistant)
#############################################################################
def csListUsers(self, group=False):
"""
Lists the users in the CS. If no group is specified return all users.
"""
return self.csAPI.listUsers(group)
#############################################################################
def csDescribeUsers(self, mask=False):
"""
List users and their properties in the CS.
If a mask is given, only users in the mask will be returned
"""
return self.csAPI.describeUsers(mask)
#############################################################################
def csModifyGroup(self, groupname, properties, createIfNonExistant=False):
"""
Modify a user in the CS. Takes the same params as in addGroup and applies
the changes
"""
return self.csAPI.modifyGroup(groupname, properties,
createIfNonExistant)
#############################################################################
def csListHosts(self):
"""
Lists the hosts in the CS
"""
return self.csAPI.listHosts()
#############################################################################
def csDescribeHosts(self, mask=False):
"""
Gets extended info for the hosts in the CS
"""
return self.csAPI.describeHosts(mask)
#############################################################################
def csModifyHost(self, hostname, properties, createIfNonExistant=False):
"""
Modify a host in the CS. Takes the same params as in addHost and applies
the changes
"""
return self.csAPI.modifyHost(hostname, properties, createIfNonExistant)
#############################################################################
def csListGroups(self):
"""
Lists groups in the CS
"""
return self.csAPI.listGroups()
#############################################################################
def csDescribeGroups(self, mask=False):
"""
List groups and their properties in the CS.
If a mask is given, only groups in the mask will be returned
"""
return self.csAPI.describeGroups(mask)
#############################################################################
def csSyncUsersWithCFG(self, usersCFG):
"""
Synchronize users in cfg with its contents
"""
return self.csAPI.syncUsersWithCFG(usersCFG)
#############################################################################
def csCommitChanges(self, sortUsers=True):
"""
Commit the changes in the CS
"""
return self.csAPI.commitChanges(sortUsers=False)
#############################################################################
def sendMail(self,
address,
subject,
body,
fromAddress=None,
localAttempt=True,
html=False):
"""
Send mail to specified address with body.
"""
notification = NotificationClient()
return notification.sendMail(address, subject, body, fromAddress,
localAttempt, html)
#############################################################################
def sendSMS(self, userName, body, fromAddress=None):
"""
Send mail to specified address with body.
"""
if len(body) > 160:
return S_ERROR('Exceeded maximum SMS length of 160 characters')
notification = NotificationClient()
return notification.sendSMS(userName, body, fromAddress)
#############################################################################
def getBDIISite(self, site, host=None):
"""
Get information about site from BDII at host
"""
return ldapSite(site, host=host)
#############################################################################
def getBDIICluster(self, ce, host=None):
"""
Get information about ce from BDII at host
"""
return ldapCluster(ce, host=host)
#############################################################################
def getBDIICE(self, ce, host=None):
"""
Get information about ce from BDII at host
"""
return ldapCE(ce, host=host)
#############################################################################
def getBDIIService(self, ce, host=None):
"""
Get information about ce from BDII at host
"""
return ldapService(ce, host=host)
#############################################################################
def getBDIICEState(self, ce, useVO=voName, host=None):
"""
Get information about ce state from BDII at host
"""
return ldapCEState(ce, useVO, host=host)
#############################################################################
def getBDIICEVOView(self, ce, useVO=voName, host=None):
"""
Get information about ce voview from BDII at host
"""
return ldapCEVOView(ce, useVO, host=host)
#############################################################################
def getBDIISE(self, site, useVO=voName, host=None):
"""
Get information about SA from BDII at host
"""
return ldapSE(site, useVO, host=host)
def __syncCSWithVOMS(self):
self.__adminMsgs = {'Errors': [], 'Info': []}
#Get DIRAC VOMS Mapping
self.log.info("Getting DIRAC VOMS mapping")
mappingSection = '/Registry/VOMS/Mapping'
ret = gConfig.getOptionsDict(mappingSection)
if not ret['OK']:
self.log.fatal('No VOMS to DIRAC Group Mapping Available')
return ret
vomsMapping = ret['Value']
self.log.info("There are %s registered voms mappings in DIRAC" %
len(vomsMapping))
#Get VOMS VO name
self.log.info("Getting VOMS VO name")
result = self.vomsSrv.admGetVOName()
if not ret['OK']:
self.log.fatal('Could not retrieve VOMS VO name')
voNameInVOMS = result['Value']
self.log.info("VOMS VO Name is %s" % voNameInVOMS)
#Get VOMS roles
self.log.info("Getting the list of registered roles in VOMS")
result = self.vomsSrv.admListRoles()
if not ret['OK']:
self.log.fatal('Could not retrieve registered roles in VOMS')
rolesInVOMS = result['Value']
self.log.info("There are %s registered roles in VOMS" %
len(rolesInVOMS))
print rolesInVOMS
rolesInVOMS.append('')
#Map VOMS roles
vomsRoles = {}
for role in rolesInVOMS:
if role:
role = "%s/%s" % (voNameInVOMS, role)
else:
role = voNameInVOMS
groupsForRole = []
for group in vomsMapping:
if vomsMapping[group] == role:
groupsForRole.append(group)
if groupsForRole:
vomsRoles[role] = {'Groups': groupsForRole, 'Users': []}
self.log.info("DIRAC valid VOMS roles are:\n\t",
"\n\t ".join(vomsRoles.keys()))
#Get DIRAC users
self.log.info("Getting the list of registered users in DIRAC")
csapi = CSAPI()
ret = csapi.listUsers()
if not ret['OK']:
self.log.fatal('Could not retrieve current list of Users')
return ret
currentUsers = ret['Value']
ret = csapi.describeUsers(currentUsers)
if not ret['OK']:
self.log.fatal('Could not retrieve current User description')
return ret
currentUsers = ret['Value']
self.__adminMsgs['Info'].append(
"There are %s registered users in DIRAC" % len(currentUsers))
self.log.info("There are %s registered users in DIRAC" %
len(currentUsers))
#Get VOMS user entries
self.log.info("Getting the list of registered user entries in VOMS")
result = self.vomsSrv.admListMembers()
if not ret['OK']:
self.log.fatal(
'Could not retrieve registered user entries in VOMS')
usersInVOMS = result['Value']
self.__adminMsgs['Info'].append(
"There are %s registered user entries in VOMS" % len(usersInVOMS))
self.log.info("There are %s registered user entries in VOMS" %
len(usersInVOMS))
#Consolidate users by nickname
usersData = {}
newUserNames = []
knownUserNames = []
obsoleteUserNames = []
self.log.info("Retrieving usernames...")
usersInVOMS.sort()
for iUPos in range(len(usersInVOMS)):
userName = ''
user = usersInVOMS[iUPos]
for oldUser in currentUsers:
if user['DN'].strip() in List.fromChar(
currentUsers[oldUser]['DN']):
userName = oldUser
if not userName:
result = self.vomsSrv.attGetUserNickname(
user['DN'], user['CA'])
if result['OK']:
userName = result['Value']
else:
self.__adminMsgs['Errors'].append(
"Could not retrieve nickname for DN %s" % user['DN'])
self.log.error("Could not get nickname for DN", user['DN'])
userName = user['mail'][:user['mail'].find('@')]
if not userName:
self.log.error("Empty nickname for DN", user['DN'])
self.__adminMsgs['Errors'].append("Empty nickname for DN %s" %
user['DN'])
continue
self.log.info(
" (%02d%%) Found username %s : %s " %
((iUPos * 100 / len(usersInVOMS)), userName, user['DN']))
if userName not in usersData:
usersData[userName] = {
'DN': [],
'CA': [],
'Email': [],
'Groups': ['user']
}
for key in ('DN', 'CA', 'mail'):
value = user[key]
if value:
if key == "mail":
List.appendUnique(usersData[userName]['Email'], value)
else:
usersData[userName][key].append(value.strip())
if userName not in currentUsers:
List.appendUnique(newUserNames, userName)
else:
List.appendUnique(knownUserNames, userName)
self.log.info("Finished retrieving usernames")
if newUserNames:
self.log.info("There are %s new users" % len(newUserNames))
else:
self.log.info("There are no new users")
#Get the list of users for each group
result = csapi.listGroups()
if not result['OK']:
self.log.error("Could not get the list of groups in DIRAC",
result['Message'])
return result
staticGroups = result['Value']
vomsGroups = []
self.log.info("Mapping users in VOMS to groups")
for vomsRole in vomsRoles:
self.log.info(" Getting users for role %s" % vomsRole)
groupsForRole = vomsRoles[vomsRole]['Groups']
vomsMap = vomsRole.split("Role=")
for g in groupsForRole:
if g in staticGroups:
staticGroups.pop(staticGroups.index(g))
else:
vomsGroups.append(g)
if len(vomsMap) == 1:
# no Role
users = usersInVOMS
else:
vomsGroup = "Role=".join(vomsMap[:-1])
if vomsGroup[-1] == "/":
vomsGroup = vomsGroup[:-1]
vomsRole = "Role=%s" % vomsMap[-1]
result = self.vomsSrv.admListUsersWithRole(vomsGroup, vomsRole)
if not result['OK']:
errorMsg = "Could not get list of users for VOMS %s" % (
vomsMapping[group])
self.__adminMsgs['Errors'].append(errorMsg)
self.log.error(errorMsg, result['Message'])
return result
users = result['Value']
numUsersInGroup = 0
for vomsUser in users:
for userName in usersData:
if vomsUser['DN'] in usersData[userName]['DN']:
numUsersInGroup += 1
usersData[userName]['Groups'].extend(groupsForRole)
infoMsg = "There are %s users in group(s) %s for VOMS Role %s" % (
numUsersInGroup, ",".join(groupsForRole), vomsRole)
self.__adminMsgs['Info'].append(infoMsg)
self.log.info(" %s" % infoMsg)
self.log.info("Checking static groups")
staticUsers = []
for group in staticGroups:
self.log.info(" Checking static group %s" % group)
numUsersInGroup = 0
result = csapi.listUsers(group)
if not result['OK']:
self.log.error(
"Could not get the list of users in DIRAC group %s" %
group, result['Message'])
return result
for userName in result['Value']:
if userName in usersData:
numUsersInGroup += 1
usersData[userName]['Groups'].append(group)
else:
if group not in vomsGroups and userName not in staticUsers:
staticUsers.append(userName)
infoMsg = "There are %s users in group %s" % (numUsersInGroup,
group)
self.__adminMsgs['Info'].append(infoMsg)
self.log.info(" %s" % infoMsg)
if staticUsers:
infoMsg = "There are %s static users: %s" % (
len(staticUsers), ', '.join(staticUsers))
self.__adminMsgs['Info'].append(infoMsg)
self.log.info("%s" % infoMsg)
for user in currentUsers:
if user not in usersData and user not in staticUsers:
self.log.info('User %s is no longer valid' % user)
obsoleteUserNames.append(user)
#Do the CS Sync
self.log.info("Updating CS...")
ret = csapi.downloadCSData()
if not ret['OK']:
self.log.fatal('Can not update from CS', ret['Message'])
return ret
usersWithMoreThanOneDN = {}
for user in usersData:
csUserData = dict(usersData[user])
if len(csUserData['DN']) > 1:
usersWithMoreThanOneDN[user] = csUserData['DN']
result = csapi.describeUsers([user])
if result['OK']:
if result['Value']:
prevUser = result['Value'][user]
prevDNs = List.fromChar(prevUser['DN'])
newDNs = csUserData['DN']
for DN in newDNs:
if DN not in prevDNs:
self.__adminMsgs['Info'].append(
"User %s has new DN %s" % (user, DN))
for DN in prevDNs:
if DN not in newDNs:
self.__adminMsgs['Info'].append(
"User %s has lost a DN %s" % (user, DN))
else:
newDNs = csUserData['DN']
for DN in newDNs:
self.__adminMsgs['Info'].append(
"New user %s has new DN %s" % (user, DN))
for k in ('DN', 'CA', 'Email'):
csUserData[k] = ", ".join(csUserData[k])
result = csapi.modifyUser(user,
csUserData,
createIfNonExistant=True)
if not result['OK']:
self.__adminMsgs['Error'].append("Cannot modify user %s: %s" %
(user, result['Message']))
self.log.error("Cannot modify user", user)
if usersWithMoreThanOneDN:
self.__adminMsgs['Info'].append("\nUsers with more than one DN:")
for uwmtod in sorted(usersWithMoreThanOneDN):
self.__adminMsgs['Info'].append(" %s" % uwmtod)
self.__adminMsgs['Info'].append(" + DN list:")
for DN in usersWithMoreThanOneDN[uwmtod]:
self.__adminMsgs['Info'].append(" - %s" % DN)
if obsoleteUserNames:
self.__adminMsgs['Info'].append("\nObsolete users:")
address = self.am_getOption('MailTo', '*****@*****.**')
fromAddress = self.am_getOption('mailFrom', '*****@*****.**')
subject = 'Obsolete LFC Users found'
body = 'Delete entries into LFC: \n'
for obsoleteUser in obsoleteUserNames:
self.log.info(subject, ", ".join(obsoleteUserNames))
body += 'for ' + obsoleteUser + '\n'
self.__adminMsgs['Info'].append(" %s" % obsoleteUser)
self.log.info("Deleting %s users" % len(obsoleteUserNames))
NotificationClient().sendMail(address,
'UsersAndGroupsAgent: %s' % subject,
body, fromAddress)
csapi.deleteUsers(obsoleteUserNames)
if newUserNames:
self.__adminMsgs['Info'].append("\nNew users:")
for newUser in newUserNames:
self.__adminMsgs['Info'].append(" %s" % newUser)
self.__adminMsgs['Info'].append(" + DN list:")
for DN in usersData[newUser]['DN']:
self.__adminMsgs['Info'].append(" - %s" % DN)
self.__adminMsgs['Info'].append(" + EMail: %s" %
usersData[newUser]['Email'])
result = csapi.commitChanges()
if not result['OK']:
self.log.error("Could not commit configuration changes",
result['Message'])
return result
self.log.info("Configuration committed")
#LFC Check
if self.am_getOption("LFCCheckEnabled", True):
result = self.checkLFCRegisteredUsers(usersData)
if not result['OK']:
return result
return S_OK()
def __syncCSWithVOMS( self ):
self.__adminMsgs = { 'Errors' : [], 'Info' : [] }
#Get DIRAC VOMS Mapping
self.log.info( "Getting DIRAC VOMS mapping" )
mappingSection = '/Registry/VOMS/Mapping'
ret = gConfig.getOptionsDict( mappingSection )
if not ret['OK']:
self.log.fatal( 'No VOMS to DIRAC Group Mapping Available' )
return ret
vomsMapping = ret['Value']
self.log.info( "There are %s registered voms mappings in DIRAC" % len( vomsMapping ) )
#Get VOMS VO name
self.log.info( "Getting VOMS VO name" )
result = self.vomsSrv.admGetVOName()
if not ret['OK']:
self.log.fatal( 'Could not retrieve VOMS VO name' )
voNameInVOMS = result[ 'Value' ]
self.log.info( "VOMS VO Name is %s" % voNameInVOMS )
#Get VOMS roles
self.log.info( "Getting the list of registered roles in VOMS" )
result = self.vomsSrv.admListRoles()
if not ret['OK']:
self.log.fatal( 'Could not retrieve registered roles in VOMS' )
rolesInVOMS = result[ 'Value' ]
self.log.info( "There are %s registered roles in VOMS" % len( rolesInVOMS ) )
print rolesInVOMS
rolesInVOMS.append( '' )
#Map VOMS roles
vomsRoles = {}
for role in rolesInVOMS:
if role:
role = "%s/%s" % ( voNameInVOMS, role )
else:
role = voNameInVOMS
groupsForRole = []
for group in vomsMapping:
if vomsMapping[ group ] == role:
groupsForRole.append( group )
if groupsForRole:
vomsRoles[ role ] = { 'Groups' : groupsForRole, 'Users' : [] }
self.log.info( "DIRAC valid VOMS roles are:\n\t", "\n\t ".join( vomsRoles.keys() ) )
#Get DIRAC users
self.log.info( "Getting the list of registered users in DIRAC" )
csapi = CSAPI()
ret = csapi.listUsers()
if not ret['OK']:
self.log.fatal( 'Could not retrieve current list of Users' )
return ret
currentUsers = ret['Value']
ret = csapi.describeUsers( currentUsers )
if not ret['OK']:
self.log.fatal( 'Could not retrieve current User description' )
return ret
currentUsers = ret['Value']
self.__adminMsgs[ 'Info' ].append( "There are %s registered users in DIRAC" % len( currentUsers ) )
self.log.info( "There are %s registered users in DIRAC" % len( currentUsers ) )
#Get VOMS user entries
self.log.info( "Getting the list of registered user entries in VOMS" )
result = self.vomsSrv.admListMembers()
if not ret['OK']:
self.log.fatal( 'Could not retrieve registered user entries in VOMS' )
usersInVOMS = result[ 'Value' ]
self.__adminMsgs[ 'Info' ].append( "There are %s registered user entries in VOMS" % len( usersInVOMS ) )
self.log.info( "There are %s registered user entries in VOMS" % len( usersInVOMS ) )
#Consolidate users by nickname
usersData = {}
newUserNames = []
knownUserNames = []
obsoleteUserNames = []
self.log.info( "Retrieving usernames..." )
usersInVOMS.sort()
for iUPos in range( len( usersInVOMS ) ):
userName = ''
user = usersInVOMS[ iUPos ]
for oldUser in currentUsers:
if user[ 'DN' ].strip() in List.fromChar( currentUsers[oldUser][ 'DN' ] ):
userName = oldUser
if not userName:
result = self.vomsSrv.attGetUserNickname( user[ 'DN' ], user[ 'CA' ] )
if result[ 'OK' ]:
userName = result[ 'Value' ]
else:
self.__adminMsgs[ 'Errors' ].append( "Could not retrieve nickname for DN %s" % user[ 'DN' ] )
self.log.error( "Could not get nickname for DN", user[ 'DN' ] )
userName = user[ 'mail' ][:user[ 'mail' ].find( '@' )]
if not userName:
self.log.error( "Empty nickname for DN", user[ 'DN' ] )
self.__adminMsgs[ 'Errors' ].append( "Empty nickname for DN %s" % user[ 'DN' ] )
continue
self.log.info( " (%02d%%) Found username %s : %s " % ( ( iUPos * 100 / len( usersInVOMS ) ), userName, user[ 'DN' ] ) )
if userName not in usersData:
usersData[ userName ] = { 'DN': [], 'CA': [], 'Email': [], 'Groups' : ['user'] }
for key in ( 'DN', 'CA', 'mail' ):
value = user[ key ]
if value:
if key == "mail":
List.appendUnique( usersData[ userName ][ 'Email' ], value )
else:
usersData[ userName ][ key ].append( value.strip() )
if userName not in currentUsers:
List.appendUnique( newUserNames, userName )
else:
List.appendUnique( knownUserNames, userName )
self.log.info( "Finished retrieving usernames" )
if newUserNames:
self.log.info( "There are %s new users" % len( newUserNames ) )
else:
self.log.info( "There are no new users" )
#Get the list of users for each group
result = csapi.listGroups()
if not result[ 'OK' ]:
self.log.error( "Could not get the list of groups in DIRAC", result[ 'Message' ] )
return result
staticGroups = result[ 'Value' ]
vomsGroups = []
self.log.info( "Mapping users in VOMS to groups" )
for vomsRole in vomsRoles:
self.log.info( " Getting users for role %s" % vomsRole )
groupsForRole = vomsRoles[ vomsRole ][ 'Groups' ]
vomsMap = vomsRole.split( "Role=" )
for g in groupsForRole:
if g in staticGroups:
staticGroups.pop( staticGroups.index( g ) )
else:
vomsGroups.append( g )
if len( vomsMap ) == 1:
# no Role
users = usersInVOMS
else:
vomsGroup = "Role=".join( vomsMap[:-1] )
if vomsGroup[-1] == "/":
vomsGroup = vomsGroup[:-1]
vomsRole = "Role=%s" % vomsMap[-1]
result = self.vomsSrv.admListUsersWithRole( vomsGroup, vomsRole )
if not result[ 'OK' ]:
errorMsg = "Could not get list of users for VOMS %s" % ( vomsMapping[ group ] )
self.__adminMsgs[ 'Errors' ].append( errorMsg )
self.log.error( errorMsg, result[ 'Message' ] )
return result
users = result['Value']
numUsersInGroup = 0
for vomsUser in users:
for userName in usersData:
if vomsUser[ 'DN' ] in usersData[ userName ][ 'DN' ]:
numUsersInGroup += 1
usersData[ userName ][ 'Groups' ].extend( groupsForRole )
infoMsg = "There are %s users in group(s) %s for VOMS Role %s" % ( numUsersInGroup, ",".join( groupsForRole ), vomsRole )
self.__adminMsgs[ 'Info' ].append( infoMsg )
self.log.info( " %s" % infoMsg )
self.log.info( "Checking static groups" )
staticUsers = []
for group in staticGroups:
self.log.info( " Checking static group %s" % group )
numUsersInGroup = 0
result = csapi.listUsers( group )
if not result[ 'OK' ]:
self.log.error( "Could not get the list of users in DIRAC group %s" % group , result[ 'Message' ] )
return result
for userName in result[ 'Value' ]:
if userName in usersData:
numUsersInGroup += 1
usersData[ userName ][ 'Groups' ].append( group )
else:
if group not in vomsGroups and userName not in staticUsers:
staticUsers.append( userName )
infoMsg = "There are %s users in group %s" % ( numUsersInGroup, group )
self.__adminMsgs[ 'Info' ].append( infoMsg )
self.log.info( " %s" % infoMsg )
if staticUsers:
infoMsg = "There are %s static users: %s" % ( len( staticUsers ) , ', '.join( staticUsers ) )
self.__adminMsgs[ 'Info' ].append( infoMsg )
self.log.info( "%s" % infoMsg )
for user in currentUsers:
if user not in usersData and user not in staticUsers:
self.log.info( 'User %s is no longer valid' % user )
obsoleteUserNames.append( user )
#Do the CS Sync
self.log.info( "Updating CS..." )
ret = csapi.downloadCSData()
if not ret['OK']:
self.log.fatal( 'Can not update from CS', ret['Message'] )
return ret
usersWithMoreThanOneDN = {}
for user in usersData:
csUserData = dict( usersData[ user ] )
if len( csUserData[ 'DN' ] ) > 1:
usersWithMoreThanOneDN[ user ] = csUserData[ 'DN' ]
result = csapi.describeUsers( [ user ] )
if result[ 'OK' ]:
if result[ 'Value' ]:
prevUser = result[ 'Value' ][ user ]
prevDNs = List.fromChar( prevUser[ 'DN' ] )
newDNs = csUserData[ 'DN' ]
for DN in newDNs:
if DN not in prevDNs:
self.__adminMsgs[ 'Info' ].append( "User %s has new DN %s" % ( user, DN ) )
for DN in prevDNs:
if DN not in newDNs:
self.__adminMsgs[ 'Info' ].append( "User %s has lost a DN %s" % ( user, DN ) )
else:
newDNs = csUserData[ 'DN' ]
for DN in newDNs:
self.__adminMsgs[ 'Info' ].append( "New user %s has new DN %s" % ( user, DN ) )
for k in ( 'DN', 'CA', 'Email' ):
csUserData[ k ] = ", ".join( csUserData[ k ] )
result = csapi.modifyUser( user, csUserData, createIfNonExistant = True )
if not result[ 'OK' ]:
self.__adminMsgs[ 'Error' ].append( "Cannot modify user %s: %s" % ( user, result[ 'Message' ] ) )
self.log.error( "Cannot modify user", user )
if usersWithMoreThanOneDN:
self.__adminMsgs[ 'Info' ].append( "\nUsers with more than one DN:" )
for uwmtod in sorted( usersWithMoreThanOneDN ):
self.__adminMsgs[ 'Info' ].append( " %s" % uwmtod )
self.__adminMsgs[ 'Info' ].append( " + DN list:" )
for DN in usersWithMoreThanOneDN[uwmtod]:
self.__adminMsgs[ 'Info' ].append( " - %s" % DN )
if obsoleteUserNames:
self.__adminMsgs[ 'Info' ].append( "\nObsolete users:" )
address = self.am_getOption( 'MailTo', '*****@*****.**' )
fromAddress = self.am_getOption( 'mailFrom', '*****@*****.**' )
subject = 'Obsolete LFC Users found'
body = 'Delete entries into LFC: \n'
for obsoleteUser in obsoleteUserNames:
self.log.info( subject, ", ".join( obsoleteUserNames ) )
body += 'for ' + obsoleteUser + '\n'
self.__adminMsgs[ 'Info' ].append( " %s" % obsoleteUser )
self.log.info( "Deleting %s users" % len( obsoleteUserNames ) )
NotificationClient().sendMail( address, 'UsersAndGroupsAgent: %s' % subject, body, fromAddress )
csapi.deleteUsers( obsoleteUserNames )
if newUserNames:
self.__adminMsgs[ 'Info' ].append( "\nNew users:" )
for newUser in newUserNames:
self.__adminMsgs[ 'Info' ].append( " %s" % newUser )
self.__adminMsgs[ 'Info' ].append( " + DN list:" )
for DN in usersData[newUser][ 'DN' ]:
self.__adminMsgs[ 'Info' ].append( " - %s" % DN )
self.__adminMsgs[ 'Info' ].append( " + EMail: %s" % usersData[newUser][ 'Email' ] )
result = csapi.commitChanges()
if not result[ 'OK' ]:
self.log.error( "Could not commit configuration changes", result[ 'Message' ] )
return result
self.log.info( "Configuration committed" )
#LFC Check
if self.am_getOption( "LFCCheckEnabled", True ):
result = self.checkLFCRegisteredUsers( usersData )
if not result[ 'OK' ]:
return result
return S_OK()
class DiracAdmin(API):
""" Administrative functionalities
"""
#############################################################################
def __init__(self):
"""Internal initialization of the DIRAC Admin API.
"""
super(DiracAdmin, self).__init__()
self.csAPI = CSAPI()
self.dbg = False
if gConfig.getValue(self.section + '/LogLevel', 'DEBUG') == 'DEBUG':
self.dbg = True
self.scratchDir = gConfig.getValue(self.section + '/ScratchDir',
'/tmp')
self.currentDir = os.getcwd()
self.rssFlag = ResourceStatus().rssFlag
self.sitestatus = SiteStatus()
self._siteSet = set(getSites().get('Value', []))
#############################################################################
def uploadProxy(self):
"""Upload a proxy to the DIRAC WMS. This method
Example usage:
>>> print diracAdmin.uploadProxy('dteam_pilot')
{'OK': True, 'Value': 0L}
:return: S_OK,S_ERROR
:param permanent: Indefinitely update proxy
:type permanent: boolean
"""
return gProxyManager.uploadProxy()
#############################################################################
def setProxyPersistency(self, userDN, userGroup, persistent=True):
"""Set the persistence of a proxy in the Proxy Manager
Example usage:
>>> gLogger.notice(diracAdmin.setProxyPersistency( 'some DN', 'dirac group', True ))
{'OK': True }
:param userDN: User DN
:type userDN: string
:param userGroup: DIRAC Group
:type userGroup: string
:param persistent: Persistent flag
:type persistent: boolean
:return: S_OK,S_ERROR
"""
return gProxyManager.setPersistency(userDN, userGroup, persistent)
#############################################################################
def checkProxyUploaded(self, userDN, userGroup, requiredTime):
"""Set the persistence of a proxy in the Proxy Manager
Example usage:
>>> gLogger.notice(diracAdmin.setProxyPersistency( 'some DN', 'dirac group', True ))
{'OK': True, 'Value' : True/False }
:param userDN: User DN
:type userDN: string
:param userGroup: DIRAC Group
:type userGroup: string
:param requiredTime: Required life time of the uploaded proxy
:type requiredTime: boolean
:return: S_OK,S_ERROR
"""
return gProxyManager.userHasProxy(userDN, userGroup, requiredTime)
#############################################################################
def getSiteMask(self, printOutput=False, status='Active'):
"""Retrieve current site mask from WMS Administrator service.
Example usage:
>>> gLogger.notice(diracAdmin.getSiteMask())
{'OK': True, 'Value': 0L}
:return: S_OK,S_ERROR
"""
result = self.sitestatus.getSites(siteState=status)
if result['OK']:
sites = result['Value']
if printOutput:
sites.sort()
for site in sites:
gLogger.notice(site)
return result
#############################################################################
def getBannedSites(self, printOutput=False):
"""Retrieve current list of banned and probing sites.
Example usage:
>>> gLogger.notice(diracAdmin.getBannedSites())
{'OK': True, 'Value': []}
:return: S_OK,S_ERROR
"""
bannedSites = self.sitestatus.getSites(siteState='Banned')
if not bannedSites['OK']:
return bannedSites
probingSites = self.sitestatus.getSites(siteState='Probing')
if not probingSites['OK']:
return probingSites
mergedList = sorted(bannedSites['Value'] + probingSites['Value'])
if printOutput:
gLogger.notice('\n'.join(mergedList))
return S_OK(mergedList)
#############################################################################
def getSiteSection(self, site, printOutput=False):
"""Simple utility to get the list of CEs for DIRAC site name.
Example usage:
>>> gLogger.notice(diracAdmin.getSiteSection('LCG.CERN.ch'))
{'OK': True, 'Value':}
:return: S_OK,S_ERROR
"""
gridType = site.split('.')[0]
if not gConfig.getSections('/Resources/Sites/%s' % (gridType))['OK']:
return S_ERROR('/Resources/Sites/%s is not a valid site section' %
(gridType))
result = gConfig.getOptionsDict('/Resources/Sites/%s/%s' %
(gridType, site))
if printOutput and result['OK']:
gLogger.notice(self.pPrint.pformat(result['Value']))
return result
#############################################################################
def allowSite(self, site, comment, printOutput=False):
"""Adds the site to the site mask.
Example usage:
>>> gLogger.notice(diracAdmin.allowSite())
{'OK': True, 'Value': }
:return: S_OK,S_ERROR
"""
result = self.__checkSiteIsValid(site)
if not result['OK']:
return result
result = self.getSiteMask(status='Active')
if not result['OK']:
return result
siteMask = result['Value']
if site in siteMask:
if printOutput:
gLogger.notice('Site %s is already Active' % site)
return S_OK('Site %s is already Active' % site)
if self.rssFlag:
result = self.sitestatus.setSiteStatus(site, 'Active', comment)
else:
result = WMSAdministratorClient().allowSite(site, comment)
if not result['OK']:
return result
if printOutput:
gLogger.notice('Site %s status is set to Active' % site)
return result
#############################################################################
def getSiteMaskLogging(self, site=None, printOutput=False):
"""Retrieves site mask logging information.
Example usage:
>>> gLogger.notice(diracAdmin.getSiteMaskLogging('LCG.AUVER.fr'))
{'OK': True, 'Value': }
:return: S_OK,S_ERROR
"""
result = self.__checkSiteIsValid(site)
if not result['OK']:
return result
if self.rssFlag:
result = ResourceStatusClient().selectStatusElement('Site',
'History',
name=site)
else:
result = WMSAdministratorClient().getSiteMaskLogging(site)
if not result['OK']:
return result
if printOutput:
if site:
gLogger.notice('\nSite Mask Logging Info for %s\n' % site)
else:
gLogger.notice('\nAll Site Mask Logging Info\n')
sitesLogging = result['Value']
if isinstance(sitesLogging, dict):
for siteName, tupleList in sitesLogging.items(
): # can be an iterator
if not siteName:
gLogger.notice('\n===> %s\n' % siteName)
for tup in tupleList:
stup = str(tup[0]).ljust(8) + str(tup[1]).ljust(20)
stup += '( ' + str(tup[2]).ljust(len(str(
tup[2]))) + ' ) "' + str(tup[3]) + '"'
gLogger.notice(stup)
gLogger.notice(' ')
elif isinstance(sitesLogging, list):
sitesLoggingList = [(sl[1], sl[3], sl[4])
for sl in sitesLogging]
for siteLog in sitesLoggingList:
gLogger.notice(siteLog)
return S_OK()
#############################################################################
def banSite(self, site, comment, printOutput=False):
"""Removes the site from the site mask.
Example usage:
>>> gLogger.notice(diracAdmin.banSite())
{'OK': True, 'Value': }
:return: S_OK,S_ERROR
"""
result = self.__checkSiteIsValid(site)
if not result['OK']:
return result
mask = self.getSiteMask(status='Banned')
if not mask['OK']:
return mask
siteMask = mask['Value']
if site in siteMask:
if printOutput:
gLogger.notice('Site %s is already Banned' % site)
return S_OK('Site %s is already Banned' % site)
if self.rssFlag:
result = self.sitestatus.setSiteStatus(site, 'Banned', comment)
else:
result = WMSAdministratorClient().banSite(site, comment)
if not result['OK']:
return result
if printOutput:
gLogger.notice('Site %s status is set to Banned' % site)
return result
#############################################################################
def __checkSiteIsValid(self, site):
"""Internal function to check that a site name is valid.
"""
if isinstance(site, (list, set, dict)):
site = set(site) - self._siteSet
if not site:
return S_OK()
elif site in self._siteSet:
return S_OK()
return S_ERROR('Specified site %s is not in list of defined sites' %
str(site))
#############################################################################
def getServicePorts(self, setup='', printOutput=False):
"""Checks the service ports for the specified setup. If not given this is
taken from the current installation (/DIRAC/Setup)
Example usage:
>>> gLogger.notice(diracAdmin.getServicePorts())
{'OK': True, 'Value':''}
:return: S_OK,S_ERROR
"""
if not setup:
setup = gConfig.getValue('/DIRAC/Setup', '')
setupList = gConfig.getSections('/DIRAC/Setups', [])
if not setupList['OK']:
return S_ERROR('Could not get /DIRAC/Setups sections')
setupList = setupList['Value']
if setup not in setupList:
return S_ERROR('Setup %s is not in allowed list: %s' %
(setup, ', '.join(setupList)))
serviceSetups = gConfig.getOptionsDict('/DIRAC/Setups/%s' % setup)
if not serviceSetups['OK']:
return S_ERROR('Could not get /DIRAC/Setups/%s options' % setup)
serviceSetups = serviceSetups['Value'] # dict
systemList = gConfig.getSections('/Systems')
if not systemList['OK']:
return S_ERROR('Could not get Systems sections')
systemList = systemList['Value']
result = {}
for system in systemList:
if system in serviceSetups:
path = '/Systems/%s/%s/Services' % (system,
serviceSetups[system])
servicesList = gConfig.getSections(path)
if not servicesList['OK']:
self.log.warn('Could not get sections in %s' % path)
else:
servicesList = servicesList['Value']
if not servicesList:
servicesList = []
self.log.verbose('System: %s ServicesList: %s' %
(system, ', '.join(servicesList)))
for service in servicesList:
spath = '%s/%s/Port' % (path, service)
servicePort = gConfig.getValue(spath, 0)
if servicePort:
self.log.verbose('Found port for %s/%s = %s' %
(system, service, servicePort))
result['%s/%s' % (system, service)] = servicePort
else:
self.log.warn('No port found for %s' % spath)
else:
self.log.warn('%s is not defined in /DIRAC/Setups/%s' %
(system, setup))
if printOutput:
gLogger.notice(self.pPrint.pformat(result))
return S_OK(result)
#############################################################################
def getProxy(self, userDN, userGroup, validity=43200, limited=False):
"""Retrieves a proxy with default 12hr validity and stores
this in a file in the local directory by default.
Example usage:
>>> gLogger.notice(diracAdmin.getProxy())
{'OK': True, 'Value': }
:return: S_OK,S_ERROR
"""
return gProxyManager.downloadProxy(userDN,
userGroup,
limited=limited,
requiredTimeLeft=validity)
#############################################################################
def getVOMSProxy(self,
userDN,
userGroup,
vomsAttr=False,
validity=43200,
limited=False):
"""Retrieves a proxy with default 12hr validity and VOMS extensions and stores
this in a file in the local directory by default.
Example usage:
>>> gLogger.notice(diracAdmin.getVOMSProxy())
{'OK': True, 'Value': }
:return: S_OK,S_ERROR
"""
return gProxyManager.downloadVOMSProxy(userDN,
userGroup,
limited=limited,
requiredVOMSAttribute=vomsAttr,
requiredTimeLeft=validity)
#############################################################################
def getPilotProxy(self, userDN, userGroup, validity=43200):
"""Retrieves a pilot proxy with default 12hr validity and stores
this in a file in the local directory by default.
Example usage:
>>> gLogger.notice(diracAdmin.getVOMSProxy())
{'OK': True, 'Value': }
:return: S_OK,S_ERROR
"""
return gProxyManager.getPilotProxyFromDIRACGroup(
userDN, userGroup, requiredTimeLeft=validity)
#############################################################################
def resetJob(self, jobID):
"""Reset a job or list of jobs in the WMS. This operation resets the reschedule
counter for a job or list of jobs and allows them to run as new.
Example::
>>> gLogger.notice(dirac.reset(12345))
{'OK': True, 'Value': [12345]}
:param job: JobID
:type job: integer or list of integers
:return: S_OK,S_ERROR
"""
if isinstance(jobID, six.string_types):
try:
jobID = int(jobID)
except Exception as x:
return self._errorReport(
str(x),
'Expected integer or convertible integer for existing jobID'
)
elif isinstance(jobID, list):
try:
jobID = [int(job) for job in jobID]
except Exception as x:
return self._errorReport(
str(x),
'Expected integer or convertible integer for existing jobIDs'
)
result = JobManagerClient(useCertificates=False).resetJob(jobID)
return result
#############################################################################
def getJobPilotOutput(self, jobID, directory=''):
"""Retrieve the pilot output for an existing job in the WMS.
The output will be retrieved in a local directory unless
otherwise specified.
>>> gLogger.notice(dirac.getJobPilotOutput(12345))
{'OK': True, StdOut:'',StdError:''}
:param job: JobID
:type job: integer or string
:return: S_OK,S_ERROR
"""
if not directory:
directory = self.currentDir
if not os.path.exists(directory):
return self._errorReport('Directory %s does not exist' % directory)
result = WMSAdministratorClient().getJobPilotOutput(jobID)
if not result['OK']:
return result
outputPath = '%s/pilot_%s' % (directory, jobID)
if os.path.exists(outputPath):
self.log.info('Remove %s and retry to continue' % outputPath)
return S_ERROR('Remove %s and retry to continue' % outputPath)
if not os.path.exists(outputPath):
self.log.verbose('Creating directory %s' % outputPath)
os.mkdir(outputPath)
outputs = result['Value']
if 'StdOut' in outputs:
stdout = '%s/std.out' % (outputPath)
with open(stdout, 'w') as fopen:
fopen.write(outputs['StdOut'])
self.log.verbose('Standard output written to %s' % (stdout))
else:
self.log.warn('No standard output returned')
if 'StdError' in outputs:
stderr = '%s/std.err' % (outputPath)
with open(stderr, 'w') as fopen:
fopen.write(outputs['StdError'])
self.log.verbose('Standard error written to %s' % (stderr))
else:
self.log.warn('No standard error returned')
self.log.always('Outputs retrieved in %s' % outputPath)
return result
#############################################################################
def getPilotOutput(self, gridReference, directory=''):
"""Retrieve the pilot output (std.out and std.err) for an existing job in the WMS.
>>> gLogger.notice(dirac.getJobPilotOutput(12345))
{'OK': True, 'Value': {}}
:param job: JobID
:type job: integer or string
:return: S_OK,S_ERROR
"""
if not isinstance(gridReference, six.string_types):
return self._errorReport('Expected string for pilot reference')
if not directory:
directory = self.currentDir
if not os.path.exists(directory):
return self._errorReport('Directory %s does not exist' % directory)
result = PilotManagerClient().getPilotOutput(gridReference)
if not result['OK']:
return result
gridReferenceSmall = gridReference.split('/')[-1]
if not gridReferenceSmall:
gridReferenceSmall = 'reference'
outputPath = '%s/pilot_%s' % (directory, gridReferenceSmall)
if os.path.exists(outputPath):
self.log.info('Remove %s and retry to continue' % outputPath)
return S_ERROR('Remove %s and retry to continue' % outputPath)
if not os.path.exists(outputPath):
self.log.verbose('Creating directory %s' % outputPath)
os.mkdir(outputPath)
outputs = result['Value']
if 'StdOut' in outputs:
stdout = '%s/std.out' % (outputPath)
with open(stdout, 'w') as fopen:
fopen.write(outputs['StdOut'])
self.log.info('Standard output written to %s' % (stdout))
else:
self.log.warn('No standard output returned')
if 'StdErr' in outputs:
stderr = '%s/std.err' % (outputPath)
with open(stderr, 'w') as fopen:
fopen.write(outputs['StdErr'])
self.log.info('Standard error written to %s' % (stderr))
else:
self.log.warn('No standard error returned')
self.log.always('Outputs retrieved in %s' % outputPath)
return result
#############################################################################
def getPilotInfo(self, gridReference):
"""Retrieve info relative to a pilot reference
>>> gLogger.notice(dirac.getPilotInfo(12345))
{'OK': True, 'Value': {}}
:param gridReference: Pilot Job Reference
:type gridReference: string
:return: S_OK,S_ERROR
"""
if not isinstance(gridReference, six.string_types):
return self._errorReport('Expected string for pilot reference')
result = PilotManagerClient().getPilotInfo(gridReference)
return result
#############################################################################
def killPilot(self, gridReference):
"""Kill the pilot specified
>>> gLogger.notice(dirac.getPilotInfo(12345))
{'OK': True, 'Value': {}}
:param gridReference: Pilot Job Reference
:return: S_OK,S_ERROR
"""
if not isinstance(gridReference, six.string_types):
return self._errorReport('Expected string for pilot reference')
result = PilotManagerClient().killPilot(gridReference)
return result
#############################################################################
def getPilotLoggingInfo(self, gridReference):
"""Retrieve the pilot logging info for an existing job in the WMS.
>>> gLogger.notice(dirac.getPilotLoggingInfo(12345))
{'OK': True, 'Value': {"The output of the command"}}
:param gridReference: Gridp pilot job reference Id
:type gridReference: string
:return: S_OK,S_ERROR
"""
if not isinstance(gridReference, six.string_types):
return self._errorReport('Expected string for pilot reference')
return PilotManagerClient().getPilotLoggingInfo(gridReference)
#############################################################################
def getJobPilots(self, jobID):
"""Extract the list of submitted pilots and their status for a given
jobID from the WMS. Useful information is printed to the screen.
>>> gLogger.notice(dirac.getJobPilots())
{'OK': True, 'Value': {PilotID:{StatusDict}}}
:param job: JobID
:type job: integer or string
:return: S_OK,S_ERROR
"""
if isinstance(jobID, six.string_types):
try:
jobID = int(jobID)
except Exception as x:
return self._errorReport(
str(x), 'Expected integer or string for existing jobID')
result = PilotManagerClient().getPilots(jobID)
if result['OK']:
gLogger.notice(self.pPrint.pformat(result['Value']))
return result
#############################################################################
def getPilotSummary(self, startDate='', endDate=''):
"""Retrieve the pilot output for an existing job in the WMS. Summary is
printed at INFO level, full dictionary of results also returned.
>>> gLogger.notice(dirac.getPilotSummary())
{'OK': True, 'Value': {CE:{Status:Count}}}
:param job: JobID
:type job: integer or string
:return: S_OK,S_ERROR
"""
result = PilotManagerClient().getPilotSummary(startDate, endDate)
if not result['OK']:
return result
ceDict = result['Value']
headers = 'CE'.ljust(28)
i = 0
for ce, summary in ceDict.iteritems():
states = summary.keys()
if len(states) > i:
i = len(states)
for i in xrange(i):
headers += 'Status'.ljust(12) + 'Count'.ljust(12)
gLogger.notice(headers)
for ce, summary in ceDict.iteritems():
line = ce.ljust(28)
states = sorted(summary)
for state in states:
count = str(summary[state])
line += state.ljust(12) + count.ljust(12)
gLogger.notice(line)
return result
#############################################################################
def setSiteProtocols(self, site, protocolsList, printOutput=False):
"""
Allows to set the defined protocols for each SE for a given site.
"""
result = self.__checkSiteIsValid(site)
if not result['OK']:
return result
siteSection = '/Resources/Sites/%s/%s/SE' % (site.split('.')[0], site)
siteSEs = gConfig.getValue(siteSection, [])
if not siteSEs:
return S_ERROR('No SEs found for site %s in section %s' %
(site, siteSection))
defaultProtocols = gConfig.getValue(
'/Resources/StorageElements/DefaultProtocols', [])
self.log.verbose('Default list of protocols are',
', '.join(defaultProtocols))
for protocol in protocolsList:
if protocol not in defaultProtocols:
return S_ERROR(
'Requested to set protocol %s in list but %s is not '
'in default list of protocols:\n%s' %
(protocol, protocol, ', '.join(defaultProtocols)))
modifiedCS = False
result = promptUser(
'Do you want to add the following default protocols:'
' %s for SE(s):\n%s' %
(', '.join(protocolsList), ', '.join(siteSEs)))
if not result['OK']:
return result
if result['Value'].lower() != 'y':
self.log.always('No protocols will be added')
return S_OK()
for se in siteSEs:
sections = gConfig.getSections('/Resources/StorageElements/%s/' %
(se))
if not sections['OK']:
return sections
for section in sections['Value']:
if gConfig.getValue(
'/Resources/StorageElements/%s/%s/ProtocolName' %
(se, section), '') == 'SRM2':
path = '/Resources/StorageElements/%s/%s/ProtocolsList' % (
se, section)
self.log.verbose('Setting %s to %s' %
(path, ', '.join(protocolsList)))
result = self.csSetOption(path, ', '.join(protocolsList))
if not result['OK']:
return result
modifiedCS = True
if modifiedCS:
result = self.csCommitChanges(False)
if not result['OK']:
return S_ERROR('CS Commit failed with message = %s' %
(result['Message']))
else:
if printOutput:
gLogger.notice('Successfully committed changes to CS')
else:
if printOutput:
gLogger.notice('No modifications to CS required')
return S_OK()
#############################################################################
def csSetOption(self, optionPath, optionValue):
"""
Function to modify an existing value in the CS.
"""
return self.csAPI.setOption(optionPath, optionValue)
#############################################################################
def csSetOptionComment(self, optionPath, comment):
"""
Function to modify an existing value in the CS.
"""
return self.csAPI.setOptionComment(optionPath, comment)
#############################################################################
def csModifyValue(self, optionPath, newValue):
"""
Function to modify an existing value in the CS.
"""
return self.csAPI.modifyValue(optionPath, newValue)
#############################################################################
def csRegisterUser(self, username, properties):
"""
Registers a user in the CS.
- username: Username of the user (easy;)
- properties: Dict containing:
- DN
- groups : list/tuple of groups the user belongs to
- <others> : More properties of the user, like mail
"""
return self.csAPI.addUser(username, properties)
#############################################################################
def csDeleteUser(self, user):
"""
Deletes a user from the CS. Can take a list of users
"""
return self.csAPI.deleteUsers(user)
#############################################################################
def csModifyUser(self, username, properties, createIfNonExistant=False):
"""
Modify a user in the CS. Takes the same params as in addUser and
applies the changes
"""
return self.csAPI.modifyUser(username, properties, createIfNonExistant)
#############################################################################
def csListUsers(self, group=False):
"""
Lists the users in the CS. If no group is specified return all users.
"""
return self.csAPI.listUsers(group)
#############################################################################
def csDescribeUsers(self, mask=False):
"""
List users and their properties in the CS.
If a mask is given, only users in the mask will be returned
"""
return self.csAPI.describeUsers(mask)
#############################################################################
def csModifyGroup(self, groupname, properties, createIfNonExistant=False):
"""
Modify a user in the CS. Takes the same params as in addGroup and applies
the changes
"""
return self.csAPI.modifyGroup(groupname, properties,
createIfNonExistant)
#############################################################################
def csListHosts(self):
"""
Lists the hosts in the CS
"""
return self.csAPI.listHosts()
#############################################################################
def csDescribeHosts(self, mask=False):
"""
Gets extended info for the hosts in the CS
"""
return self.csAPI.describeHosts(mask)
#############################################################################
def csModifyHost(self, hostname, properties, createIfNonExistant=False):
"""
Modify a host in the CS. Takes the same params as in addHost and applies
the changes
"""
return self.csAPI.modifyHost(hostname, properties, createIfNonExistant)
#############################################################################
def csListGroups(self):
"""
Lists groups in the CS
"""
return self.csAPI.listGroups()
#############################################################################
def csDescribeGroups(self, mask=False):
"""
List groups and their properties in the CS.
If a mask is given, only groups in the mask will be returned
"""
return self.csAPI.describeGroups(mask)
#############################################################################
def csSyncUsersWithCFG(self, usersCFG):
"""
Synchronize users in cfg with its contents
"""
return self.csAPI.syncUsersWithCFG(usersCFG)
#############################################################################
def csCommitChanges(self, sortUsers=True):
"""
Commit the changes in the CS
"""
return self.csAPI.commitChanges(sortUsers=False)
#############################################################################
def sendMail(self,
address,
subject,
body,
fromAddress=None,
localAttempt=True,
html=False):
"""
Send mail to specified address with body.
"""
notification = NotificationClient()
return notification.sendMail(address, subject, body, fromAddress,
localAttempt, html)
#############################################################################
def sendSMS(self, userName, body, fromAddress=None):
"""
Send mail to specified address with body.
"""
if len(body) > 160:
return S_ERROR('Exceeded maximum SMS length of 160 characters')
notification = NotificationClient()
return notification.sendSMS(userName, body, fromAddress)
#############################################################################
def getBDIISite(self, site, host=None):
"""
Get information about site from BDII at host
"""
return ldapSite(site, host=host)
#############################################################################
def getBDIICluster(self, ce, host=None):
"""
Get information about ce from BDII at host
"""
return ldapCluster(ce, host=host)
#############################################################################
def getBDIICE(self, ce, host=None):
"""
Get information about ce from BDII at host
"""
return ldapCE(ce, host=host)
#############################################################################
def getBDIIService(self, ce, host=None):
"""
Get information about ce from BDII at host
"""
return ldapService(ce, host=host)
#############################################################################
def getBDIICEState(self, ce, useVO=voName, host=None):
"""
Get information about ce state from BDII at host
"""
return ldapCEState(ce, useVO, host=host)
#############################################################################
def getBDIICEVOView(self, ce, useVO=voName, host=None):
"""
Get information about ce voview from BDII at host
"""
return ldapCEVOView(ce, useVO, host=host)
class DiracAdmin( API ):
""" Administrative functionalities
"""
#############################################################################
def __init__( self ):
"""Internal initialization of the DIRAC Admin API.
"""
super( DiracAdmin, self ).__init__()
self.csAPI = CSAPI()
self.dbg = False
if gConfig.getValue( self.section + '/LogLevel', 'DEBUG' ) == 'DEBUG':
self.dbg = True
self.scratchDir = gConfig.getValue( self.section + '/ScratchDir', '/tmp' )
self.currentDir = os.getcwd()
#############################################################################
def uploadProxy( self, group ):
"""Upload a proxy to the DIRAC WMS. This method
Example usage:
>>> print diracAdmin.uploadProxy('lhcb_pilot')
{'OK': True, 'Value': 0L}
:param group: DIRAC Group
:type job: string
:return: S_OK,S_ERROR
:param permanent: Indefinitely update proxy
:type permanent: boolean
"""
return gProxyManager.uploadProxy( diracGroup = group )
#############################################################################
def setProxyPersistency( self, userDN, userGroup, persistent = True ):
"""Set the persistence of a proxy in the Proxy Manager
Example usage:
>>> print diracAdmin.setProxyPersistency( 'some DN', 'dirac group', True )
{'OK': True }
:param userDN: User DN
:type userDN: string
:param userGroup: DIRAC Group
:type userGroup: string
:param persistent: Persistent flag
:type persistent: boolean
:return: S_OK,S_ERROR
"""
return gProxyManager.setPersistency( userDN, userGroup, persistent )
#############################################################################
def checkProxyUploaded( self, userDN, userGroup, requiredTime ):
"""Set the persistence of a proxy in the Proxy Manager
Example usage:
>>> print diracAdmin.setProxyPersistency( 'some DN', 'dirac group', True )
{'OK': True, 'Value' : True/False }
:param userDN: User DN
:type userDN: string
:param userGroup: DIRAC Group
:type userGroup: string
:param requiredTime: Required life time of the uploaded proxy
:type requiredTime: boolean
:return: S_OK,S_ERROR
"""
return gProxyManager.userHasProxy( userDN, userGroup, requiredTime )
#############################################################################
def getSiteMask( self, printOutput = False ):
"""Retrieve current site mask from WMS Administrator service.
Example usage:
>>> print diracAdmin.getSiteMask()
{'OK': True, 'Value': 0L}
:return: S_OK,S_ERROR
"""
wmsAdmin = RPCClient( 'WorkloadManagement/WMSAdministrator' )
result = wmsAdmin.getSiteMask()
if result['OK']:
sites = result['Value']
if printOutput:
sites.sort()
for site in sites:
print site
return result
#############################################################################
def getBannedSites( self, gridType = [], printOutput = False ):
"""Retrieve current list of banned sites.
Example usage:
>>> print diracAdmin.getBannedSites()
{'OK': True, 'Value': []}
:return: S_OK,S_ERROR
"""
wmsAdmin = RPCClient( 'WorkloadManagement/WMSAdministrator' )
bannedSites = []
totalList = []
result = wmsAdmin.getSiteMask()
if not result['OK']:
self.log.warn( result['Message'] )
return result
sites = result['Value']
if not gridType:
result = gConfig.getSections( '/Resources/Sites' )
if not result['OK']:
return result
gridType = result['Value']
for grid in gridType:
result = gConfig.getSections( '/Resources/Sites/%s' % grid )
if not result['OK']:
return result
totalList += result['Value']
for site in totalList:
if not site in sites:
bannedSites.append( site )
bannedSites.sort()
if printOutput:
print '\n'.join( bannedSites )
return S_OK( bannedSites )
#############################################################################
def getSiteSection( self, site, printOutput = False ):
"""Simple utility to get the list of CEs for DIRAC site name.
Example usage:
>>> print diracAdmin.getSiteSection('LCG.CERN.ch')
{'OK': True, 'Value':}
:return: S_OK,S_ERROR
"""
gridType = site.split( '.' )[0]
if not gConfig.getSections( '/Resources/Sites/%s' % ( gridType ) )['OK']:
return S_ERROR( '/Resources/Sites/%s is not a valid site section' % ( gridType ) )
result = gConfig.getOptionsDict( '/Resources/Sites/%s/%s' % ( gridType, site ) )
if printOutput and result['OK']:
print self.pPrint.pformat( result['Value'] )
return result
#############################################################################
def addSiteInMask( self, site, comment, printOutput = False ):
"""Adds the site to the site mask.
Example usage:
>>> print diracAdmin.addSiteInMask()
{'OK': True, 'Value': }
:return: S_OK,S_ERROR
"""
result = self.__checkSiteIsValid( site )
if not result['OK']:
return result
mask = self.getSiteMask()
if not mask['OK']:
return mask
siteMask = mask['Value']
if site in siteMask:
return S_ERROR( 'Site %s already in mask of allowed sites' % site )
wmsAdmin = RPCClient( 'WorkloadManagement/WMSAdministrator' )
result = wmsAdmin.allowSite( site, comment )
if not result['OK']:
return result
if printOutput:
print 'Allowing %s in site mask' % site
return result
#############################################################################
def getSiteMaskLogging( self, site = None, printOutput = False ):
"""Retrieves site mask logging information.
Example usage:
>>> print diracAdmin.getSiteMaskLogging('LCG.AUVER.fr')
{'OK': True, 'Value': }
:return: S_OK,S_ERROR
"""
result = self.__checkSiteIsValid( site )
if not result['OK']:
return result
wmsAdmin = RPCClient( 'WorkloadManagement/WMSAdministrator' )
result = wmsAdmin.getSiteMaskLogging( site )
if not result['OK']:
return result
if site:
if not result['Value'].has_key( site ):
return S_ERROR( 'Site mask information not available for %s' % ( site ) )
if printOutput:
if site:
print '\nSite Mask Logging Info for %s\n' % site
else:
print '\nAll Site Mask Logging Info\n'
siteDict = result['Value']
for site, tupleList in siteDict.iteritems():
if not site:
print '\n===> %s\n' % site
for tup in tupleList:
print str( tup[0] ).ljust( 8 ) + str( tup[1] ).ljust( 20 ) + \
'( ' + str( tup[2] ).ljust( len( str( tup[2] ) ) ) + ' ) "' + str( tup[3] ) + '"'
print ' '
return result
#############################################################################
def banSiteFromMask( self, site, comment, printOutput = False ):
"""Removes the site from the site mask.
Example usage:
>>> print diracAdmin.banSiteFromMask()
{'OK': True, 'Value': }
:return: S_OK,S_ERROR
"""
result = self.__checkSiteIsValid( site )
if not result['OK']:
return result
mask = self.getSiteMask()
if not mask['OK']:
return mask
siteMask = mask['Value']
if not site in siteMask:
return S_ERROR( 'Site %s is already banned' % site )
wmsAdmin = RPCClient( 'WorkloadManagement/WMSAdministrator' )
result = wmsAdmin.banSite( site, comment )
if not result['OK']:
return result
if printOutput:
print 'Removing %s from site mask' % site
return result
#############################################################################
@classmethod
def __checkSiteIsValid( self, site ):
"""Internal function to check that a site name is valid.
"""
sites = getSiteCEMapping()
if not sites['OK']:
return S_ERROR( 'Could not get site CE mapping' )
siteList = sites['Value'].keys()
if not site in siteList:
return S_ERROR( 'Specified site %s is not in list of defined sites' % site )
return S_OK( '%s is valid' % site )
#############################################################################
def clearMask( self ):
"""Removes all sites from the site mask. Should be used with care.
Example usage:
>>> print diracAdmin.clearMask()
{'OK': True, 'Value':''}
:return: S_OK,S_ERROR
"""
wmsAdmin = RPCClient( 'WorkloadManagement/WMSAdministrator' )
result = wmsAdmin.clearMask()
return result
#############################################################################
def getServicePorts( self, setup = '', printOutput = False ):
"""Checks the service ports for the specified setup. If not given this is
taken from the current installation (/DIRAC/Setup)
Example usage:
>>> print diracAdmin.getServicePorts()
{'OK': True, 'Value':''}
:return: S_OK,S_ERROR
"""
if not setup:
setup = gConfig.getValue( '/DIRAC/Setup', '' )
setupList = gConfig.getSections( '/DIRAC/Setups', [] )
if not setupList['OK']:
return S_ERROR( 'Could not get /DIRAC/Setups sections' )
setupList = setupList['Value']
if not setup in setupList:
return S_ERROR( 'Setup %s is not in allowed list: %s' % ( setup, ', '.join( setupList ) ) )
serviceSetups = gConfig.getOptionsDict( '/DIRAC/Setups/%s' % setup )
if not serviceSetups['OK']:
return S_ERROR( 'Could not get /DIRAC/Setups/%s options' % setup )
serviceSetups = serviceSetups['Value'] # dict
systemList = gConfig.getSections( '/Systems' )
if not systemList['OK']:
return S_ERROR( 'Could not get Systems sections' )
systemList = systemList['Value']
result = {}
for system in systemList:
if serviceSetups.has_key( system ):
path = '/Systems/%s/%s/Services' % ( system, serviceSetups[system] )
servicesList = gConfig.getSections( path )
if not servicesList['OK']:
self.log.warn( 'Could not get sections in %s' % path )
else:
servicesList = servicesList['Value']
if not servicesList:
servicesList = []
self.log.verbose( 'System: %s ServicesList: %s' % ( system, ', '.join( servicesList ) ) )
for service in servicesList:
spath = '%s/%s/Port' % ( path, service )
servicePort = gConfig.getValue( spath, 0 )
if servicePort:
self.log.verbose( 'Found port for %s/%s = %s' % ( system, service, servicePort ) )
result['%s/%s' % ( system, service )] = servicePort
else:
self.log.warn( 'No port found for %s' % spath )
else:
self.log.warn( '%s is not defined in /DIRAC/Setups/%s' % ( system, setup ) )
if printOutput:
print self.pPrint.pformat( result )
return S_OK( result )
#############################################################################
def getProxy( self, userDN, userGroup, validity = 43200, limited = False ):
"""Retrieves a proxy with default 12hr validity and stores
this in a file in the local directory by default.
Example usage:
>>> print diracAdmin.getProxy()
{'OK': True, 'Value': }
:return: S_OK,S_ERROR
"""
return gProxyManager.downloadProxy( userDN, userGroup, limited = limited,
requiredTimeLeft = validity )
#############################################################################
def getVOMSProxy( self, userDN, userGroup, vomsAttr = False, validity = 43200, limited = False ):
"""Retrieves a proxy with default 12hr validity and VOMS extensions and stores
this in a file in the local directory by default.
Example usage:
>>> print diracAdmin.getVOMSProxy()
{'OK': True, 'Value': }
:return: S_OK,S_ERROR
"""
return gProxyManager.downloadVOMSProxy( userDN, userGroup, limited = limited,
requiredVOMSAttribute = vomsAttr,
requiredTimeLeft = validity )
#############################################################################
def getPilotProxy( self, userDN, userGroup, validity = 43200 ):
"""Retrieves a pilot proxy with default 12hr validity and stores
this in a file in the local directory by default.
Example usage:
>>> print diracAdmin.getVOMSProxy()
{'OK': True, 'Value': }
:return: S_OK,S_ERROR
"""
return gProxyManager.getPilotProxyFromDIRACGroup( userDN, userGroup, requiredTimeLeft = validity )
#############################################################################
def resetJob( self, jobID ):
"""Reset a job or list of jobs in the WMS. This operation resets the reschedule
counter for a job or list of jobs and allows them to run as new.
Example::
>>> print dirac.reset(12345)
{'OK': True, 'Value': [12345]}
:param job: JobID
:type job: integer or list of integers
:return: S_OK,S_ERROR
"""
if isinstance( jobID, basestring ):
try:
jobID = int( jobID )
except Exception as x:
return self._errorReport( str( x ), 'Expected integer or convertible integer for existing jobID' )
elif isinstance( jobID, list ):
try:
jobID = [int( job ) for job in jobID]
except Exception as x:
return self._errorReport( str( x ), 'Expected integer or convertible integer for existing jobIDs' )
jobManager = RPCClient( 'WorkloadManagement/JobManager', useCertificates = False )
result = jobManager.resetJob( jobID )
return result
#############################################################################
def getJobPilotOutput( self, jobID, directory = '' ):
"""Retrieve the pilot output for an existing job in the WMS.
The output will be retrieved in a local directory unless
otherwise specified.
>>> print dirac.getJobPilotOutput(12345)
{'OK': True, StdOut:'',StdError:''}
:param job: JobID
:type job: integer or string
:return: S_OK,S_ERROR
"""
if not directory:
directory = self.currentDir
if not os.path.exists( directory ):
return self._errorReport( 'Directory %s does not exist' % directory )
wmsAdmin = RPCClient( 'WorkloadManagement/WMSAdministrator' )
result = wmsAdmin.getJobPilotOutput( jobID )
if not result['OK']:
return result
outputPath = '%s/pilot_%s' % ( directory, jobID )
if os.path.exists( outputPath ):
self.log.info( 'Remove %s and retry to continue' % outputPath )
return S_ERROR( 'Remove %s and retry to continue' % outputPath )
if not os.path.exists( outputPath ):
self.log.verbose( 'Creating directory %s' % outputPath )
os.mkdir( outputPath )
outputs = result['Value']
if outputs.has_key( 'StdOut' ):
stdout = '%s/std.out' % ( outputPath )
with open( stdout, 'w' ) as fopen:
fopen.write( outputs['StdOut'] )
self.log.verbose( 'Standard output written to %s' % ( stdout ) )
else:
self.log.warn( 'No standard output returned' )
if outputs.has_key( 'StdError' ):
stderr = '%s/std.err' % ( outputPath )
with open( stderr, 'w' ) as fopen:
fopen.write( outputs['StdError'] )
self.log.verbose( 'Standard error written to %s' % ( stderr ) )
else:
self.log.warn( 'No standard error returned' )
self.log.always( 'Outputs retrieved in %s' % outputPath )
return result
#############################################################################
def getPilotOutput( self, gridReference, directory = '' ):
"""Retrieve the pilot output (std.out and std.err) for an existing job in the WMS.
>>> print dirac.getJobPilotOutput(12345)
{'OK': True, 'Value': {}}
:param job: JobID
:type job: integer or string
:return: S_OK,S_ERROR
"""
if not isinstance( gridReference, basestring ):
return self._errorReport( 'Expected string for pilot reference' )
if not directory:
directory = self.currentDir
if not os.path.exists( directory ):
return self._errorReport( 'Directory %s does not exist' % directory )
wmsAdmin = RPCClient( 'WorkloadManagement/WMSAdministrator' )
result = wmsAdmin.getPilotOutput( gridReference )
if not result['OK']:
return result
gridReferenceSmall = gridReference.split( '/' )[-1]
if not gridReferenceSmall:
gridReferenceSmall = 'reference'
outputPath = '%s/pilot_%s' % ( directory, gridReferenceSmall )
if os.path.exists( outputPath ):
self.log.info( 'Remove %s and retry to continue' % outputPath )
return S_ERROR( 'Remove %s and retry to continue' % outputPath )
if not os.path.exists( outputPath ):
self.log.verbose( 'Creating directory %s' % outputPath )
os.mkdir( outputPath )
outputs = result['Value']
if outputs.has_key( 'StdOut' ):
stdout = '%s/std.out' % ( outputPath )
with open( stdout, 'w' ) as fopen:
fopen.write( outputs['StdOut'] )
self.log.info( 'Standard output written to %s' % ( stdout ) )
else:
self.log.warn( 'No standard output returned' )
if outputs.has_key( 'StdErr' ):
stderr = '%s/std.err' % ( outputPath )
with open( stderr, 'w' ) as fopen:
fopen.write( outputs['StdErr'] )
self.log.info( 'Standard error written to %s' % ( stderr ) )
else:
self.log.warn( 'No standard error returned' )
self.log.always( 'Outputs retrieved in %s' % outputPath )
return result
#############################################################################
def getPilotInfo( self, gridReference ):
"""Retrieve info relative to a pilot reference
>>> print dirac.getPilotInfo(12345)
{'OK': True, 'Value': {}}
:param gridReference: Pilot Job Reference
:type gridReference: string
:return: S_OK,S_ERROR
"""
if not isinstance( gridReference, basestring ):
return self._errorReport( 'Expected string for pilot reference' )
wmsAdmin = RPCClient( 'WorkloadManagement/WMSAdministrator' )
result = wmsAdmin.getPilotInfo( gridReference )
return result
#############################################################################
def killPilot( self, gridReference ):
"""Kill the pilot specified
>>> print dirac.getPilotInfo(12345)
{'OK': True, 'Value': {}}
:param gridReference: Pilot Job Reference
:return: S_OK,S_ERROR
"""
if not isinstance( gridReference, basestring ):
return self._errorReport( 'Expected string for pilot reference' )
wmsAdmin = RPCClient( 'WorkloadManagement/WMSAdministrator' )
result = wmsAdmin.killPilot( gridReference )
return result
#############################################################################
def getPilotLoggingInfo( self, gridReference ):
"""Retrieve the pilot logging info for an existing job in the WMS.
>>> print dirac.getPilotLoggingInfo(12345)
{'OK': True, 'Value': {"The output of the command"}}
:param gridReference: Gridp pilot job reference Id
:type gridReference: string
:return: S_OK,S_ERROR
"""
if type( gridReference ) not in types.StringTypes:
return self._errorReport( 'Expected string for pilot reference' )
wmsAdmin = RPCClient( 'WorkloadManagement/WMSAdministrator' )
return wmsAdmin.getPilotLoggingInfo( gridReference )
#############################################################################
def getJobPilots( self, jobID ):
"""Extract the list of submitted pilots and their status for a given
jobID from the WMS. Useful information is printed to the screen.
>>> print dirac.getJobPilots()
{'OK': True, 'Value': {PilotID:{StatusDict}}}
:param job: JobID
:type job: integer or string
:return: S_OK,S_ERROR
"""
if isinstance( jobID, basestring ):
try:
jobID = int( jobID )
except Exception as x:
return self._errorReport( str( x ), 'Expected integer or string for existing jobID' )
wmsAdmin = RPCClient( 'WorkloadManagement/WMSAdministrator' )
result = wmsAdmin.getPilots( jobID )
if result['OK']:
print self.pPrint.pformat( result['Value'] )
return result
#############################################################################
def getPilotSummary( self, startDate = '', endDate = '' ):
"""Retrieve the pilot output for an existing job in the WMS. Summary is
printed at INFO level, full dictionary of results also returned.
>>> print dirac.getPilotSummary()
{'OK': True, 'Value': {CE:{Status:Count}}}
:param job: JobID
:type job: integer or string
:return: S_OK,S_ERROR
"""
wmsAdmin = RPCClient( 'WorkloadManagement/WMSAdministrator' )
result = wmsAdmin.getPilotSummary( startDate, endDate )
if not result['OK']:
return result
ceDict = result['Value']
headers = 'CE'.ljust( 28 )
i = 0
for ce, summary in ceDict.iteritems():
states = summary.keys()
if len( states ) > i:
i = len( states )
for i in xrange( i ):
headers += 'Status'.ljust( 12 ) + 'Count'.ljust( 12 )
print headers
for ce, summary in ceDict.iteritems():
line = ce.ljust( 28 )
states = summary.keys()
states.sort()
for state in states:
count = str( summary[state] )
line += state.ljust( 12 ) + count.ljust( 12 )
print line
return result
#############################################################################
def selectRequests( self, jobID = None, requestID = None, requestName = None,
requestType = None, status = None, operation = None, ownerDN = None,
ownerGroup = None, requestStart = 0, limit = 100, printOutput = False ):
"""Select requests from the request management system. A few notes on the selection criteria:
- jobID is the WMS JobID for the request (if applicable)
- requestID is assigned during submission of the request
- requestName is the corresponding XML file name
- requestType e.g. 'transfer'
- status e.g. Done
- operation e.g. replicateAndRegister
- requestStart e.g. the first request to consider (start from 0 by default)
- limit e.g. selection limit (default 100)
>>> dirac.selectRequests(jobID='4894')
{'OK': True, 'Value': [[<Requests>]]}
"""
options = {'RequestID':requestID, 'RequestName':requestName, 'JobID':jobID, 'OwnerDN':ownerDN,
'OwnerGroup':ownerGroup, 'RequestType':requestType, 'Status':status, 'Operation':operation}
conditions = {}
for key, value in options.iteritems():
if value:
try:
conditions[key] = str( value )
except Exception as x:
return self._errorReport( str( x ), 'Expected string for %s field' % key )
try:
requestStart = int( requestStart )
limit = int( limit )
except Exception as x:
return self._errorReport( str( x ), 'Expected integer for %s field' % limit )
self.log.verbose( 'Will select requests with the following conditions' )
self.log.verbose( self.pPrint.pformat( conditions ) )
requestClient = RPCClient( "RequestManagement/centralURL" )
result = requestClient.getRequestSummaryWeb( conditions, [], requestStart, limit )
if not result['OK']:
self.log.warn( result['Message'] )
return result
requestIDs = result['Value']
conds = []
for key, value in conditions.iteritems():
if value:
conds.append( '%s = %s' % ( key, value ) )
self.log.verbose( '%s request(s) selected with conditions %s and limit %s' % ( len( requestIDs['Records'] ),
', '.join( conds ), limit ) )
if printOutput:
requests = []
if len( requestIDs['Records'] ) > limit:
requestList = requestIDs['Records']
requests = requestList[:limit]
else:
requests = requestIDs['Records']
print '%s request(s) selected with conditions %s and limit %s' % ( len( requestIDs['Records'] ),
', '.join( conds ), limit )
print requestIDs['ParameterNames']
for request in requests:
print request
if not requestIDs:
return S_ERROR( 'No requests selected for conditions: %s' % conditions )
else:
return result
#############################################################################
def getRequestSummary( self, printOutput = False ):
"""
Get a summary of the requests in the request DB.
"""
requestClient = RPCClient( "RequestManagement/centralURL", timeout = 120 )
result = requestClient.getDBSummary()
if not result['OK']:
self.log.warn( result['Message'] )
return result
if printOutput:
print self.pPrint.pformat( result['Value'] )
return result
#############################################################################
def getExternalPackageVersions( self ):
"""
Simple function that attempts to obtain the external versions for
the local DIRAC installation (frequently needed for debugging purposes).
"""
gLogger.info( 'DIRAC version v%dr%d build %d' % ( DIRAC.majorVersion, DIRAC.minorVersion, DIRAC.patchLevel ) )
try:
import lcg_util
infoStr = 'Using lcg_util from: \n%s' % lcg_util.__file__
gLogger.info( infoStr )
infoStr = "The version of lcg_utils is %s" % lcg_util.lcg_util_version()
gLogger.info( infoStr )
except Exception as x:
errStr = "SRM2Storage.__init__: Failed to import lcg_util: %s" % ( x )
gLogger.exception( errStr )
try:
import gfalthr as gfal
infoStr = "Using gfalthr from: \n%s" % gfal.__file__
gLogger.info( infoStr )
infoStr = "The version of gfalthr is %s" % gfal.gfal_version()
gLogger.info( infoStr )
except Exception as x:
errStr = "SRM2Storage.__init__: Failed to import gfalthr: %s." % ( x )
gLogger.warn( errStr )
try:
import gfal
infoStr = "Using gfal from: %s" % gfal.__file__
gLogger.info( infoStr )
infoStr = "The version of gfal is %s" % gfal.gfal_version()
gLogger.info( infoStr )
except Exception as x:
errStr = "SRM2Storage.__init__: Failed to import gfal: %s" % ( x )
gLogger.exception( errStr )
defaultProtocols = gConfig.getValue( '/Resources/StorageElements/DefaultProtocols', [] )
gLogger.info( 'Default list of protocols are: %s' % ( ', '.join( defaultProtocols ) ) )
return S_OK()
#############################################################################
def getSiteProtocols( self, site, printOutput = False ):
"""
Allows to check the defined protocols for each site SE.
"""
result = self.__checkSiteIsValid( site )
if not result['OK']:
return result
siteSection = '/Resources/Sites/%s/%s/SE' % ( site.split( '.' )[0], site )
siteSEs = gConfig.getValue( siteSection, [] )
if not siteSEs:
return S_ERROR( 'No SEs found for site %s in section %s' % ( site, siteSection ) )
defaultProtocols = gConfig.getValue( '/Resources/StorageElements/DefaultProtocols', [] )
self.log.verbose( 'Default list of protocols are' ', '.join( defaultProtocols ) )
seInfo = {}
siteSEs.sort()
for se in siteSEs:
sections = gConfig.getSections( '/Resources/StorageElements/%s/' % ( se ) )
if not sections['OK']:
return sections
for section in sections['Value']:
if gConfig.getValue( '/Resources/StorageElements/%s/%s/ProtocolName' % ( se, section ), '' ) == 'SRM2':
path = '/Resources/StorageElements/%s/%s/ProtocolsList' % ( se, section )
seProtocols = gConfig.getValue( path, [] )
if not seProtocols:
seProtocols = defaultProtocols
seInfo[se] = seProtocols
if printOutput:
print '\nSummary of protocols for StorageElements at site %s' % site
print '\nStorageElement'.ljust( 30 ) + 'ProtocolsList'.ljust( 30 ) + '\n'
for se, protocols in seInfo.iteritems():
print se.ljust( 30 ) + ', '.join( protocols ).ljust( 30 )
return S_OK( seInfo )
#############################################################################
def setSiteProtocols( self, site, protocolsList, printOutput = False ):
"""
Allows to set the defined protocols for each SE for a given site.
"""
result = self.__checkSiteIsValid( site )
if not result['OK']:
return result
siteSection = '/Resources/Sites/%s/%s/SE' % ( site.split( '.' )[0], site )
siteSEs = gConfig.getValue( siteSection, [] )
if not siteSEs:
return S_ERROR( 'No SEs found for site %s in section %s' % ( site, siteSection ) )
defaultProtocols = gConfig.getValue( '/Resources/StorageElements/DefaultProtocols', [] )
self.log.verbose( 'Default list of protocols are', ', '.join( defaultProtocols ) )
for protocol in protocolsList:
if not protocol in defaultProtocols:
return S_ERROR( 'Requested to set protocol %s in list but %s is not '
'in default list of protocols:\n%s' % ( protocol, protocol, ', '.join( defaultProtocols ) ) )
modifiedCS = False
result = promptUser( 'Do you want to add the following default protocols:'
' %s for SE(s):\n%s' % ( ', '.join( protocolsList ), ', '.join( siteSEs ) ) )
if not result['OK']:
return result
if result['Value'].lower() != 'y':
self.log.always( 'No protocols will be added' )
return S_OK()
for se in siteSEs:
sections = gConfig.getSections( '/Resources/StorageElements/%s/' % ( se ) )
if not sections['OK']:
return sections
for section in sections['Value']:
if gConfig.getValue( '/Resources/StorageElements/%s/%s/ProtocolName' % ( se, section ), '' ) == 'SRM2':
path = '/Resources/StorageElements/%s/%s/ProtocolsList' % ( se, section )
self.log.verbose( 'Setting %s to %s' % ( path, ', '.join( protocolsList ) ) )
result = self.csSetOption( path, ', '.join( protocolsList ) )
if not result['OK']:
return result
modifiedCS = True
if modifiedCS:
result = self.csCommitChanges( False )
if not result[ 'OK' ]:
return S_ERROR( 'CS Commit failed with message = %s' % ( result[ 'Message' ] ) )
else:
if printOutput:
print 'Successfully committed changes to CS'
else:
if printOutput:
print 'No modifications to CS required'
return S_OK()
#############################################################################
def csSetOption( self, optionPath, optionValue ):
"""
Function to modify an existing value in the CS.
"""
return self.csAPI.setOption( optionPath, optionValue )
#############################################################################
def csSetOptionComment( self, optionPath, comment ):
"""
Function to modify an existing value in the CS.
"""
return self.csAPI.setOptionComment( optionPath, comment )
#############################################################################
def csModifyValue( self, optionPath, newValue ):
"""
Function to modify an existing value in the CS.
"""
return self.csAPI.modifyValue( optionPath, newValue )
#############################################################################
def csRegisterUser( self, username, properties ):
"""
Registers a user in the CS.
- username: Username of the user (easy;)
- properties: Dict containing:
- DN
- groups : list/tuple of groups the user belongs to
- <others> : More properties of the user, like mail
"""
return self.csAPI.addUser( username, properties )
#############################################################################
def csDeleteUser( self, user ):
"""
Deletes a user from the CS. Can take a list of users
"""
return self.csAPI.deleteUsers( user )
#############################################################################
def csModifyUser( self, username, properties, createIfNonExistant = False ):
"""
Modify a user in the CS. Takes the same params as in addUser and
applies the changes
"""
return self.csAPI.modifyUser( username, properties, createIfNonExistant )
#############################################################################
def csListUsers( self, group = False ):
"""
Lists the users in the CS. If no group is specified return all users.
"""
return self.csAPI.listUsers( group )
#############################################################################
def csDescribeUsers( self, mask = False ):
"""
List users and their properties in the CS.
If a mask is given, only users in the mask will be returned
"""
return self.csAPI.describeUsers( mask )
#############################################################################
def csModifyGroup( self, groupname, properties, createIfNonExistant = False ):
"""
Modify a user in the CS. Takes the same params as in addGroup and applies
the changes
"""
return self.csAPI.modifyGroup( groupname, properties, createIfNonExistant )
#############################################################################
def csListHosts( self ):
"""
Lists the hosts in the CS
"""
return self.csAPI.listHosts()
#############################################################################
def csDescribeHosts( self, mask = False ):
"""
Gets extended info for the hosts in the CS
"""
return self.csAPI.describeHosts( mask )
#############################################################################
def csModifyHost( self, hostname, properties, createIfNonExistant = False ):
"""
Modify a host in the CS. Takes the same params as in addHost and applies
the changes
"""
return self.csAPI.modifyHost( hostname, properties, createIfNonExistant )
#############################################################################
def csListGroups( self ):
"""
Lists groups in the CS
"""
return self.csAPI.listGroups()
#############################################################################
def csDescribeGroups( self, mask = False ):
"""
List groups and their properties in the CS.
If a mask is given, only groups in the mask will be returned
"""
return self.csAPI.describeGroups( mask )
#############################################################################
def csSyncUsersWithCFG( self, usersCFG ):
"""
Synchronize users in cfg with its contents
"""
return self.csAPI.syncUsersWithCFG( usersCFG )
#############################################################################
def csCommitChanges( self, sortUsers = True ):
"""
Commit the changes in the CS
"""
return self.csAPI.commitChanges( sortUsers = False )
#############################################################################
def sendMail( self, address, subject, body, fromAddress = None, localAttempt = True, html = False ):
"""
Send mail to specified address with body.
"""
notification = NotificationClient()
return notification.sendMail( address, subject, body, fromAddress, localAttempt, html )
#############################################################################
def sendSMS( self, userName, body, fromAddress = None ):
"""
Send mail to specified address with body.
"""
if len( body ) > 160:
return S_ERROR( 'Exceeded maximum SMS length of 160 characters' )
notification = NotificationClient()
return notification.sendSMS( userName, body, fromAddress )
#############################################################################
def getBDIISite( self, site, host = None ):
"""
Get information about site from BDII at host
"""
return ldapSite( site, host = host )
#############################################################################
def getBDIICluster( self, ce, host = None ):
"""
Get information about ce from BDII at host
"""
return ldapCluster( ce, host = host )
#############################################################################
def getBDIICE( self, ce, host = None ):
"""
Get information about ce from BDII at host
"""
return ldapCE( ce, host = host )
#############################################################################
def getBDIIService( self, ce, host = None ):
"""
Get information about ce from BDII at host
"""
return ldapService( ce, host = host )
#############################################################################
def getBDIICEState( self, ce, useVO = voName, host = None ):
"""
Get information about ce state from BDII at host
"""
return ldapCEState( ce, useVO, host = host )
#############################################################################
def getBDIICEVOView( self, ce, useVO = voName, host = None ):
"""
Get information about ce voview from BDII at host
"""
return ldapCEVOView( ce, useVO, host = host )
#############################################################################
def getBDIISE( self, site, useVO = voName, host = None ):
"""
Get information about SA from BDII at host
"""
return ldapSE( site, useVO, host = host )
