class DiracAdmin(API): """ Administrative functionalities """ ############################################################################# def __init__(self): """Internal initialization of the DIRAC Admin API. """ super(DiracAdmin, self).__init__() self.csAPI = CSAPI() self.dbg = False if gConfig.getValue(self.section + '/LogLevel', 'DEBUG') == 'DEBUG': self.dbg = True self.scratchDir = gConfig.getValue(self.section + '/ScratchDir', '/tmp') self.currentDir = os.getcwd() self.rssFlag = ResourceStatus().rssFlag self.sitestatus = SiteStatus() ############################################################################# def uploadProxy(self, group): """Upload a proxy to the DIRAC WMS. This method Example usage: >>> print diracAdmin.uploadProxy('lhcb_pilot') {'OK': True, 'Value': 0L} :param group: DIRAC Group :type job: string :return: S_OK,S_ERROR :param permanent: Indefinitely update proxy :type permanent: boolean """ return gProxyManager.uploadProxy(diracGroup=group) ############################################################################# def setProxyPersistency(self, userDN, userGroup, persistent=True): """Set the persistence of a proxy in the Proxy Manager Example usage: >>> print diracAdmin.setProxyPersistency( 'some DN', 'dirac group', True ) {'OK': True } :param userDN: User DN :type userDN: string :param userGroup: DIRAC Group :type userGroup: string :param persistent: Persistent flag :type persistent: boolean :return: S_OK,S_ERROR """ return gProxyManager.setPersistency(userDN, userGroup, persistent) ############################################################################# def checkProxyUploaded(self, userDN, userGroup, requiredTime): """Set the persistence of a proxy in the Proxy Manager Example usage: >>> print diracAdmin.setProxyPersistency( 'some DN', 'dirac group', True ) {'OK': True, 'Value' : True/False } :param userDN: User DN :type userDN: string :param userGroup: DIRAC Group :type userGroup: string :param requiredTime: Required life time of the uploaded proxy :type requiredTime: boolean :return: S_OK,S_ERROR """ return gProxyManager.userHasProxy(userDN, userGroup, requiredTime) ############################################################################# def getSiteMask(self, printOutput=False, status='Active'): """Retrieve current site mask from WMS Administrator service. Example usage: >>> print diracAdmin.getSiteMask() {'OK': True, 'Value': 0L} :return: S_OK,S_ERROR """ result = self.sitestatus.getSites(siteState=status) if result['OK']: sites = result['Value'] if printOutput: sites.sort() for site in sites: print site return result ############################################################################# def getBannedSites(self, gridType=[], printOutput=False): """Retrieve current list of banned and probing sites. Example usage: >>> print diracAdmin.getBannedSites() {'OK': True, 'Value': []} :return: S_OK,S_ERROR """ bannedSites = self.sitestatus.getSites(siteState='Banned') if not bannedSites['OK']: return bannedSites probingSites = self.sitestatus.getSites(siteState='Probing') if not probingSites['OK']: return probingSites mergedList = bannedSites['Value'] + probingSites['Value'] mergedList.sort() if printOutput: print '\n'.join(mergedList) return S_OK(mergedList) ############################################################################# def getSiteSection(self, site, printOutput=False): """Simple utility to get the list of CEs for DIRAC site name. Example usage: >>> print diracAdmin.getSiteSection('LCG.CERN.ch') {'OK': True, 'Value':} :return: S_OK,S_ERROR """ gridType = site.split('.')[0] if not gConfig.getSections('/Resources/Sites/%s' % (gridType))['OK']: return S_ERROR('/Resources/Sites/%s is not a valid site section' % (gridType)) result = gConfig.getOptionsDict('/Resources/Sites/%s/%s' % (gridType, site)) if printOutput and result['OK']: print self.pPrint.pformat(result['Value']) return result ############################################################################# def allowSite(self, site, comment, printOutput=False): """Adds the site to the site mask. Example usage: >>> print diracAdmin.allowSite() {'OK': True, 'Value': } :return: S_OK,S_ERROR """ result = self.__checkSiteIsValid(site) if not result['OK']: return result result = self.getSiteMask(status='Active') if not result['OK']: return result siteMask = result['Value'] if site in siteMask: if printOutput: print 'Site %s is already Active' % site return S_OK('Site %s is already Active' % site) if self.rssFlag: result = self.sitestatus.setSiteStatus(site, 'Active', comment) else: wmsAdmin = RPCClient('WorkloadManagement/WMSAdministrator') result = wmsAdmin.allowSite(site, comment) if not result['OK']: return result if printOutput: print 'Site %s status is set to Active' % site return result ############################################################################# def getSiteMaskLogging(self, site=None, printOutput=False): """Retrieves site mask logging information. Example usage: >>> print diracAdmin.getSiteMaskLogging('LCG.AUVER.fr') {'OK': True, 'Value': } :return: S_OK,S_ERROR """ result = self.__checkSiteIsValid(site) if not result['OK']: return result wmsAdmin = RPCClient('WorkloadManagement/WMSAdministrator') result = wmsAdmin.getSiteMaskLogging(site) if not result['OK']: return result if site: if not result['Value'].has_key(site): return S_ERROR('Site mask information not available for %s' % (site)) if printOutput: if site: print '\nSite Mask Logging Info for %s\n' % site else: print '\nAll Site Mask Logging Info\n' siteDict = result['Value'] for site, tupleList in siteDict.iteritems(): if not site: print '\n===> %s\n' % site for tup in tupleList: print str( tup[0] ).ljust( 8 ) + str( tup[1] ).ljust( 20 ) + \ '( ' + str( tup[2] ).ljust( len( str( tup[2] ) ) ) + ' ) "' + str( tup[3] ) + '"' print ' ' return result ############################################################################# def banSite(self, site, comment, printOutput=False): """Removes the site from the site mask. Example usage: >>> print diracAdmin.banSite() {'OK': True, 'Value': } :return: S_OK,S_ERROR """ result = self.__checkSiteIsValid(site) if not result['OK']: return result mask = self.getSiteMask(status='Banned') if not mask['OK']: return mask siteMask = mask['Value'] if site in siteMask: if printOutput: print 'Site %s is already Banned' % site return S_OK('Site %s is already Banned' % site) if self.rssFlag: result = self.sitestatus.setSiteStatus(site, 'Banned', comment) else: wmsAdmin = RPCClient('WorkloadManagement/WMSAdministrator') result = wmsAdmin.banSite(site, comment) if not result['OK']: return result if printOutput: print 'Site %s status is set to Banned' % site return result ############################################################################# def __checkSiteIsValid(self, site): """Internal function to check that a site name is valid. """ sites = getSiteCEMapping() if not sites['OK']: return S_ERROR('Could not get site CE mapping') siteList = sites['Value'].keys() if not site in siteList: return S_ERROR( 'Specified site %s is not in list of defined sites' % site) return S_OK('%s is valid' % site) ############################################################################# def clearMask(self): """Removes all sites from the site mask. Should be used with care. Example usage: >>> print diracAdmin.clearMask() {'OK': True, 'Value':''} :return: S_OK,S_ERROR """ wmsAdmin = RPCClient('WorkloadManagement/WMSAdministrator') result = wmsAdmin.clearMask() return result ############################################################################# def getServicePorts(self, setup='', printOutput=False): """Checks the service ports for the specified setup. If not given this is taken from the current installation (/DIRAC/Setup) Example usage: >>> print diracAdmin.getServicePorts() {'OK': True, 'Value':''} :return: S_OK,S_ERROR """ if not setup: setup = gConfig.getValue('/DIRAC/Setup', '') setupList = gConfig.getSections('/DIRAC/Setups', []) if not setupList['OK']: return S_ERROR('Could not get /DIRAC/Setups sections') setupList = setupList['Value'] if not setup in setupList: return S_ERROR('Setup %s is not in allowed list: %s' % (setup, ', '.join(setupList))) serviceSetups = gConfig.getOptionsDict('/DIRAC/Setups/%s' % setup) if not serviceSetups['OK']: return S_ERROR('Could not get /DIRAC/Setups/%s options' % setup) serviceSetups = serviceSetups['Value'] # dict systemList = gConfig.getSections('/Systems') if not systemList['OK']: return S_ERROR('Could not get Systems sections') systemList = systemList['Value'] result = {} for system in systemList: if serviceSetups.has_key(system): path = '/Systems/%s/%s/Services' % (system, serviceSetups[system]) servicesList = gConfig.getSections(path) if not servicesList['OK']: self.log.warn('Could not get sections in %s' % path) else: servicesList = servicesList['Value'] if not servicesList: servicesList = [] self.log.verbose('System: %s ServicesList: %s' % (system, ', '.join(servicesList))) for service in servicesList: spath = '%s/%s/Port' % (path, service) servicePort = gConfig.getValue(spath, 0) if servicePort: self.log.verbose('Found port for %s/%s = %s' % (system, service, servicePort)) result['%s/%s' % (system, service)] = servicePort else: self.log.warn('No port found for %s' % spath) else: self.log.warn('%s is not defined in /DIRAC/Setups/%s' % (system, setup)) if printOutput: print self.pPrint.pformat(result) return S_OK(result) ############################################################################# def getProxy(self, userDN, userGroup, validity=43200, limited=False): """Retrieves a proxy with default 12hr validity and stores this in a file in the local directory by default. Example usage: >>> print diracAdmin.getProxy() {'OK': True, 'Value': } :return: S_OK,S_ERROR """ return gProxyManager.downloadProxy(userDN, userGroup, limited=limited, requiredTimeLeft=validity) ############################################################################# def getVOMSProxy(self, userDN, userGroup, vomsAttr=False, validity=43200, limited=False): """Retrieves a proxy with default 12hr validity and VOMS extensions and stores this in a file in the local directory by default. Example usage: >>> print diracAdmin.getVOMSProxy() {'OK': True, 'Value': } :return: S_OK,S_ERROR """ return gProxyManager.downloadVOMSProxy(userDN, userGroup, limited=limited, requiredVOMSAttribute=vomsAttr, requiredTimeLeft=validity) ############################################################################# def getPilotProxy(self, userDN, userGroup, validity=43200): """Retrieves a pilot proxy with default 12hr validity and stores this in a file in the local directory by default. Example usage: >>> print diracAdmin.getVOMSProxy() {'OK': True, 'Value': } :return: S_OK,S_ERROR """ return gProxyManager.getPilotProxyFromDIRACGroup( userDN, userGroup, requiredTimeLeft=validity) ############################################################################# def resetJob(self, jobID): """Reset a job or list of jobs in the WMS. This operation resets the reschedule counter for a job or list of jobs and allows them to run as new. Example:: >>> print dirac.reset(12345) {'OK': True, 'Value': [12345]} :param job: JobID :type job: integer or list of integers :return: S_OK,S_ERROR """ if isinstance(jobID, basestring): try: jobID = int(jobID) except Exception as x: return self._errorReport( str(x), 'Expected integer or convertible integer for existing jobID' ) elif isinstance(jobID, list): try: jobID = [int(job) for job in jobID] except Exception as x: return self._errorReport( str(x), 'Expected integer or convertible integer for existing jobIDs' ) jobManager = RPCClient('WorkloadManagement/JobManager', useCertificates=False) result = jobManager.resetJob(jobID) return result ############################################################################# def getJobPilotOutput(self, jobID, directory=''): """Retrieve the pilot output for an existing job in the WMS. The output will be retrieved in a local directory unless otherwise specified. >>> print dirac.getJobPilotOutput(12345) {'OK': True, StdOut:'',StdError:''} :param job: JobID :type job: integer or string :return: S_OK,S_ERROR """ if not directory: directory = self.currentDir if not os.path.exists(directory): return self._errorReport('Directory %s does not exist' % directory) wmsAdmin = RPCClient('WorkloadManagement/WMSAdministrator') result = wmsAdmin.getJobPilotOutput(jobID) if not result['OK']: return result outputPath = '%s/pilot_%s' % (directory, jobID) if os.path.exists(outputPath): self.log.info('Remove %s and retry to continue' % outputPath) return S_ERROR('Remove %s and retry to continue' % outputPath) if not os.path.exists(outputPath): self.log.verbose('Creating directory %s' % outputPath) os.mkdir(outputPath) outputs = result['Value'] if outputs.has_key('StdOut'): stdout = '%s/std.out' % (outputPath) with open(stdout, 'w') as fopen: fopen.write(outputs['StdOut']) self.log.verbose('Standard output written to %s' % (stdout)) else: self.log.warn('No standard output returned') if outputs.has_key('StdError'): stderr = '%s/std.err' % (outputPath) with open(stderr, 'w') as fopen: fopen.write(outputs['StdError']) self.log.verbose('Standard error written to %s' % (stderr)) else: self.log.warn('No standard error returned') self.log.always('Outputs retrieved in %s' % outputPath) return result ############################################################################# def getPilotOutput(self, gridReference, directory=''): """Retrieve the pilot output (std.out and std.err) for an existing job in the WMS. >>> print dirac.getJobPilotOutput(12345) {'OK': True, 'Value': {}} :param job: JobID :type job: integer or string :return: S_OK,S_ERROR """ if not isinstance(gridReference, basestring): return self._errorReport('Expected string for pilot reference') if not directory: directory = self.currentDir if not os.path.exists(directory): return self._errorReport('Directory %s does not exist' % directory) wmsAdmin = RPCClient('WorkloadManagement/WMSAdministrator') result = wmsAdmin.getPilotOutput(gridReference) if not result['OK']: return result gridReferenceSmall = gridReference.split('/')[-1] if not gridReferenceSmall: gridReferenceSmall = 'reference' outputPath = '%s/pilot_%s' % (directory, gridReferenceSmall) if os.path.exists(outputPath): self.log.info('Remove %s and retry to continue' % outputPath) return S_ERROR('Remove %s and retry to continue' % outputPath) if not os.path.exists(outputPath): self.log.verbose('Creating directory %s' % outputPath) os.mkdir(outputPath) outputs = result['Value'] if outputs.has_key('StdOut'): stdout = '%s/std.out' % (outputPath) with open(stdout, 'w') as fopen: fopen.write(outputs['StdOut']) self.log.info('Standard output written to %s' % (stdout)) else: self.log.warn('No standard output returned') if outputs.has_key('StdErr'): stderr = '%s/std.err' % (outputPath) with open(stderr, 'w') as fopen: fopen.write(outputs['StdErr']) self.log.info('Standard error written to %s' % (stderr)) else: self.log.warn('No standard error returned') self.log.always('Outputs retrieved in %s' % outputPath) return result ############################################################################# def getPilotInfo(self, gridReference): """Retrieve info relative to a pilot reference >>> print dirac.getPilotInfo(12345) {'OK': True, 'Value': {}} :param gridReference: Pilot Job Reference :type gridReference: string :return: S_OK,S_ERROR """ if not isinstance(gridReference, basestring): return self._errorReport('Expected string for pilot reference') wmsAdmin = RPCClient('WorkloadManagement/WMSAdministrator') result = wmsAdmin.getPilotInfo(gridReference) return result ############################################################################# def killPilot(self, gridReference): """Kill the pilot specified >>> print dirac.getPilotInfo(12345) {'OK': True, 'Value': {}} :param gridReference: Pilot Job Reference :return: S_OK,S_ERROR """ if not isinstance(gridReference, basestring): return self._errorReport('Expected string for pilot reference') wmsAdmin = RPCClient('WorkloadManagement/WMSAdministrator') result = wmsAdmin.killPilot(gridReference) return result ############################################################################# def getPilotLoggingInfo(self, gridReference): """Retrieve the pilot logging info for an existing job in the WMS. >>> print dirac.getPilotLoggingInfo(12345) {'OK': True, 'Value': {"The output of the command"}} :param gridReference: Gridp pilot job reference Id :type gridReference: string :return: S_OK,S_ERROR """ if type(gridReference) not in types.StringTypes: return self._errorReport('Expected string for pilot reference') wmsAdmin = RPCClient('WorkloadManagement/WMSAdministrator') return wmsAdmin.getPilotLoggingInfo(gridReference) ############################################################################# def getJobPilots(self, jobID): """Extract the list of submitted pilots and their status for a given jobID from the WMS. Useful information is printed to the screen. >>> print dirac.getJobPilots() {'OK': True, 'Value': {PilotID:{StatusDict}}} :param job: JobID :type job: integer or string :return: S_OK,S_ERROR """ if isinstance(jobID, basestring): try: jobID = int(jobID) except Exception as x: return self._errorReport( str(x), 'Expected integer or string for existing jobID') wmsAdmin = RPCClient('WorkloadManagement/WMSAdministrator') result = wmsAdmin.getPilots(jobID) if result['OK']: print self.pPrint.pformat(result['Value']) return result ############################################################################# def getPilotSummary(self, startDate='', endDate=''): """Retrieve the pilot output for an existing job in the WMS. Summary is printed at INFO level, full dictionary of results also returned. >>> print dirac.getPilotSummary() {'OK': True, 'Value': {CE:{Status:Count}}} :param job: JobID :type job: integer or string :return: S_OK,S_ERROR """ wmsAdmin = RPCClient('WorkloadManagement/WMSAdministrator') result = wmsAdmin.getPilotSummary(startDate, endDate) if not result['OK']: return result ceDict = result['Value'] headers = 'CE'.ljust(28) i = 0 for ce, summary in ceDict.iteritems(): states = summary.keys() if len(states) > i: i = len(states) for i in xrange(i): headers += 'Status'.ljust(12) + 'Count'.ljust(12) print headers for ce, summary in ceDict.iteritems(): line = ce.ljust(28) states = summary.keys() states.sort() for state in states: count = str(summary[state]) line += state.ljust(12) + count.ljust(12) print line return result ############################################################################# def selectRequests(self, jobID=None, requestID=None, requestName=None, requestType=None, status=None, operation=None, ownerDN=None, ownerGroup=None, requestStart=0, limit=100, printOutput=False): """Select requests from the request management system. A few notes on the selection criteria: - jobID is the WMS JobID for the request (if applicable) - requestID is assigned during submission of the request - requestName is the corresponding XML file name - requestType e.g. 'transfer' - status e.g. Done - operation e.g. replicateAndRegister - requestStart e.g. the first request to consider (start from 0 by default) - limit e.g. selection limit (default 100) >>> dirac.selectRequests(jobID='4894') {'OK': True, 'Value': [[<Requests>]]} """ options = { 'RequestID': requestID, 'RequestName': requestName, 'JobID': jobID, 'OwnerDN': ownerDN, 'OwnerGroup': ownerGroup, 'RequestType': requestType, 'Status': status, 'Operation': operation } conditions = {} for key, value in options.iteritems(): if value: try: conditions[key] = str(value) except Exception as x: return self._errorReport( str(x), 'Expected string for %s field' % key) try: requestStart = int(requestStart) limit = int(limit) except Exception as x: return self._errorReport(str(x), 'Expected integer for %s field' % limit) self.log.verbose('Will select requests with the following conditions') self.log.verbose(self.pPrint.pformat(conditions)) requestClient = RPCClient("RequestManagement/centralURL") result = requestClient.getRequestSummaryWeb(conditions, [], requestStart, limit) if not result['OK']: self.log.warn(result['Message']) return result requestIDs = result['Value'] conds = [] for key, value in conditions.iteritems(): if value: conds.append('%s = %s' % (key, value)) self.log.verbose( '%s request(s) selected with conditions %s and limit %s' % (len(requestIDs['Records']), ', '.join(conds), limit)) if printOutput: requests = [] if len(requestIDs['Records']) > limit: requestList = requestIDs['Records'] requests = requestList[:limit] else: requests = requestIDs['Records'] print '%s request(s) selected with conditions %s and limit %s' % ( len(requestIDs['Records']), ', '.join(conds), limit) print requestIDs['ParameterNames'] for request in requests: print request if not requestIDs: return S_ERROR('No requests selected for conditions: %s' % conditions) else: return result ############################################################################# def getRequestSummary(self, printOutput=False): """ Get a summary of the requests in the request DB. """ requestClient = RPCClient("RequestManagement/centralURL", timeout=120) result = requestClient.getDBSummary() if not result['OK']: self.log.warn(result['Message']) return result if printOutput: print self.pPrint.pformat(result['Value']) return result ############################################################################# def getExternalPackageVersions(self): """ Simple function that attempts to obtain the external versions for the local DIRAC installation (frequently needed for debugging purposes). """ gLogger.info( 'DIRAC version v%dr%d build %d' % (DIRAC.majorVersion, DIRAC.minorVersion, DIRAC.patchLevel)) try: import lcg_util infoStr = 'Using lcg_util from: \n%s' % lcg_util.__file__ gLogger.info(infoStr) infoStr = "The version of lcg_utils is %s" % lcg_util.lcg_util_version( ) gLogger.info(infoStr) except Exception as x: errStr = "SRM2Storage.__init__: Failed to import lcg_util: %s" % ( x) gLogger.exception(errStr) try: import gfalthr as gfal infoStr = "Using gfalthr from: \n%s" % gfal.__file__ gLogger.info(infoStr) infoStr = "The version of gfalthr is %s" % gfal.gfal_version() gLogger.info(infoStr) except Exception as x: errStr = "SRM2Storage.__init__: Failed to import gfalthr: %s." % ( x) gLogger.warn(errStr) try: import gfal infoStr = "Using gfal from: %s" % gfal.__file__ gLogger.info(infoStr) infoStr = "The version of gfal is %s" % gfal.gfal_version() gLogger.info(infoStr) except Exception as x: errStr = "SRM2Storage.__init__: Failed to import gfal: %s" % ( x) gLogger.exception(errStr) defaultProtocols = gConfig.getValue( '/Resources/StorageElements/DefaultProtocols', []) gLogger.info('Default list of protocols are: %s' % (', '.join(defaultProtocols))) return S_OK() ############################################################################# def getSiteProtocols(self, site, printOutput=False): """ Allows to check the defined protocols for each site SE. """ result = self.__checkSiteIsValid(site) if not result['OK']: return result siteSection = '/Resources/Sites/%s/%s/SE' % (site.split('.')[0], site) siteSEs = gConfig.getValue(siteSection, []) if not siteSEs: return S_ERROR('No SEs found for site %s in section %s' % (site, siteSection)) defaultProtocols = gConfig.getValue( '/Resources/StorageElements/DefaultProtocols', []) self.log.verbose('Default list of protocols are' ', '.join(defaultProtocols)) seInfo = {} siteSEs.sort() for se in siteSEs: sections = gConfig.getSections('/Resources/StorageElements/%s/' % (se)) if not sections['OK']: return sections for section in sections['Value']: if gConfig.getValue( '/Resources/StorageElements/%s/%s/ProtocolName' % (se, section), '') == 'SRM2': path = '/Resources/StorageElements/%s/%s/ProtocolsList' % ( se, section) seProtocols = gConfig.getValue(path, []) if not seProtocols: seProtocols = defaultProtocols seInfo[se] = seProtocols if printOutput: print '\nSummary of protocols for StorageElements at site %s' % site print '\nStorageElement'.ljust(30) + 'ProtocolsList'.ljust( 30) + '\n' for se, protocols in seInfo.iteritems(): print se.ljust(30) + ', '.join(protocols).ljust(30) return S_OK(seInfo) ############################################################################# def setSiteProtocols(self, site, protocolsList, printOutput=False): """ Allows to set the defined protocols for each SE for a given site. """ result = self.__checkSiteIsValid(site) if not result['OK']: return result siteSection = '/Resources/Sites/%s/%s/SE' % (site.split('.')[0], site) siteSEs = gConfig.getValue(siteSection, []) if not siteSEs: return S_ERROR('No SEs found for site %s in section %s' % (site, siteSection)) defaultProtocols = gConfig.getValue( '/Resources/StorageElements/DefaultProtocols', []) self.log.verbose('Default list of protocols are', ', '.join(defaultProtocols)) for protocol in protocolsList: if not protocol in defaultProtocols: return S_ERROR( 'Requested to set protocol %s in list but %s is not ' 'in default list of protocols:\n%s' % (protocol, protocol, ', '.join(defaultProtocols))) modifiedCS = False result = promptUser( 'Do you want to add the following default protocols:' ' %s for SE(s):\n%s' % (', '.join(protocolsList), ', '.join(siteSEs))) if not result['OK']: return result if result['Value'].lower() != 'y': self.log.always('No protocols will be added') return S_OK() for se in siteSEs: sections = gConfig.getSections('/Resources/StorageElements/%s/' % (se)) if not sections['OK']: return sections for section in sections['Value']: if gConfig.getValue( '/Resources/StorageElements/%s/%s/ProtocolName' % (se, section), '') == 'SRM2': path = '/Resources/StorageElements/%s/%s/ProtocolsList' % ( se, section) self.log.verbose('Setting %s to %s' % (path, ', '.join(protocolsList))) result = self.csSetOption(path, ', '.join(protocolsList)) if not result['OK']: return result modifiedCS = True if modifiedCS: result = self.csCommitChanges(False) if not result['OK']: return S_ERROR('CS Commit failed with message = %s' % (result['Message'])) else: if printOutput: print 'Successfully committed changes to CS' else: if printOutput: print 'No modifications to CS required' return S_OK() ############################################################################# def csSetOption(self, optionPath, optionValue): """ Function to modify an existing value in the CS. """ return self.csAPI.setOption(optionPath, optionValue) ############################################################################# def csSetOptionComment(self, optionPath, comment): """ Function to modify an existing value in the CS. """ return self.csAPI.setOptionComment(optionPath, comment) ############################################################################# def csModifyValue(self, optionPath, newValue): """ Function to modify an existing value in the CS. """ return self.csAPI.modifyValue(optionPath, newValue) ############################################################################# def csRegisterUser(self, username, properties): """ Registers a user in the CS. - username: Username of the user (easy;) - properties: Dict containing: - DN - groups : list/tuple of groups the user belongs to - <others> : More properties of the user, like mail """ return self.csAPI.addUser(username, properties) ############################################################################# def csDeleteUser(self, user): """ Deletes a user from the CS. Can take a list of users """ return self.csAPI.deleteUsers(user) ############################################################################# def csModifyUser(self, username, properties, createIfNonExistant=False): """ Modify a user in the CS. Takes the same params as in addUser and applies the changes """ return self.csAPI.modifyUser(username, properties, createIfNonExistant) ############################################################################# def csListUsers(self, group=False): """ Lists the users in the CS. If no group is specified return all users. """ return self.csAPI.listUsers(group) ############################################################################# def csDescribeUsers(self, mask=False): """ List users and their properties in the CS. If a mask is given, only users in the mask will be returned """ return self.csAPI.describeUsers(mask) ############################################################################# def csModifyGroup(self, groupname, properties, createIfNonExistant=False): """ Modify a user in the CS. Takes the same params as in addGroup and applies the changes """ return self.csAPI.modifyGroup(groupname, properties, createIfNonExistant) ############################################################################# def csListHosts(self): """ Lists the hosts in the CS """ return self.csAPI.listHosts() ############################################################################# def csDescribeHosts(self, mask=False): """ Gets extended info for the hosts in the CS """ return self.csAPI.describeHosts(mask) ############################################################################# def csModifyHost(self, hostname, properties, createIfNonExistant=False): """ Modify a host in the CS. Takes the same params as in addHost and applies the changes """ return self.csAPI.modifyHost(hostname, properties, createIfNonExistant) ############################################################################# def csListGroups(self): """ Lists groups in the CS """ return self.csAPI.listGroups() ############################################################################# def csDescribeGroups(self, mask=False): """ List groups and their properties in the CS. If a mask is given, only groups in the mask will be returned """ return self.csAPI.describeGroups(mask) ############################################################################# def csSyncUsersWithCFG(self, usersCFG): """ Synchronize users in cfg with its contents """ return self.csAPI.syncUsersWithCFG(usersCFG) ############################################################################# def csCommitChanges(self, sortUsers=True): """ Commit the changes in the CS """ return self.csAPI.commitChanges(sortUsers=False) ############################################################################# def sendMail(self, address, subject, body, fromAddress=None, localAttempt=True, html=False): """ Send mail to specified address with body. """ notification = NotificationClient() return notification.sendMail(address, subject, body, fromAddress, localAttempt, html) ############################################################################# def sendSMS(self, userName, body, fromAddress=None): """ Send mail to specified address with body. """ if len(body) > 160: return S_ERROR('Exceeded maximum SMS length of 160 characters') notification = NotificationClient() return notification.sendSMS(userName, body, fromAddress) ############################################################################# def getBDIISite(self, site, host=None): """ Get information about site from BDII at host """ return ldapSite(site, host=host) ############################################################################# def getBDIICluster(self, ce, host=None): """ Get information about ce from BDII at host """ return ldapCluster(ce, host=host) ############################################################################# def getBDIICE(self, ce, host=None): """ Get information about ce from BDII at host """ return ldapCE(ce, host=host) ############################################################################# def getBDIIService(self, ce, host=None): """ Get information about ce from BDII at host """ return ldapService(ce, host=host) ############################################################################# def getBDIICEState(self, ce, useVO=voName, host=None): """ Get information about ce state from BDII at host """ return ldapCEState(ce, useVO, host=host) ############################################################################# def getBDIICEVOView(self, ce, useVO=voName, host=None): """ Get information about ce voview from BDII at host """ return ldapCEVOView(ce, useVO, host=host) ############################################################################# def getBDIISE(self, site, useVO=voName, host=None): """ Get information about SA from BDII at host """ return ldapSE(site, useVO, host=host)
def __syncCSWithVOMS(self): self.__adminMsgs = {'Errors': [], 'Info': []} #Get DIRAC VOMS Mapping self.log.info("Getting DIRAC VOMS mapping") mappingSection = '/Registry/VOMS/Mapping' ret = gConfig.getOptionsDict(mappingSection) if not ret['OK']: self.log.fatal('No VOMS to DIRAC Group Mapping Available') return ret vomsMapping = ret['Value'] self.log.info("There are %s registered voms mappings in DIRAC" % len(vomsMapping)) #Get VOMS VO name self.log.info("Getting VOMS VO name") result = self.vomsSrv.admGetVOName() if not ret['OK']: self.log.fatal('Could not retrieve VOMS VO name') voNameInVOMS = result['Value'] self.log.info("VOMS VO Name is %s" % voNameInVOMS) #Get VOMS roles self.log.info("Getting the list of registered roles in VOMS") result = self.vomsSrv.admListRoles() if not ret['OK']: self.log.fatal('Could not retrieve registered roles in VOMS') rolesInVOMS = result['Value'] self.log.info("There are %s registered roles in VOMS" % len(rolesInVOMS)) print rolesInVOMS rolesInVOMS.append('') #Map VOMS roles vomsRoles = {} for role in rolesInVOMS: if role: role = "%s/%s" % (voNameInVOMS, role) else: role = voNameInVOMS groupsForRole = [] for group in vomsMapping: if vomsMapping[group] == role: groupsForRole.append(group) if groupsForRole: vomsRoles[role] = {'Groups': groupsForRole, 'Users': []} self.log.info("DIRAC valid VOMS roles are:\n\t", "\n\t ".join(vomsRoles.keys())) #Get DIRAC users self.log.info("Getting the list of registered users in DIRAC") csapi = CSAPI() ret = csapi.listUsers() if not ret['OK']: self.log.fatal('Could not retrieve current list of Users') return ret currentUsers = ret['Value'] ret = csapi.describeUsers(currentUsers) if not ret['OK']: self.log.fatal('Could not retrieve current User description') return ret currentUsers = ret['Value'] self.__adminMsgs['Info'].append( "There are %s registered users in DIRAC" % len(currentUsers)) self.log.info("There are %s registered users in DIRAC" % len(currentUsers)) #Get VOMS user entries self.log.info("Getting the list of registered user entries in VOMS") result = self.vomsSrv.admListMembers() if not ret['OK']: self.log.fatal( 'Could not retrieve registered user entries in VOMS') usersInVOMS = result['Value'] self.__adminMsgs['Info'].append( "There are %s registered user entries in VOMS" % len(usersInVOMS)) self.log.info("There are %s registered user entries in VOMS" % len(usersInVOMS)) #Consolidate users by nickname usersData = {} newUserNames = [] knownUserNames = [] obsoleteUserNames = [] self.log.info("Retrieving usernames...") usersInVOMS.sort() for iUPos in range(len(usersInVOMS)): userName = '' user = usersInVOMS[iUPos] for oldUser in currentUsers: if user['DN'].strip() in List.fromChar( currentUsers[oldUser]['DN']): userName = oldUser if not userName: result = self.vomsSrv.attGetUserNickname( user['DN'], user['CA']) if result['OK']: userName = result['Value'] else: self.__adminMsgs['Errors'].append( "Could not retrieve nickname for DN %s" % user['DN']) self.log.error("Could not get nickname for DN", user['DN']) userName = user['mail'][:user['mail'].find('@')] if not userName: self.log.error("Empty nickname for DN", user['DN']) self.__adminMsgs['Errors'].append("Empty nickname for DN %s" % user['DN']) continue self.log.info( " (%02d%%) Found username %s : %s " % ((iUPos * 100 / len(usersInVOMS)), userName, user['DN'])) if userName not in usersData: usersData[userName] = { 'DN': [], 'CA': [], 'Email': [], 'Groups': ['user'] } for key in ('DN', 'CA', 'mail'): value = user[key] if value: if key == "mail": List.appendUnique(usersData[userName]['Email'], value) else: usersData[userName][key].append(value.strip()) if userName not in currentUsers: List.appendUnique(newUserNames, userName) else: List.appendUnique(knownUserNames, userName) self.log.info("Finished retrieving usernames") if newUserNames: self.log.info("There are %s new users" % len(newUserNames)) else: self.log.info("There are no new users") #Get the list of users for each group result = csapi.listGroups() if not result['OK']: self.log.error("Could not get the list of groups in DIRAC", result['Message']) return result staticGroups = result['Value'] vomsGroups = [] self.log.info("Mapping users in VOMS to groups") for vomsRole in vomsRoles: self.log.info(" Getting users for role %s" % vomsRole) groupsForRole = vomsRoles[vomsRole]['Groups'] vomsMap = vomsRole.split("Role=") for g in groupsForRole: if g in staticGroups: staticGroups.pop(staticGroups.index(g)) else: vomsGroups.append(g) if len(vomsMap) == 1: # no Role users = usersInVOMS else: vomsGroup = "Role=".join(vomsMap[:-1]) if vomsGroup[-1] == "/": vomsGroup = vomsGroup[:-1] vomsRole = "Role=%s" % vomsMap[-1] result = self.vomsSrv.admListUsersWithRole(vomsGroup, vomsRole) if not result['OK']: errorMsg = "Could not get list of users for VOMS %s" % ( vomsMapping[group]) self.__adminMsgs['Errors'].append(errorMsg) self.log.error(errorMsg, result['Message']) return result users = result['Value'] numUsersInGroup = 0 for vomsUser in users: for userName in usersData: if vomsUser['DN'] in usersData[userName]['DN']: numUsersInGroup += 1 usersData[userName]['Groups'].extend(groupsForRole) infoMsg = "There are %s users in group(s) %s for VOMS Role %s" % ( numUsersInGroup, ",".join(groupsForRole), vomsRole) self.__adminMsgs['Info'].append(infoMsg) self.log.info(" %s" % infoMsg) self.log.info("Checking static groups") staticUsers = [] for group in staticGroups: self.log.info(" Checking static group %s" % group) numUsersInGroup = 0 result = csapi.listUsers(group) if not result['OK']: self.log.error( "Could not get the list of users in DIRAC group %s" % group, result['Message']) return result for userName in result['Value']: if userName in usersData: numUsersInGroup += 1 usersData[userName]['Groups'].append(group) else: if group not in vomsGroups and userName not in staticUsers: staticUsers.append(userName) infoMsg = "There are %s users in group %s" % (numUsersInGroup, group) self.__adminMsgs['Info'].append(infoMsg) self.log.info(" %s" % infoMsg) if staticUsers: infoMsg = "There are %s static users: %s" % ( len(staticUsers), ', '.join(staticUsers)) self.__adminMsgs['Info'].append(infoMsg) self.log.info("%s" % infoMsg) for user in currentUsers: if user not in usersData and user not in staticUsers: self.log.info('User %s is no longer valid' % user) obsoleteUserNames.append(user) #Do the CS Sync self.log.info("Updating CS...") ret = csapi.downloadCSData() if not ret['OK']: self.log.fatal('Can not update from CS', ret['Message']) return ret usersWithMoreThanOneDN = {} for user in usersData: csUserData = dict(usersData[user]) if len(csUserData['DN']) > 1: usersWithMoreThanOneDN[user] = csUserData['DN'] result = csapi.describeUsers([user]) if result['OK']: if result['Value']: prevUser = result['Value'][user] prevDNs = List.fromChar(prevUser['DN']) newDNs = csUserData['DN'] for DN in newDNs: if DN not in prevDNs: self.__adminMsgs['Info'].append( "User %s has new DN %s" % (user, DN)) for DN in prevDNs: if DN not in newDNs: self.__adminMsgs['Info'].append( "User %s has lost a DN %s" % (user, DN)) else: newDNs = csUserData['DN'] for DN in newDNs: self.__adminMsgs['Info'].append( "New user %s has new DN %s" % (user, DN)) for k in ('DN', 'CA', 'Email'): csUserData[k] = ", ".join(csUserData[k]) result = csapi.modifyUser(user, csUserData, createIfNonExistant=True) if not result['OK']: self.__adminMsgs['Error'].append("Cannot modify user %s: %s" % (user, result['Message'])) self.log.error("Cannot modify user", user) if usersWithMoreThanOneDN: self.__adminMsgs['Info'].append("\nUsers with more than one DN:") for uwmtod in sorted(usersWithMoreThanOneDN): self.__adminMsgs['Info'].append(" %s" % uwmtod) self.__adminMsgs['Info'].append(" + DN list:") for DN in usersWithMoreThanOneDN[uwmtod]: self.__adminMsgs['Info'].append(" - %s" % DN) if obsoleteUserNames: self.__adminMsgs['Info'].append("\nObsolete users:") address = self.am_getOption('MailTo', '*****@*****.**') fromAddress = self.am_getOption('mailFrom', '*****@*****.**') subject = 'Obsolete LFC Users found' body = 'Delete entries into LFC: \n' for obsoleteUser in obsoleteUserNames: self.log.info(subject, ", ".join(obsoleteUserNames)) body += 'for ' + obsoleteUser + '\n' self.__adminMsgs['Info'].append(" %s" % obsoleteUser) self.log.info("Deleting %s users" % len(obsoleteUserNames)) NotificationClient().sendMail(address, 'UsersAndGroupsAgent: %s' % subject, body, fromAddress) csapi.deleteUsers(obsoleteUserNames) if newUserNames: self.__adminMsgs['Info'].append("\nNew users:") for newUser in newUserNames: self.__adminMsgs['Info'].append(" %s" % newUser) self.__adminMsgs['Info'].append(" + DN list:") for DN in usersData[newUser]['DN']: self.__adminMsgs['Info'].append(" - %s" % DN) self.__adminMsgs['Info'].append(" + EMail: %s" % usersData[newUser]['Email']) result = csapi.commitChanges() if not result['OK']: self.log.error("Could not commit configuration changes", result['Message']) return result self.log.info("Configuration committed") #LFC Check if self.am_getOption("LFCCheckEnabled", True): result = self.checkLFCRegisteredUsers(usersData) if not result['OK']: return result return S_OK()
def __syncCSWithVOMS( self ): self.__adminMsgs = { 'Errors' : [], 'Info' : [] } #Get DIRAC VOMS Mapping self.log.info( "Getting DIRAC VOMS mapping" ) mappingSection = '/Registry/VOMS/Mapping' ret = gConfig.getOptionsDict( mappingSection ) if not ret['OK']: self.log.fatal( 'No VOMS to DIRAC Group Mapping Available' ) return ret vomsMapping = ret['Value'] self.log.info( "There are %s registered voms mappings in DIRAC" % len( vomsMapping ) ) #Get VOMS VO name self.log.info( "Getting VOMS VO name" ) result = self.vomsSrv.admGetVOName() if not ret['OK']: self.log.fatal( 'Could not retrieve VOMS VO name' ) voNameInVOMS = result[ 'Value' ] self.log.info( "VOMS VO Name is %s" % voNameInVOMS ) #Get VOMS roles self.log.info( "Getting the list of registered roles in VOMS" ) result = self.vomsSrv.admListRoles() if not ret['OK']: self.log.fatal( 'Could not retrieve registered roles in VOMS' ) rolesInVOMS = result[ 'Value' ] self.log.info( "There are %s registered roles in VOMS" % len( rolesInVOMS ) ) print rolesInVOMS rolesInVOMS.append( '' ) #Map VOMS roles vomsRoles = {} for role in rolesInVOMS: if role: role = "%s/%s" % ( voNameInVOMS, role ) else: role = voNameInVOMS groupsForRole = [] for group in vomsMapping: if vomsMapping[ group ] == role: groupsForRole.append( group ) if groupsForRole: vomsRoles[ role ] = { 'Groups' : groupsForRole, 'Users' : [] } self.log.info( "DIRAC valid VOMS roles are:\n\t", "\n\t ".join( vomsRoles.keys() ) ) #Get DIRAC users self.log.info( "Getting the list of registered users in DIRAC" ) csapi = CSAPI() ret = csapi.listUsers() if not ret['OK']: self.log.fatal( 'Could not retrieve current list of Users' ) return ret currentUsers = ret['Value'] ret = csapi.describeUsers( currentUsers ) if not ret['OK']: self.log.fatal( 'Could not retrieve current User description' ) return ret currentUsers = ret['Value'] self.__adminMsgs[ 'Info' ].append( "There are %s registered users in DIRAC" % len( currentUsers ) ) self.log.info( "There are %s registered users in DIRAC" % len( currentUsers ) ) #Get VOMS user entries self.log.info( "Getting the list of registered user entries in VOMS" ) result = self.vomsSrv.admListMembers() if not ret['OK']: self.log.fatal( 'Could not retrieve registered user entries in VOMS' ) usersInVOMS = result[ 'Value' ] self.__adminMsgs[ 'Info' ].append( "There are %s registered user entries in VOMS" % len( usersInVOMS ) ) self.log.info( "There are %s registered user entries in VOMS" % len( usersInVOMS ) ) #Consolidate users by nickname usersData = {} newUserNames = [] knownUserNames = [] obsoleteUserNames = [] self.log.info( "Retrieving usernames..." ) usersInVOMS.sort() for iUPos in range( len( usersInVOMS ) ): userName = '' user = usersInVOMS[ iUPos ] for oldUser in currentUsers: if user[ 'DN' ].strip() in List.fromChar( currentUsers[oldUser][ 'DN' ] ): userName = oldUser if not userName: result = self.vomsSrv.attGetUserNickname( user[ 'DN' ], user[ 'CA' ] ) if result[ 'OK' ]: userName = result[ 'Value' ] else: self.__adminMsgs[ 'Errors' ].append( "Could not retrieve nickname for DN %s" % user[ 'DN' ] ) self.log.error( "Could not get nickname for DN", user[ 'DN' ] ) userName = user[ 'mail' ][:user[ 'mail' ].find( '@' )] if not userName: self.log.error( "Empty nickname for DN", user[ 'DN' ] ) self.__adminMsgs[ 'Errors' ].append( "Empty nickname for DN %s" % user[ 'DN' ] ) continue self.log.info( " (%02d%%) Found username %s : %s " % ( ( iUPos * 100 / len( usersInVOMS ) ), userName, user[ 'DN' ] ) ) if userName not in usersData: usersData[ userName ] = { 'DN': [], 'CA': [], 'Email': [], 'Groups' : ['user'] } for key in ( 'DN', 'CA', 'mail' ): value = user[ key ] if value: if key == "mail": List.appendUnique( usersData[ userName ][ 'Email' ], value ) else: usersData[ userName ][ key ].append( value.strip() ) if userName not in currentUsers: List.appendUnique( newUserNames, userName ) else: List.appendUnique( knownUserNames, userName ) self.log.info( "Finished retrieving usernames" ) if newUserNames: self.log.info( "There are %s new users" % len( newUserNames ) ) else: self.log.info( "There are no new users" ) #Get the list of users for each group result = csapi.listGroups() if not result[ 'OK' ]: self.log.error( "Could not get the list of groups in DIRAC", result[ 'Message' ] ) return result staticGroups = result[ 'Value' ] vomsGroups = [] self.log.info( "Mapping users in VOMS to groups" ) for vomsRole in vomsRoles: self.log.info( " Getting users for role %s" % vomsRole ) groupsForRole = vomsRoles[ vomsRole ][ 'Groups' ] vomsMap = vomsRole.split( "Role=" ) for g in groupsForRole: if g in staticGroups: staticGroups.pop( staticGroups.index( g ) ) else: vomsGroups.append( g ) if len( vomsMap ) == 1: # no Role users = usersInVOMS else: vomsGroup = "Role=".join( vomsMap[:-1] ) if vomsGroup[-1] == "/": vomsGroup = vomsGroup[:-1] vomsRole = "Role=%s" % vomsMap[-1] result = self.vomsSrv.admListUsersWithRole( vomsGroup, vomsRole ) if not result[ 'OK' ]: errorMsg = "Could not get list of users for VOMS %s" % ( vomsMapping[ group ] ) self.__adminMsgs[ 'Errors' ].append( errorMsg ) self.log.error( errorMsg, result[ 'Message' ] ) return result users = result['Value'] numUsersInGroup = 0 for vomsUser in users: for userName in usersData: if vomsUser[ 'DN' ] in usersData[ userName ][ 'DN' ]: numUsersInGroup += 1 usersData[ userName ][ 'Groups' ].extend( groupsForRole ) infoMsg = "There are %s users in group(s) %s for VOMS Role %s" % ( numUsersInGroup, ",".join( groupsForRole ), vomsRole ) self.__adminMsgs[ 'Info' ].append( infoMsg ) self.log.info( " %s" % infoMsg ) self.log.info( "Checking static groups" ) staticUsers = [] for group in staticGroups: self.log.info( " Checking static group %s" % group ) numUsersInGroup = 0 result = csapi.listUsers( group ) if not result[ 'OK' ]: self.log.error( "Could not get the list of users in DIRAC group %s" % group , result[ 'Message' ] ) return result for userName in result[ 'Value' ]: if userName in usersData: numUsersInGroup += 1 usersData[ userName ][ 'Groups' ].append( group ) else: if group not in vomsGroups and userName not in staticUsers: staticUsers.append( userName ) infoMsg = "There are %s users in group %s" % ( numUsersInGroup, group ) self.__adminMsgs[ 'Info' ].append( infoMsg ) self.log.info( " %s" % infoMsg ) if staticUsers: infoMsg = "There are %s static users: %s" % ( len( staticUsers ) , ', '.join( staticUsers ) ) self.__adminMsgs[ 'Info' ].append( infoMsg ) self.log.info( "%s" % infoMsg ) for user in currentUsers: if user not in usersData and user not in staticUsers: self.log.info( 'User %s is no longer valid' % user ) obsoleteUserNames.append( user ) #Do the CS Sync self.log.info( "Updating CS..." ) ret = csapi.downloadCSData() if not ret['OK']: self.log.fatal( 'Can not update from CS', ret['Message'] ) return ret usersWithMoreThanOneDN = {} for user in usersData: csUserData = dict( usersData[ user ] ) if len( csUserData[ 'DN' ] ) > 1: usersWithMoreThanOneDN[ user ] = csUserData[ 'DN' ] result = csapi.describeUsers( [ user ] ) if result[ 'OK' ]: if result[ 'Value' ]: prevUser = result[ 'Value' ][ user ] prevDNs = List.fromChar( prevUser[ 'DN' ] ) newDNs = csUserData[ 'DN' ] for DN in newDNs: if DN not in prevDNs: self.__adminMsgs[ 'Info' ].append( "User %s has new DN %s" % ( user, DN ) ) for DN in prevDNs: if DN not in newDNs: self.__adminMsgs[ 'Info' ].append( "User %s has lost a DN %s" % ( user, DN ) ) else: newDNs = csUserData[ 'DN' ] for DN in newDNs: self.__adminMsgs[ 'Info' ].append( "New user %s has new DN %s" % ( user, DN ) ) for k in ( 'DN', 'CA', 'Email' ): csUserData[ k ] = ", ".join( csUserData[ k ] ) result = csapi.modifyUser( user, csUserData, createIfNonExistant = True ) if not result[ 'OK' ]: self.__adminMsgs[ 'Error' ].append( "Cannot modify user %s: %s" % ( user, result[ 'Message' ] ) ) self.log.error( "Cannot modify user", user ) if usersWithMoreThanOneDN: self.__adminMsgs[ 'Info' ].append( "\nUsers with more than one DN:" ) for uwmtod in sorted( usersWithMoreThanOneDN ): self.__adminMsgs[ 'Info' ].append( " %s" % uwmtod ) self.__adminMsgs[ 'Info' ].append( " + DN list:" ) for DN in usersWithMoreThanOneDN[uwmtod]: self.__adminMsgs[ 'Info' ].append( " - %s" % DN ) if obsoleteUserNames: self.__adminMsgs[ 'Info' ].append( "\nObsolete users:" ) address = self.am_getOption( 'MailTo', '*****@*****.**' ) fromAddress = self.am_getOption( 'mailFrom', '*****@*****.**' ) subject = 'Obsolete LFC Users found' body = 'Delete entries into LFC: \n' for obsoleteUser in obsoleteUserNames: self.log.info( subject, ", ".join( obsoleteUserNames ) ) body += 'for ' + obsoleteUser + '\n' self.__adminMsgs[ 'Info' ].append( " %s" % obsoleteUser ) self.log.info( "Deleting %s users" % len( obsoleteUserNames ) ) NotificationClient().sendMail( address, 'UsersAndGroupsAgent: %s' % subject, body, fromAddress ) csapi.deleteUsers( obsoleteUserNames ) if newUserNames: self.__adminMsgs[ 'Info' ].append( "\nNew users:" ) for newUser in newUserNames: self.__adminMsgs[ 'Info' ].append( " %s" % newUser ) self.__adminMsgs[ 'Info' ].append( " + DN list:" ) for DN in usersData[newUser][ 'DN' ]: self.__adminMsgs[ 'Info' ].append( " - %s" % DN ) self.__adminMsgs[ 'Info' ].append( " + EMail: %s" % usersData[newUser][ 'Email' ] ) result = csapi.commitChanges() if not result[ 'OK' ]: self.log.error( "Could not commit configuration changes", result[ 'Message' ] ) return result self.log.info( "Configuration committed" ) #LFC Check if self.am_getOption( "LFCCheckEnabled", True ): result = self.checkLFCRegisteredUsers( usersData ) if not result[ 'OK' ]: return result return S_OK()
class DiracAdmin(API): """ Administrative functionalities """ ############################################################################# def __init__(self): """Internal initialization of the DIRAC Admin API. """ super(DiracAdmin, self).__init__() self.csAPI = CSAPI() self.dbg = False if gConfig.getValue(self.section + '/LogLevel', 'DEBUG') == 'DEBUG': self.dbg = True self.scratchDir = gConfig.getValue(self.section + '/ScratchDir', '/tmp') self.currentDir = os.getcwd() self.rssFlag = ResourceStatus().rssFlag self.sitestatus = SiteStatus() self._siteSet = set(getSites().get('Value', [])) ############################################################################# def uploadProxy(self): """Upload a proxy to the DIRAC WMS. This method Example usage: >>> print diracAdmin.uploadProxy('dteam_pilot') {'OK': True, 'Value': 0L} :return: S_OK,S_ERROR :param permanent: Indefinitely update proxy :type permanent: boolean """ return gProxyManager.uploadProxy() ############################################################################# def setProxyPersistency(self, userDN, userGroup, persistent=True): """Set the persistence of a proxy in the Proxy Manager Example usage: >>> gLogger.notice(diracAdmin.setProxyPersistency( 'some DN', 'dirac group', True )) {'OK': True } :param userDN: User DN :type userDN: string :param userGroup: DIRAC Group :type userGroup: string :param persistent: Persistent flag :type persistent: boolean :return: S_OK,S_ERROR """ return gProxyManager.setPersistency(userDN, userGroup, persistent) ############################################################################# def checkProxyUploaded(self, userDN, userGroup, requiredTime): """Set the persistence of a proxy in the Proxy Manager Example usage: >>> gLogger.notice(diracAdmin.setProxyPersistency( 'some DN', 'dirac group', True )) {'OK': True, 'Value' : True/False } :param userDN: User DN :type userDN: string :param userGroup: DIRAC Group :type userGroup: string :param requiredTime: Required life time of the uploaded proxy :type requiredTime: boolean :return: S_OK,S_ERROR """ return gProxyManager.userHasProxy(userDN, userGroup, requiredTime) ############################################################################# def getSiteMask(self, printOutput=False, status='Active'): """Retrieve current site mask from WMS Administrator service. Example usage: >>> gLogger.notice(diracAdmin.getSiteMask()) {'OK': True, 'Value': 0L} :return: S_OK,S_ERROR """ result = self.sitestatus.getSites(siteState=status) if result['OK']: sites = result['Value'] if printOutput: sites.sort() for site in sites: gLogger.notice(site) return result ############################################################################# def getBannedSites(self, printOutput=False): """Retrieve current list of banned and probing sites. Example usage: >>> gLogger.notice(diracAdmin.getBannedSites()) {'OK': True, 'Value': []} :return: S_OK,S_ERROR """ bannedSites = self.sitestatus.getSites(siteState='Banned') if not bannedSites['OK']: return bannedSites probingSites = self.sitestatus.getSites(siteState='Probing') if not probingSites['OK']: return probingSites mergedList = sorted(bannedSites['Value'] + probingSites['Value']) if printOutput: gLogger.notice('\n'.join(mergedList)) return S_OK(mergedList) ############################################################################# def getSiteSection(self, site, printOutput=False): """Simple utility to get the list of CEs for DIRAC site name. Example usage: >>> gLogger.notice(diracAdmin.getSiteSection('LCG.CERN.ch')) {'OK': True, 'Value':} :return: S_OK,S_ERROR """ gridType = site.split('.')[0] if not gConfig.getSections('/Resources/Sites/%s' % (gridType))['OK']: return S_ERROR('/Resources/Sites/%s is not a valid site section' % (gridType)) result = gConfig.getOptionsDict('/Resources/Sites/%s/%s' % (gridType, site)) if printOutput and result['OK']: gLogger.notice(self.pPrint.pformat(result['Value'])) return result ############################################################################# def allowSite(self, site, comment, printOutput=False): """Adds the site to the site mask. Example usage: >>> gLogger.notice(diracAdmin.allowSite()) {'OK': True, 'Value': } :return: S_OK,S_ERROR """ result = self.__checkSiteIsValid(site) if not result['OK']: return result result = self.getSiteMask(status='Active') if not result['OK']: return result siteMask = result['Value'] if site in siteMask: if printOutput: gLogger.notice('Site %s is already Active' % site) return S_OK('Site %s is already Active' % site) if self.rssFlag: result = self.sitestatus.setSiteStatus(site, 'Active', comment) else: result = WMSAdministratorClient().allowSite(site, comment) if not result['OK']: return result if printOutput: gLogger.notice('Site %s status is set to Active' % site) return result ############################################################################# def getSiteMaskLogging(self, site=None, printOutput=False): """Retrieves site mask logging information. Example usage: >>> gLogger.notice(diracAdmin.getSiteMaskLogging('LCG.AUVER.fr')) {'OK': True, 'Value': } :return: S_OK,S_ERROR """ result = self.__checkSiteIsValid(site) if not result['OK']: return result if self.rssFlag: result = ResourceStatusClient().selectStatusElement('Site', 'History', name=site) else: result = WMSAdministratorClient().getSiteMaskLogging(site) if not result['OK']: return result if printOutput: if site: gLogger.notice('\nSite Mask Logging Info for %s\n' % site) else: gLogger.notice('\nAll Site Mask Logging Info\n') sitesLogging = result['Value'] if isinstance(sitesLogging, dict): for siteName, tupleList in sitesLogging.items( ): # can be an iterator if not siteName: gLogger.notice('\n===> %s\n' % siteName) for tup in tupleList: stup = str(tup[0]).ljust(8) + str(tup[1]).ljust(20) stup += '( ' + str(tup[2]).ljust(len(str( tup[2]))) + ' ) "' + str(tup[3]) + '"' gLogger.notice(stup) gLogger.notice(' ') elif isinstance(sitesLogging, list): sitesLoggingList = [(sl[1], sl[3], sl[4]) for sl in sitesLogging] for siteLog in sitesLoggingList: gLogger.notice(siteLog) return S_OK() ############################################################################# def banSite(self, site, comment, printOutput=False): """Removes the site from the site mask. Example usage: >>> gLogger.notice(diracAdmin.banSite()) {'OK': True, 'Value': } :return: S_OK,S_ERROR """ result = self.__checkSiteIsValid(site) if not result['OK']: return result mask = self.getSiteMask(status='Banned') if not mask['OK']: return mask siteMask = mask['Value'] if site in siteMask: if printOutput: gLogger.notice('Site %s is already Banned' % site) return S_OK('Site %s is already Banned' % site) if self.rssFlag: result = self.sitestatus.setSiteStatus(site, 'Banned', comment) else: result = WMSAdministratorClient().banSite(site, comment) if not result['OK']: return result if printOutput: gLogger.notice('Site %s status is set to Banned' % site) return result ############################################################################# def __checkSiteIsValid(self, site): """Internal function to check that a site name is valid. """ if isinstance(site, (list, set, dict)): site = set(site) - self._siteSet if not site: return S_OK() elif site in self._siteSet: return S_OK() return S_ERROR('Specified site %s is not in list of defined sites' % str(site)) ############################################################################# def getServicePorts(self, setup='', printOutput=False): """Checks the service ports for the specified setup. If not given this is taken from the current installation (/DIRAC/Setup) Example usage: >>> gLogger.notice(diracAdmin.getServicePorts()) {'OK': True, 'Value':''} :return: S_OK,S_ERROR """ if not setup: setup = gConfig.getValue('/DIRAC/Setup', '') setupList = gConfig.getSections('/DIRAC/Setups', []) if not setupList['OK']: return S_ERROR('Could not get /DIRAC/Setups sections') setupList = setupList['Value'] if setup not in setupList: return S_ERROR('Setup %s is not in allowed list: %s' % (setup, ', '.join(setupList))) serviceSetups = gConfig.getOptionsDict('/DIRAC/Setups/%s' % setup) if not serviceSetups['OK']: return S_ERROR('Could not get /DIRAC/Setups/%s options' % setup) serviceSetups = serviceSetups['Value'] # dict systemList = gConfig.getSections('/Systems') if not systemList['OK']: return S_ERROR('Could not get Systems sections') systemList = systemList['Value'] result = {} for system in systemList: if system in serviceSetups: path = '/Systems/%s/%s/Services' % (system, serviceSetups[system]) servicesList = gConfig.getSections(path) if not servicesList['OK']: self.log.warn('Could not get sections in %s' % path) else: servicesList = servicesList['Value'] if not servicesList: servicesList = [] self.log.verbose('System: %s ServicesList: %s' % (system, ', '.join(servicesList))) for service in servicesList: spath = '%s/%s/Port' % (path, service) servicePort = gConfig.getValue(spath, 0) if servicePort: self.log.verbose('Found port for %s/%s = %s' % (system, service, servicePort)) result['%s/%s' % (system, service)] = servicePort else: self.log.warn('No port found for %s' % spath) else: self.log.warn('%s is not defined in /DIRAC/Setups/%s' % (system, setup)) if printOutput: gLogger.notice(self.pPrint.pformat(result)) return S_OK(result) ############################################################################# def getProxy(self, userDN, userGroup, validity=43200, limited=False): """Retrieves a proxy with default 12hr validity and stores this in a file in the local directory by default. Example usage: >>> gLogger.notice(diracAdmin.getProxy()) {'OK': True, 'Value': } :return: S_OK,S_ERROR """ return gProxyManager.downloadProxy(userDN, userGroup, limited=limited, requiredTimeLeft=validity) ############################################################################# def getVOMSProxy(self, userDN, userGroup, vomsAttr=False, validity=43200, limited=False): """Retrieves a proxy with default 12hr validity and VOMS extensions and stores this in a file in the local directory by default. Example usage: >>> gLogger.notice(diracAdmin.getVOMSProxy()) {'OK': True, 'Value': } :return: S_OK,S_ERROR """ return gProxyManager.downloadVOMSProxy(userDN, userGroup, limited=limited, requiredVOMSAttribute=vomsAttr, requiredTimeLeft=validity) ############################################################################# def getPilotProxy(self, userDN, userGroup, validity=43200): """Retrieves a pilot proxy with default 12hr validity and stores this in a file in the local directory by default. Example usage: >>> gLogger.notice(diracAdmin.getVOMSProxy()) {'OK': True, 'Value': } :return: S_OK,S_ERROR """ return gProxyManager.getPilotProxyFromDIRACGroup( userDN, userGroup, requiredTimeLeft=validity) ############################################################################# def resetJob(self, jobID): """Reset a job or list of jobs in the WMS. This operation resets the reschedule counter for a job or list of jobs and allows them to run as new. Example:: >>> gLogger.notice(dirac.reset(12345)) {'OK': True, 'Value': [12345]} :param job: JobID :type job: integer or list of integers :return: S_OK,S_ERROR """ if isinstance(jobID, six.string_types): try: jobID = int(jobID) except Exception as x: return self._errorReport( str(x), 'Expected integer or convertible integer for existing jobID' ) elif isinstance(jobID, list): try: jobID = [int(job) for job in jobID] except Exception as x: return self._errorReport( str(x), 'Expected integer or convertible integer for existing jobIDs' ) result = JobManagerClient(useCertificates=False).resetJob(jobID) return result ############################################################################# def getJobPilotOutput(self, jobID, directory=''): """Retrieve the pilot output for an existing job in the WMS. The output will be retrieved in a local directory unless otherwise specified. >>> gLogger.notice(dirac.getJobPilotOutput(12345)) {'OK': True, StdOut:'',StdError:''} :param job: JobID :type job: integer or string :return: S_OK,S_ERROR """ if not directory: directory = self.currentDir if not os.path.exists(directory): return self._errorReport('Directory %s does not exist' % directory) result = WMSAdministratorClient().getJobPilotOutput(jobID) if not result['OK']: return result outputPath = '%s/pilot_%s' % (directory, jobID) if os.path.exists(outputPath): self.log.info('Remove %s and retry to continue' % outputPath) return S_ERROR('Remove %s and retry to continue' % outputPath) if not os.path.exists(outputPath): self.log.verbose('Creating directory %s' % outputPath) os.mkdir(outputPath) outputs = result['Value'] if 'StdOut' in outputs: stdout = '%s/std.out' % (outputPath) with open(stdout, 'w') as fopen: fopen.write(outputs['StdOut']) self.log.verbose('Standard output written to %s' % (stdout)) else: self.log.warn('No standard output returned') if 'StdError' in outputs: stderr = '%s/std.err' % (outputPath) with open(stderr, 'w') as fopen: fopen.write(outputs['StdError']) self.log.verbose('Standard error written to %s' % (stderr)) else: self.log.warn('No standard error returned') self.log.always('Outputs retrieved in %s' % outputPath) return result ############################################################################# def getPilotOutput(self, gridReference, directory=''): """Retrieve the pilot output (std.out and std.err) for an existing job in the WMS. >>> gLogger.notice(dirac.getJobPilotOutput(12345)) {'OK': True, 'Value': {}} :param job: JobID :type job: integer or string :return: S_OK,S_ERROR """ if not isinstance(gridReference, six.string_types): return self._errorReport('Expected string for pilot reference') if not directory: directory = self.currentDir if not os.path.exists(directory): return self._errorReport('Directory %s does not exist' % directory) result = PilotManagerClient().getPilotOutput(gridReference) if not result['OK']: return result gridReferenceSmall = gridReference.split('/')[-1] if not gridReferenceSmall: gridReferenceSmall = 'reference' outputPath = '%s/pilot_%s' % (directory, gridReferenceSmall) if os.path.exists(outputPath): self.log.info('Remove %s and retry to continue' % outputPath) return S_ERROR('Remove %s and retry to continue' % outputPath) if not os.path.exists(outputPath): self.log.verbose('Creating directory %s' % outputPath) os.mkdir(outputPath) outputs = result['Value'] if 'StdOut' in outputs: stdout = '%s/std.out' % (outputPath) with open(stdout, 'w') as fopen: fopen.write(outputs['StdOut']) self.log.info('Standard output written to %s' % (stdout)) else: self.log.warn('No standard output returned') if 'StdErr' in outputs: stderr = '%s/std.err' % (outputPath) with open(stderr, 'w') as fopen: fopen.write(outputs['StdErr']) self.log.info('Standard error written to %s' % (stderr)) else: self.log.warn('No standard error returned') self.log.always('Outputs retrieved in %s' % outputPath) return result ############################################################################# def getPilotInfo(self, gridReference): """Retrieve info relative to a pilot reference >>> gLogger.notice(dirac.getPilotInfo(12345)) {'OK': True, 'Value': {}} :param gridReference: Pilot Job Reference :type gridReference: string :return: S_OK,S_ERROR """ if not isinstance(gridReference, six.string_types): return self._errorReport('Expected string for pilot reference') result = PilotManagerClient().getPilotInfo(gridReference) return result ############################################################################# def killPilot(self, gridReference): """Kill the pilot specified >>> gLogger.notice(dirac.getPilotInfo(12345)) {'OK': True, 'Value': {}} :param gridReference: Pilot Job Reference :return: S_OK,S_ERROR """ if not isinstance(gridReference, six.string_types): return self._errorReport('Expected string for pilot reference') result = PilotManagerClient().killPilot(gridReference) return result ############################################################################# def getPilotLoggingInfo(self, gridReference): """Retrieve the pilot logging info for an existing job in the WMS. >>> gLogger.notice(dirac.getPilotLoggingInfo(12345)) {'OK': True, 'Value': {"The output of the command"}} :param gridReference: Gridp pilot job reference Id :type gridReference: string :return: S_OK,S_ERROR """ if not isinstance(gridReference, six.string_types): return self._errorReport('Expected string for pilot reference') return PilotManagerClient().getPilotLoggingInfo(gridReference) ############################################################################# def getJobPilots(self, jobID): """Extract the list of submitted pilots and their status for a given jobID from the WMS. Useful information is printed to the screen. >>> gLogger.notice(dirac.getJobPilots()) {'OK': True, 'Value': {PilotID:{StatusDict}}} :param job: JobID :type job: integer or string :return: S_OK,S_ERROR """ if isinstance(jobID, six.string_types): try: jobID = int(jobID) except Exception as x: return self._errorReport( str(x), 'Expected integer or string for existing jobID') result = PilotManagerClient().getPilots(jobID) if result['OK']: gLogger.notice(self.pPrint.pformat(result['Value'])) return result ############################################################################# def getPilotSummary(self, startDate='', endDate=''): """Retrieve the pilot output for an existing job in the WMS. Summary is printed at INFO level, full dictionary of results also returned. >>> gLogger.notice(dirac.getPilotSummary()) {'OK': True, 'Value': {CE:{Status:Count}}} :param job: JobID :type job: integer or string :return: S_OK,S_ERROR """ result = PilotManagerClient().getPilotSummary(startDate, endDate) if not result['OK']: return result ceDict = result['Value'] headers = 'CE'.ljust(28) i = 0 for ce, summary in ceDict.iteritems(): states = summary.keys() if len(states) > i: i = len(states) for i in xrange(i): headers += 'Status'.ljust(12) + 'Count'.ljust(12) gLogger.notice(headers) for ce, summary in ceDict.iteritems(): line = ce.ljust(28) states = sorted(summary) for state in states: count = str(summary[state]) line += state.ljust(12) + count.ljust(12) gLogger.notice(line) return result ############################################################################# def setSiteProtocols(self, site, protocolsList, printOutput=False): """ Allows to set the defined protocols for each SE for a given site. """ result = self.__checkSiteIsValid(site) if not result['OK']: return result siteSection = '/Resources/Sites/%s/%s/SE' % (site.split('.')[0], site) siteSEs = gConfig.getValue(siteSection, []) if not siteSEs: return S_ERROR('No SEs found for site %s in section %s' % (site, siteSection)) defaultProtocols = gConfig.getValue( '/Resources/StorageElements/DefaultProtocols', []) self.log.verbose('Default list of protocols are', ', '.join(defaultProtocols)) for protocol in protocolsList: if protocol not in defaultProtocols: return S_ERROR( 'Requested to set protocol %s in list but %s is not ' 'in default list of protocols:\n%s' % (protocol, protocol, ', '.join(defaultProtocols))) modifiedCS = False result = promptUser( 'Do you want to add the following default protocols:' ' %s for SE(s):\n%s' % (', '.join(protocolsList), ', '.join(siteSEs))) if not result['OK']: return result if result['Value'].lower() != 'y': self.log.always('No protocols will be added') return S_OK() for se in siteSEs: sections = gConfig.getSections('/Resources/StorageElements/%s/' % (se)) if not sections['OK']: return sections for section in sections['Value']: if gConfig.getValue( '/Resources/StorageElements/%s/%s/ProtocolName' % (se, section), '') == 'SRM2': path = '/Resources/StorageElements/%s/%s/ProtocolsList' % ( se, section) self.log.verbose('Setting %s to %s' % (path, ', '.join(protocolsList))) result = self.csSetOption(path, ', '.join(protocolsList)) if not result['OK']: return result modifiedCS = True if modifiedCS: result = self.csCommitChanges(False) if not result['OK']: return S_ERROR('CS Commit failed with message = %s' % (result['Message'])) else: if printOutput: gLogger.notice('Successfully committed changes to CS') else: if printOutput: gLogger.notice('No modifications to CS required') return S_OK() ############################################################################# def csSetOption(self, optionPath, optionValue): """ Function to modify an existing value in the CS. """ return self.csAPI.setOption(optionPath, optionValue) ############################################################################# def csSetOptionComment(self, optionPath, comment): """ Function to modify an existing value in the CS. """ return self.csAPI.setOptionComment(optionPath, comment) ############################################################################# def csModifyValue(self, optionPath, newValue): """ Function to modify an existing value in the CS. """ return self.csAPI.modifyValue(optionPath, newValue) ############################################################################# def csRegisterUser(self, username, properties): """ Registers a user in the CS. - username: Username of the user (easy;) - properties: Dict containing: - DN - groups : list/tuple of groups the user belongs to - <others> : More properties of the user, like mail """ return self.csAPI.addUser(username, properties) ############################################################################# def csDeleteUser(self, user): """ Deletes a user from the CS. Can take a list of users """ return self.csAPI.deleteUsers(user) ############################################################################# def csModifyUser(self, username, properties, createIfNonExistant=False): """ Modify a user in the CS. Takes the same params as in addUser and applies the changes """ return self.csAPI.modifyUser(username, properties, createIfNonExistant) ############################################################################# def csListUsers(self, group=False): """ Lists the users in the CS. If no group is specified return all users. """ return self.csAPI.listUsers(group) ############################################################################# def csDescribeUsers(self, mask=False): """ List users and their properties in the CS. If a mask is given, only users in the mask will be returned """ return self.csAPI.describeUsers(mask) ############################################################################# def csModifyGroup(self, groupname, properties, createIfNonExistant=False): """ Modify a user in the CS. Takes the same params as in addGroup and applies the changes """ return self.csAPI.modifyGroup(groupname, properties, createIfNonExistant) ############################################################################# def csListHosts(self): """ Lists the hosts in the CS """ return self.csAPI.listHosts() ############################################################################# def csDescribeHosts(self, mask=False): """ Gets extended info for the hosts in the CS """ return self.csAPI.describeHosts(mask) ############################################################################# def csModifyHost(self, hostname, properties, createIfNonExistant=False): """ Modify a host in the CS. Takes the same params as in addHost and applies the changes """ return self.csAPI.modifyHost(hostname, properties, createIfNonExistant) ############################################################################# def csListGroups(self): """ Lists groups in the CS """ return self.csAPI.listGroups() ############################################################################# def csDescribeGroups(self, mask=False): """ List groups and their properties in the CS. If a mask is given, only groups in the mask will be returned """ return self.csAPI.describeGroups(mask) ############################################################################# def csSyncUsersWithCFG(self, usersCFG): """ Synchronize users in cfg with its contents """ return self.csAPI.syncUsersWithCFG(usersCFG) ############################################################################# def csCommitChanges(self, sortUsers=True): """ Commit the changes in the CS """ return self.csAPI.commitChanges(sortUsers=False) ############################################################################# def sendMail(self, address, subject, body, fromAddress=None, localAttempt=True, html=False): """ Send mail to specified address with body. """ notification = NotificationClient() return notification.sendMail(address, subject, body, fromAddress, localAttempt, html) ############################################################################# def sendSMS(self, userName, body, fromAddress=None): """ Send mail to specified address with body. """ if len(body) > 160: return S_ERROR('Exceeded maximum SMS length of 160 characters') notification = NotificationClient() return notification.sendSMS(userName, body, fromAddress) ############################################################################# def getBDIISite(self, site, host=None): """ Get information about site from BDII at host """ return ldapSite(site, host=host) ############################################################################# def getBDIICluster(self, ce, host=None): """ Get information about ce from BDII at host """ return ldapCluster(ce, host=host) ############################################################################# def getBDIICE(self, ce, host=None): """ Get information about ce from BDII at host """ return ldapCE(ce, host=host) ############################################################################# def getBDIIService(self, ce, host=None): """ Get information about ce from BDII at host """ return ldapService(ce, host=host) ############################################################################# def getBDIICEState(self, ce, useVO=voName, host=None): """ Get information about ce state from BDII at host """ return ldapCEState(ce, useVO, host=host) ############################################################################# def getBDIICEVOView(self, ce, useVO=voName, host=None): """ Get information about ce voview from BDII at host """ return ldapCEVOView(ce, useVO, host=host)
class DiracAdmin( API ): """ Administrative functionalities """ ############################################################################# def __init__( self ): """Internal initialization of the DIRAC Admin API. """ super( DiracAdmin, self ).__init__() self.csAPI = CSAPI() self.dbg = False if gConfig.getValue( self.section + '/LogLevel', 'DEBUG' ) == 'DEBUG': self.dbg = True self.scratchDir = gConfig.getValue( self.section + '/ScratchDir', '/tmp' ) self.currentDir = os.getcwd() ############################################################################# def uploadProxy( self, group ): """Upload a proxy to the DIRAC WMS. This method Example usage: >>> print diracAdmin.uploadProxy('lhcb_pilot') {'OK': True, 'Value': 0L} :param group: DIRAC Group :type job: string :return: S_OK,S_ERROR :param permanent: Indefinitely update proxy :type permanent: boolean """ return gProxyManager.uploadProxy( diracGroup = group ) ############################################################################# def setProxyPersistency( self, userDN, userGroup, persistent = True ): """Set the persistence of a proxy in the Proxy Manager Example usage: >>> print diracAdmin.setProxyPersistency( 'some DN', 'dirac group', True ) {'OK': True } :param userDN: User DN :type userDN: string :param userGroup: DIRAC Group :type userGroup: string :param persistent: Persistent flag :type persistent: boolean :return: S_OK,S_ERROR """ return gProxyManager.setPersistency( userDN, userGroup, persistent ) ############################################################################# def checkProxyUploaded( self, userDN, userGroup, requiredTime ): """Set the persistence of a proxy in the Proxy Manager Example usage: >>> print diracAdmin.setProxyPersistency( 'some DN', 'dirac group', True ) {'OK': True, 'Value' : True/False } :param userDN: User DN :type userDN: string :param userGroup: DIRAC Group :type userGroup: string :param requiredTime: Required life time of the uploaded proxy :type requiredTime: boolean :return: S_OK,S_ERROR """ return gProxyManager.userHasProxy( userDN, userGroup, requiredTime ) ############################################################################# def getSiteMask( self, printOutput = False ): """Retrieve current site mask from WMS Administrator service. Example usage: >>> print diracAdmin.getSiteMask() {'OK': True, 'Value': 0L} :return: S_OK,S_ERROR """ wmsAdmin = RPCClient( 'WorkloadManagement/WMSAdministrator' ) result = wmsAdmin.getSiteMask() if result['OK']: sites = result['Value'] if printOutput: sites.sort() for site in sites: print site return result ############################################################################# def getBannedSites( self, gridType = [], printOutput = False ): """Retrieve current list of banned sites. Example usage: >>> print diracAdmin.getBannedSites() {'OK': True, 'Value': []} :return: S_OK,S_ERROR """ wmsAdmin = RPCClient( 'WorkloadManagement/WMSAdministrator' ) bannedSites = [] totalList = [] result = wmsAdmin.getSiteMask() if not result['OK']: self.log.warn( result['Message'] ) return result sites = result['Value'] if not gridType: result = gConfig.getSections( '/Resources/Sites' ) if not result['OK']: return result gridType = result['Value'] for grid in gridType: result = gConfig.getSections( '/Resources/Sites/%s' % grid ) if not result['OK']: return result totalList += result['Value'] for site in totalList: if not site in sites: bannedSites.append( site ) bannedSites.sort() if printOutput: print '\n'.join( bannedSites ) return S_OK( bannedSites ) ############################################################################# def getSiteSection( self, site, printOutput = False ): """Simple utility to get the list of CEs for DIRAC site name. Example usage: >>> print diracAdmin.getSiteSection('LCG.CERN.ch') {'OK': True, 'Value':} :return: S_OK,S_ERROR """ gridType = site.split( '.' )[0] if not gConfig.getSections( '/Resources/Sites/%s' % ( gridType ) )['OK']: return S_ERROR( '/Resources/Sites/%s is not a valid site section' % ( gridType ) ) result = gConfig.getOptionsDict( '/Resources/Sites/%s/%s' % ( gridType, site ) ) if printOutput and result['OK']: print self.pPrint.pformat( result['Value'] ) return result ############################################################################# def addSiteInMask( self, site, comment, printOutput = False ): """Adds the site to the site mask. Example usage: >>> print diracAdmin.addSiteInMask() {'OK': True, 'Value': } :return: S_OK,S_ERROR """ result = self.__checkSiteIsValid( site ) if not result['OK']: return result mask = self.getSiteMask() if not mask['OK']: return mask siteMask = mask['Value'] if site in siteMask: return S_ERROR( 'Site %s already in mask of allowed sites' % site ) wmsAdmin = RPCClient( 'WorkloadManagement/WMSAdministrator' ) result = wmsAdmin.allowSite( site, comment ) if not result['OK']: return result if printOutput: print 'Allowing %s in site mask' % site return result ############################################################################# def getSiteMaskLogging( self, site = None, printOutput = False ): """Retrieves site mask logging information. Example usage: >>> print diracAdmin.getSiteMaskLogging('LCG.AUVER.fr') {'OK': True, 'Value': } :return: S_OK,S_ERROR """ result = self.__checkSiteIsValid( site ) if not result['OK']: return result wmsAdmin = RPCClient( 'WorkloadManagement/WMSAdministrator' ) result = wmsAdmin.getSiteMaskLogging( site ) if not result['OK']: return result if site: if not result['Value'].has_key( site ): return S_ERROR( 'Site mask information not available for %s' % ( site ) ) if printOutput: if site: print '\nSite Mask Logging Info for %s\n' % site else: print '\nAll Site Mask Logging Info\n' siteDict = result['Value'] for site, tupleList in siteDict.iteritems(): if not site: print '\n===> %s\n' % site for tup in tupleList: print str( tup[0] ).ljust( 8 ) + str( tup[1] ).ljust( 20 ) + \ '( ' + str( tup[2] ).ljust( len( str( tup[2] ) ) ) + ' ) "' + str( tup[3] ) + '"' print ' ' return result ############################################################################# def banSiteFromMask( self, site, comment, printOutput = False ): """Removes the site from the site mask. Example usage: >>> print diracAdmin.banSiteFromMask() {'OK': True, 'Value': } :return: S_OK,S_ERROR """ result = self.__checkSiteIsValid( site ) if not result['OK']: return result mask = self.getSiteMask() if not mask['OK']: return mask siteMask = mask['Value'] if not site in siteMask: return S_ERROR( 'Site %s is already banned' % site ) wmsAdmin = RPCClient( 'WorkloadManagement/WMSAdministrator' ) result = wmsAdmin.banSite( site, comment ) if not result['OK']: return result if printOutput: print 'Removing %s from site mask' % site return result ############################################################################# @classmethod def __checkSiteIsValid( self, site ): """Internal function to check that a site name is valid. """ sites = getSiteCEMapping() if not sites['OK']: return S_ERROR( 'Could not get site CE mapping' ) siteList = sites['Value'].keys() if not site in siteList: return S_ERROR( 'Specified site %s is not in list of defined sites' % site ) return S_OK( '%s is valid' % site ) ############################################################################# def clearMask( self ): """Removes all sites from the site mask. Should be used with care. Example usage: >>> print diracAdmin.clearMask() {'OK': True, 'Value':''} :return: S_OK,S_ERROR """ wmsAdmin = RPCClient( 'WorkloadManagement/WMSAdministrator' ) result = wmsAdmin.clearMask() return result ############################################################################# def getServicePorts( self, setup = '', printOutput = False ): """Checks the service ports for the specified setup. If not given this is taken from the current installation (/DIRAC/Setup) Example usage: >>> print diracAdmin.getServicePorts() {'OK': True, 'Value':''} :return: S_OK,S_ERROR """ if not setup: setup = gConfig.getValue( '/DIRAC/Setup', '' ) setupList = gConfig.getSections( '/DIRAC/Setups', [] ) if not setupList['OK']: return S_ERROR( 'Could not get /DIRAC/Setups sections' ) setupList = setupList['Value'] if not setup in setupList: return S_ERROR( 'Setup %s is not in allowed list: %s' % ( setup, ', '.join( setupList ) ) ) serviceSetups = gConfig.getOptionsDict( '/DIRAC/Setups/%s' % setup ) if not serviceSetups['OK']: return S_ERROR( 'Could not get /DIRAC/Setups/%s options' % setup ) serviceSetups = serviceSetups['Value'] # dict systemList = gConfig.getSections( '/Systems' ) if not systemList['OK']: return S_ERROR( 'Could not get Systems sections' ) systemList = systemList['Value'] result = {} for system in systemList: if serviceSetups.has_key( system ): path = '/Systems/%s/%s/Services' % ( system, serviceSetups[system] ) servicesList = gConfig.getSections( path ) if not servicesList['OK']: self.log.warn( 'Could not get sections in %s' % path ) else: servicesList = servicesList['Value'] if not servicesList: servicesList = [] self.log.verbose( 'System: %s ServicesList: %s' % ( system, ', '.join( servicesList ) ) ) for service in servicesList: spath = '%s/%s/Port' % ( path, service ) servicePort = gConfig.getValue( spath, 0 ) if servicePort: self.log.verbose( 'Found port for %s/%s = %s' % ( system, service, servicePort ) ) result['%s/%s' % ( system, service )] = servicePort else: self.log.warn( 'No port found for %s' % spath ) else: self.log.warn( '%s is not defined in /DIRAC/Setups/%s' % ( system, setup ) ) if printOutput: print self.pPrint.pformat( result ) return S_OK( result ) ############################################################################# def getProxy( self, userDN, userGroup, validity = 43200, limited = False ): """Retrieves a proxy with default 12hr validity and stores this in a file in the local directory by default. Example usage: >>> print diracAdmin.getProxy() {'OK': True, 'Value': } :return: S_OK,S_ERROR """ return gProxyManager.downloadProxy( userDN, userGroup, limited = limited, requiredTimeLeft = validity ) ############################################################################# def getVOMSProxy( self, userDN, userGroup, vomsAttr = False, validity = 43200, limited = False ): """Retrieves a proxy with default 12hr validity and VOMS extensions and stores this in a file in the local directory by default. Example usage: >>> print diracAdmin.getVOMSProxy() {'OK': True, 'Value': } :return: S_OK,S_ERROR """ return gProxyManager.downloadVOMSProxy( userDN, userGroup, limited = limited, requiredVOMSAttribute = vomsAttr, requiredTimeLeft = validity ) ############################################################################# def getPilotProxy( self, userDN, userGroup, validity = 43200 ): """Retrieves a pilot proxy with default 12hr validity and stores this in a file in the local directory by default. Example usage: >>> print diracAdmin.getVOMSProxy() {'OK': True, 'Value': } :return: S_OK,S_ERROR """ return gProxyManager.getPilotProxyFromDIRACGroup( userDN, userGroup, requiredTimeLeft = validity ) ############################################################################# def resetJob( self, jobID ): """Reset a job or list of jobs in the WMS. This operation resets the reschedule counter for a job or list of jobs and allows them to run as new. Example:: >>> print dirac.reset(12345) {'OK': True, 'Value': [12345]} :param job: JobID :type job: integer or list of integers :return: S_OK,S_ERROR """ if isinstance( jobID, basestring ): try: jobID = int( jobID ) except Exception as x: return self._errorReport( str( x ), 'Expected integer or convertible integer for existing jobID' ) elif isinstance( jobID, list ): try: jobID = [int( job ) for job in jobID] except Exception as x: return self._errorReport( str( x ), 'Expected integer or convertible integer for existing jobIDs' ) jobManager = RPCClient( 'WorkloadManagement/JobManager', useCertificates = False ) result = jobManager.resetJob( jobID ) return result ############################################################################# def getJobPilotOutput( self, jobID, directory = '' ): """Retrieve the pilot output for an existing job in the WMS. The output will be retrieved in a local directory unless otherwise specified. >>> print dirac.getJobPilotOutput(12345) {'OK': True, StdOut:'',StdError:''} :param job: JobID :type job: integer or string :return: S_OK,S_ERROR """ if not directory: directory = self.currentDir if not os.path.exists( directory ): return self._errorReport( 'Directory %s does not exist' % directory ) wmsAdmin = RPCClient( 'WorkloadManagement/WMSAdministrator' ) result = wmsAdmin.getJobPilotOutput( jobID ) if not result['OK']: return result outputPath = '%s/pilot_%s' % ( directory, jobID ) if os.path.exists( outputPath ): self.log.info( 'Remove %s and retry to continue' % outputPath ) return S_ERROR( 'Remove %s and retry to continue' % outputPath ) if not os.path.exists( outputPath ): self.log.verbose( 'Creating directory %s' % outputPath ) os.mkdir( outputPath ) outputs = result['Value'] if outputs.has_key( 'StdOut' ): stdout = '%s/std.out' % ( outputPath ) with open( stdout, 'w' ) as fopen: fopen.write( outputs['StdOut'] ) self.log.verbose( 'Standard output written to %s' % ( stdout ) ) else: self.log.warn( 'No standard output returned' ) if outputs.has_key( 'StdError' ): stderr = '%s/std.err' % ( outputPath ) with open( stderr, 'w' ) as fopen: fopen.write( outputs['StdError'] ) self.log.verbose( 'Standard error written to %s' % ( stderr ) ) else: self.log.warn( 'No standard error returned' ) self.log.always( 'Outputs retrieved in %s' % outputPath ) return result ############################################################################# def getPilotOutput( self, gridReference, directory = '' ): """Retrieve the pilot output (std.out and std.err) for an existing job in the WMS. >>> print dirac.getJobPilotOutput(12345) {'OK': True, 'Value': {}} :param job: JobID :type job: integer or string :return: S_OK,S_ERROR """ if not isinstance( gridReference, basestring ): return self._errorReport( 'Expected string for pilot reference' ) if not directory: directory = self.currentDir if not os.path.exists( directory ): return self._errorReport( 'Directory %s does not exist' % directory ) wmsAdmin = RPCClient( 'WorkloadManagement/WMSAdministrator' ) result = wmsAdmin.getPilotOutput( gridReference ) if not result['OK']: return result gridReferenceSmall = gridReference.split( '/' )[-1] if not gridReferenceSmall: gridReferenceSmall = 'reference' outputPath = '%s/pilot_%s' % ( directory, gridReferenceSmall ) if os.path.exists( outputPath ): self.log.info( 'Remove %s and retry to continue' % outputPath ) return S_ERROR( 'Remove %s and retry to continue' % outputPath ) if not os.path.exists( outputPath ): self.log.verbose( 'Creating directory %s' % outputPath ) os.mkdir( outputPath ) outputs = result['Value'] if outputs.has_key( 'StdOut' ): stdout = '%s/std.out' % ( outputPath ) with open( stdout, 'w' ) as fopen: fopen.write( outputs['StdOut'] ) self.log.info( 'Standard output written to %s' % ( stdout ) ) else: self.log.warn( 'No standard output returned' ) if outputs.has_key( 'StdErr' ): stderr = '%s/std.err' % ( outputPath ) with open( stderr, 'w' ) as fopen: fopen.write( outputs['StdErr'] ) self.log.info( 'Standard error written to %s' % ( stderr ) ) else: self.log.warn( 'No standard error returned' ) self.log.always( 'Outputs retrieved in %s' % outputPath ) return result ############################################################################# def getPilotInfo( self, gridReference ): """Retrieve info relative to a pilot reference >>> print dirac.getPilotInfo(12345) {'OK': True, 'Value': {}} :param gridReference: Pilot Job Reference :type gridReference: string :return: S_OK,S_ERROR """ if not isinstance( gridReference, basestring ): return self._errorReport( 'Expected string for pilot reference' ) wmsAdmin = RPCClient( 'WorkloadManagement/WMSAdministrator' ) result = wmsAdmin.getPilotInfo( gridReference ) return result ############################################################################# def killPilot( self, gridReference ): """Kill the pilot specified >>> print dirac.getPilotInfo(12345) {'OK': True, 'Value': {}} :param gridReference: Pilot Job Reference :return: S_OK,S_ERROR """ if not isinstance( gridReference, basestring ): return self._errorReport( 'Expected string for pilot reference' ) wmsAdmin = RPCClient( 'WorkloadManagement/WMSAdministrator' ) result = wmsAdmin.killPilot( gridReference ) return result ############################################################################# def getPilotLoggingInfo( self, gridReference ): """Retrieve the pilot logging info for an existing job in the WMS. >>> print dirac.getPilotLoggingInfo(12345) {'OK': True, 'Value': {"The output of the command"}} :param gridReference: Gridp pilot job reference Id :type gridReference: string :return: S_OK,S_ERROR """ if type( gridReference ) not in types.StringTypes: return self._errorReport( 'Expected string for pilot reference' ) wmsAdmin = RPCClient( 'WorkloadManagement/WMSAdministrator' ) return wmsAdmin.getPilotLoggingInfo( gridReference ) ############################################################################# def getJobPilots( self, jobID ): """Extract the list of submitted pilots and their status for a given jobID from the WMS. Useful information is printed to the screen. >>> print dirac.getJobPilots() {'OK': True, 'Value': {PilotID:{StatusDict}}} :param job: JobID :type job: integer or string :return: S_OK,S_ERROR """ if isinstance( jobID, basestring ): try: jobID = int( jobID ) except Exception as x: return self._errorReport( str( x ), 'Expected integer or string for existing jobID' ) wmsAdmin = RPCClient( 'WorkloadManagement/WMSAdministrator' ) result = wmsAdmin.getPilots( jobID ) if result['OK']: print self.pPrint.pformat( result['Value'] ) return result ############################################################################# def getPilotSummary( self, startDate = '', endDate = '' ): """Retrieve the pilot output for an existing job in the WMS. Summary is printed at INFO level, full dictionary of results also returned. >>> print dirac.getPilotSummary() {'OK': True, 'Value': {CE:{Status:Count}}} :param job: JobID :type job: integer or string :return: S_OK,S_ERROR """ wmsAdmin = RPCClient( 'WorkloadManagement/WMSAdministrator' ) result = wmsAdmin.getPilotSummary( startDate, endDate ) if not result['OK']: return result ceDict = result['Value'] headers = 'CE'.ljust( 28 ) i = 0 for ce, summary in ceDict.iteritems(): states = summary.keys() if len( states ) > i: i = len( states ) for i in xrange( i ): headers += 'Status'.ljust( 12 ) + 'Count'.ljust( 12 ) print headers for ce, summary in ceDict.iteritems(): line = ce.ljust( 28 ) states = summary.keys() states.sort() for state in states: count = str( summary[state] ) line += state.ljust( 12 ) + count.ljust( 12 ) print line return result ############################################################################# def selectRequests( self, jobID = None, requestID = None, requestName = None, requestType = None, status = None, operation = None, ownerDN = None, ownerGroup = None, requestStart = 0, limit = 100, printOutput = False ): """Select requests from the request management system. A few notes on the selection criteria: - jobID is the WMS JobID for the request (if applicable) - requestID is assigned during submission of the request - requestName is the corresponding XML file name - requestType e.g. 'transfer' - status e.g. Done - operation e.g. replicateAndRegister - requestStart e.g. the first request to consider (start from 0 by default) - limit e.g. selection limit (default 100) >>> dirac.selectRequests(jobID='4894') {'OK': True, 'Value': [[<Requests>]]} """ options = {'RequestID':requestID, 'RequestName':requestName, 'JobID':jobID, 'OwnerDN':ownerDN, 'OwnerGroup':ownerGroup, 'RequestType':requestType, 'Status':status, 'Operation':operation} conditions = {} for key, value in options.iteritems(): if value: try: conditions[key] = str( value ) except Exception as x: return self._errorReport( str( x ), 'Expected string for %s field' % key ) try: requestStart = int( requestStart ) limit = int( limit ) except Exception as x: return self._errorReport( str( x ), 'Expected integer for %s field' % limit ) self.log.verbose( 'Will select requests with the following conditions' ) self.log.verbose( self.pPrint.pformat( conditions ) ) requestClient = RPCClient( "RequestManagement/centralURL" ) result = requestClient.getRequestSummaryWeb( conditions, [], requestStart, limit ) if not result['OK']: self.log.warn( result['Message'] ) return result requestIDs = result['Value'] conds = [] for key, value in conditions.iteritems(): if value: conds.append( '%s = %s' % ( key, value ) ) self.log.verbose( '%s request(s) selected with conditions %s and limit %s' % ( len( requestIDs['Records'] ), ', '.join( conds ), limit ) ) if printOutput: requests = [] if len( requestIDs['Records'] ) > limit: requestList = requestIDs['Records'] requests = requestList[:limit] else: requests = requestIDs['Records'] print '%s request(s) selected with conditions %s and limit %s' % ( len( requestIDs['Records'] ), ', '.join( conds ), limit ) print requestIDs['ParameterNames'] for request in requests: print request if not requestIDs: return S_ERROR( 'No requests selected for conditions: %s' % conditions ) else: return result ############################################################################# def getRequestSummary( self, printOutput = False ): """ Get a summary of the requests in the request DB. """ requestClient = RPCClient( "RequestManagement/centralURL", timeout = 120 ) result = requestClient.getDBSummary() if not result['OK']: self.log.warn( result['Message'] ) return result if printOutput: print self.pPrint.pformat( result['Value'] ) return result ############################################################################# def getExternalPackageVersions( self ): """ Simple function that attempts to obtain the external versions for the local DIRAC installation (frequently needed for debugging purposes). """ gLogger.info( 'DIRAC version v%dr%d build %d' % ( DIRAC.majorVersion, DIRAC.minorVersion, DIRAC.patchLevel ) ) try: import lcg_util infoStr = 'Using lcg_util from: \n%s' % lcg_util.__file__ gLogger.info( infoStr ) infoStr = "The version of lcg_utils is %s" % lcg_util.lcg_util_version() gLogger.info( infoStr ) except Exception as x: errStr = "SRM2Storage.__init__: Failed to import lcg_util: %s" % ( x ) gLogger.exception( errStr ) try: import gfalthr as gfal infoStr = "Using gfalthr from: \n%s" % gfal.__file__ gLogger.info( infoStr ) infoStr = "The version of gfalthr is %s" % gfal.gfal_version() gLogger.info( infoStr ) except Exception as x: errStr = "SRM2Storage.__init__: Failed to import gfalthr: %s." % ( x ) gLogger.warn( errStr ) try: import gfal infoStr = "Using gfal from: %s" % gfal.__file__ gLogger.info( infoStr ) infoStr = "The version of gfal is %s" % gfal.gfal_version() gLogger.info( infoStr ) except Exception as x: errStr = "SRM2Storage.__init__: Failed to import gfal: %s" % ( x ) gLogger.exception( errStr ) defaultProtocols = gConfig.getValue( '/Resources/StorageElements/DefaultProtocols', [] ) gLogger.info( 'Default list of protocols are: %s' % ( ', '.join( defaultProtocols ) ) ) return S_OK() ############################################################################# def getSiteProtocols( self, site, printOutput = False ): """ Allows to check the defined protocols for each site SE. """ result = self.__checkSiteIsValid( site ) if not result['OK']: return result siteSection = '/Resources/Sites/%s/%s/SE' % ( site.split( '.' )[0], site ) siteSEs = gConfig.getValue( siteSection, [] ) if not siteSEs: return S_ERROR( 'No SEs found for site %s in section %s' % ( site, siteSection ) ) defaultProtocols = gConfig.getValue( '/Resources/StorageElements/DefaultProtocols', [] ) self.log.verbose( 'Default list of protocols are' ', '.join( defaultProtocols ) ) seInfo = {} siteSEs.sort() for se in siteSEs: sections = gConfig.getSections( '/Resources/StorageElements/%s/' % ( se ) ) if not sections['OK']: return sections for section in sections['Value']: if gConfig.getValue( '/Resources/StorageElements/%s/%s/ProtocolName' % ( se, section ), '' ) == 'SRM2': path = '/Resources/StorageElements/%s/%s/ProtocolsList' % ( se, section ) seProtocols = gConfig.getValue( path, [] ) if not seProtocols: seProtocols = defaultProtocols seInfo[se] = seProtocols if printOutput: print '\nSummary of protocols for StorageElements at site %s' % site print '\nStorageElement'.ljust( 30 ) + 'ProtocolsList'.ljust( 30 ) + '\n' for se, protocols in seInfo.iteritems(): print se.ljust( 30 ) + ', '.join( protocols ).ljust( 30 ) return S_OK( seInfo ) ############################################################################# def setSiteProtocols( self, site, protocolsList, printOutput = False ): """ Allows to set the defined protocols for each SE for a given site. """ result = self.__checkSiteIsValid( site ) if not result['OK']: return result siteSection = '/Resources/Sites/%s/%s/SE' % ( site.split( '.' )[0], site ) siteSEs = gConfig.getValue( siteSection, [] ) if not siteSEs: return S_ERROR( 'No SEs found for site %s in section %s' % ( site, siteSection ) ) defaultProtocols = gConfig.getValue( '/Resources/StorageElements/DefaultProtocols', [] ) self.log.verbose( 'Default list of protocols are', ', '.join( defaultProtocols ) ) for protocol in protocolsList: if not protocol in defaultProtocols: return S_ERROR( 'Requested to set protocol %s in list but %s is not ' 'in default list of protocols:\n%s' % ( protocol, protocol, ', '.join( defaultProtocols ) ) ) modifiedCS = False result = promptUser( 'Do you want to add the following default protocols:' ' %s for SE(s):\n%s' % ( ', '.join( protocolsList ), ', '.join( siteSEs ) ) ) if not result['OK']: return result if result['Value'].lower() != 'y': self.log.always( 'No protocols will be added' ) return S_OK() for se in siteSEs: sections = gConfig.getSections( '/Resources/StorageElements/%s/' % ( se ) ) if not sections['OK']: return sections for section in sections['Value']: if gConfig.getValue( '/Resources/StorageElements/%s/%s/ProtocolName' % ( se, section ), '' ) == 'SRM2': path = '/Resources/StorageElements/%s/%s/ProtocolsList' % ( se, section ) self.log.verbose( 'Setting %s to %s' % ( path, ', '.join( protocolsList ) ) ) result = self.csSetOption( path, ', '.join( protocolsList ) ) if not result['OK']: return result modifiedCS = True if modifiedCS: result = self.csCommitChanges( False ) if not result[ 'OK' ]: return S_ERROR( 'CS Commit failed with message = %s' % ( result[ 'Message' ] ) ) else: if printOutput: print 'Successfully committed changes to CS' else: if printOutput: print 'No modifications to CS required' return S_OK() ############################################################################# def csSetOption( self, optionPath, optionValue ): """ Function to modify an existing value in the CS. """ return self.csAPI.setOption( optionPath, optionValue ) ############################################################################# def csSetOptionComment( self, optionPath, comment ): """ Function to modify an existing value in the CS. """ return self.csAPI.setOptionComment( optionPath, comment ) ############################################################################# def csModifyValue( self, optionPath, newValue ): """ Function to modify an existing value in the CS. """ return self.csAPI.modifyValue( optionPath, newValue ) ############################################################################# def csRegisterUser( self, username, properties ): """ Registers a user in the CS. - username: Username of the user (easy;) - properties: Dict containing: - DN - groups : list/tuple of groups the user belongs to - <others> : More properties of the user, like mail """ return self.csAPI.addUser( username, properties ) ############################################################################# def csDeleteUser( self, user ): """ Deletes a user from the CS. Can take a list of users """ return self.csAPI.deleteUsers( user ) ############################################################################# def csModifyUser( self, username, properties, createIfNonExistant = False ): """ Modify a user in the CS. Takes the same params as in addUser and applies the changes """ return self.csAPI.modifyUser( username, properties, createIfNonExistant ) ############################################################################# def csListUsers( self, group = False ): """ Lists the users in the CS. If no group is specified return all users. """ return self.csAPI.listUsers( group ) ############################################################################# def csDescribeUsers( self, mask = False ): """ List users and their properties in the CS. If a mask is given, only users in the mask will be returned """ return self.csAPI.describeUsers( mask ) ############################################################################# def csModifyGroup( self, groupname, properties, createIfNonExistant = False ): """ Modify a user in the CS. Takes the same params as in addGroup and applies the changes """ return self.csAPI.modifyGroup( groupname, properties, createIfNonExistant ) ############################################################################# def csListHosts( self ): """ Lists the hosts in the CS """ return self.csAPI.listHosts() ############################################################################# def csDescribeHosts( self, mask = False ): """ Gets extended info for the hosts in the CS """ return self.csAPI.describeHosts( mask ) ############################################################################# def csModifyHost( self, hostname, properties, createIfNonExistant = False ): """ Modify a host in the CS. Takes the same params as in addHost and applies the changes """ return self.csAPI.modifyHost( hostname, properties, createIfNonExistant ) ############################################################################# def csListGroups( self ): """ Lists groups in the CS """ return self.csAPI.listGroups() ############################################################################# def csDescribeGroups( self, mask = False ): """ List groups and their properties in the CS. If a mask is given, only groups in the mask will be returned """ return self.csAPI.describeGroups( mask ) ############################################################################# def csSyncUsersWithCFG( self, usersCFG ): """ Synchronize users in cfg with its contents """ return self.csAPI.syncUsersWithCFG( usersCFG ) ############################################################################# def csCommitChanges( self, sortUsers = True ): """ Commit the changes in the CS """ return self.csAPI.commitChanges( sortUsers = False ) ############################################################################# def sendMail( self, address, subject, body, fromAddress = None, localAttempt = True, html = False ): """ Send mail to specified address with body. """ notification = NotificationClient() return notification.sendMail( address, subject, body, fromAddress, localAttempt, html ) ############################################################################# def sendSMS( self, userName, body, fromAddress = None ): """ Send mail to specified address with body. """ if len( body ) > 160: return S_ERROR( 'Exceeded maximum SMS length of 160 characters' ) notification = NotificationClient() return notification.sendSMS( userName, body, fromAddress ) ############################################################################# def getBDIISite( self, site, host = None ): """ Get information about site from BDII at host """ return ldapSite( site, host = host ) ############################################################################# def getBDIICluster( self, ce, host = None ): """ Get information about ce from BDII at host """ return ldapCluster( ce, host = host ) ############################################################################# def getBDIICE( self, ce, host = None ): """ Get information about ce from BDII at host """ return ldapCE( ce, host = host ) ############################################################################# def getBDIIService( self, ce, host = None ): """ Get information about ce from BDII at host """ return ldapService( ce, host = host ) ############################################################################# def getBDIICEState( self, ce, useVO = voName, host = None ): """ Get information about ce state from BDII at host """ return ldapCEState( ce, useVO, host = host ) ############################################################################# def getBDIICEVOView( self, ce, useVO = voName, host = None ): """ Get information about ce voview from BDII at host """ return ldapCEVOView( ce, useVO, host = host ) ############################################################################# def getBDIISE( self, site, useVO = voName, host = None ): """ Get information about SA from BDII at host """ return ldapSE( site, useVO, host = host )