def printCEInfo(voName): resultQueues = Resources.getQueues(community=voName) if not resultQueues['OK']: gLogger.error('Failed to get CE information') DIRACExit(-1) fields = ("Site", 'CE', 'CEType', 'Queue', 'Status') records = [] # get list of usable sites within this cycle resultMask = SiteStatus().getUsableSites() if not resultMask['OK']: return resultMask siteMaskList = resultMask.get('Value', []) rssClient = ResourceStatus() for site in resultQueues['Value']: siteStatus = "Active" if site in siteMaskList else "InActive" siteNew = True for ce in resultQueues['Value'][site]: ceStatus = siteStatus if rssClient.rssFlag: result = rssClient.getElementStatus(ce, "ComputingElement") if result['OK']: ceStatus = result['Value'][ce]['all'] ceNew = True for queue in resultQueues['Value'][site][ce]['Queues']: pSite = site if siteNew else '' pCE = '' ceType = '' if ceNew: pCE = ce ceType = resultQueues['Value'][site][ce]['CEType'] records.append((pSite, pCE, ceType, queue, ceStatus)) ceNew = False siteNew = False gLogger.notice( printTable(fields, records, printOut=False, columnSeparator=' ')) return S_OK()
def printCEInfo(voName): resultQueues = Resources.getQueues(community=voName) if not resultQueues["OK"]: gLogger.error("Failed to get CE information") DIRACExit(-1) fields = ("Site", "CE", "CEType", "Queue", "Status") records = [] # get list of usable sites within this cycle resultMask = SiteStatus().getUsableSites() if not resultMask["OK"]: return resultMask siteMaskList = resultMask.get("Value", []) rssClient = ResourceStatus() for site in resultQueues["Value"]: siteStatus = "Active" if site in siteMaskList else "InActive" siteNew = True for ce in resultQueues["Value"][site]: ceStatus = siteStatus if rssClient.rssFlag: result = rssClient.getElementStatus(ce, "ComputingElement") if result["OK"]: ceStatus = result["Value"][ce]["all"] ceNew = True for queue in resultQueues["Value"][site][ce]["Queues"]: pSite = site if siteNew else "" pCE = "" ceType = "" if ceNew: pCE = ce ceType = resultQueues["Value"][site][ce]["CEType"] records.append((pSite, pCE, ceType, queue, ceStatus)) ceNew = False siteNew = False gLogger.notice( printTable(fields, records, printOut=False, columnSeparator=" ")) return S_OK()
def checkSEs(self, printOutput=False): # pylint: disable=no-self-use """Check the status of read and write operations in the DIRAC SE mask. Example usage: >>> print dirac.checkSEs() {'OK': True, 'Value': {<LFN>:{'<Name>':'<Value>',...},...}} @param printOutput: Optional flag to print result @type printOutput: boolean @return: S_OK,S_ERROR """ res = gConfig.getSections('/Resources/StorageElements', True) if not res['OK']: self.log.error('Failed to get storage element information', res['Message']) return res if printOutput: self.log.notice('%s %s %s' % ('Storage Element'.ljust(25), 'Read Status'.rjust(15), 'Write Status'.rjust(15))) seList = sorted(res['Value']) result = {} rss = ResourceStatus() for se in seList: res = rss.getElementStatus(se, 'StorageElement') if not res['OK']: self.log.error("Failed to get StorageElement status for %s" % se) else: readState = res['Value'].get('ReadAccess', 'Active') writeState = res['Value'].get('WriteAccess', 'Active') result[se] = {'ReadStatus': readState, 'WriteStatus': writeState} if printOutput: self.log.notice('%s %s %s' % (se.ljust(25), readState.rjust(15), writeState.rjust(15))) return S_OK(result)
def main(): read = True write = True check = True remove = True site = '' mute = False Script.registerSwitch("r", "BanRead", " Ban only reading from the storage element") Script.registerSwitch("w", "BanWrite", " Ban writing to the storage element") Script.registerSwitch("k", "BanCheck", " Ban check access to the storage element") Script.registerSwitch("v", "BanRemove", " Ban remove access to the storage element") Script.registerSwitch("a", "All", " Ban all access to the storage element") Script.registerSwitch("m", "Mute", " Do not send email") Script.registerSwitch( "S:", "Site=", " Ban all SEs associate to site (note that if writing is allowed, check is always allowed)") Script.parseCommandLine(ignoreErrors=True) ses = Script.getPositionalArgs() for switch in Script.getUnprocessedSwitches(): if switch[0].lower() in ("r", "banread"): write = False check = False remove = False if switch[0].lower() in ("w", "banwrite"): read = False check = False remove = False if switch[0].lower() in ("k", "bancheck"): read = False write = False remove = False if switch[0].lower() in ("v", "banremove"): read = False write = False check = False if switch[0].lower() in ("a", "all"): pass if switch[0].lower() in ("m", "mute"): mute = True if switch[0].lower() in ("s", "site"): site = switch[1] # from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI from DIRAC import gConfig, gLogger from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getSites from DIRAC.Core.Security.ProxyInfo import getProxyInfo from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin from DIRAC.ResourceStatusSystem.Client.ResourceStatus import ResourceStatus from DIRAC.DataManagementSystem.Utilities.DMSHelpers import resolveSEGroup ses = resolveSEGroup(ses) diracAdmin = DiracAdmin() errorList = [] setup = gConfig.getValue('/DIRAC/Setup', '') if not setup: print('ERROR: Could not contact Configuration Service') DIRAC.exit(2) res = getProxyInfo() if not res['OK']: gLogger.error('Failed to get proxy information', res['Message']) DIRAC.exit(2) userName = res['Value'].get('username') if not userName: gLogger.error('Failed to get username for proxy') DIRAC.exit(2) if site: res = getSites() if not res['OK']: gLogger.error(res['Message']) DIRAC.exit(-1) if site not in res['Value']: gLogger.error('The provided site (%s) is not known.' % site) DIRAC.exit(-1) ses.extend(res['Value']['SE'].replace(' ', '').split(',')) if not ses: gLogger.error('There were no SEs provided') DIRAC.exit(-1) readBanned = [] writeBanned = [] checkBanned = [] removeBanned = [] resourceStatus = ResourceStatus() res = resourceStatus.getElementStatus(ses, "StorageElement") if not res['OK']: gLogger.error("Storage Element %s does not exist" % ses) DIRAC.exit(-1) reason = 'Forced with dirac-admin-ban-se by %s' % userName for se, seOptions in res['Value'].items(): resW = resC = resR = {'OK': False} # Eventually, we will get rid of the notion of InActive, as we always write Banned. if read and 'ReadAccess' in seOptions: if not seOptions['ReadAccess'] in ['Active', 'Degraded', 'Probing']: gLogger.notice('Read option for %s is %s, instead of %s' % (se, seOptions['ReadAccess'], ['Active', 'Degraded', 'Probing'])) gLogger.notice('Try specifying the command switches') else: resR = resourceStatus.setElementStatus(se, 'StorageElement', 'ReadAccess', 'Banned', reason, userName) # res = csAPI.setOption( "%s/%s/ReadAccess" % ( storageCFGBase, se ), "InActive" ) if not resR['OK']: gLogger.error('Failed to update %s read access to Banned' % se) else: gLogger.notice('Successfully updated %s read access to Banned' % se) readBanned.append(se) # Eventually, we will get rid of the notion of InActive, as we always write Banned. if write and 'WriteAccess' in seOptions: if not seOptions['WriteAccess'] in ['Active', 'Degraded', 'Probing']: gLogger.notice('Write option for %s is %s, instead of %s' % (se, seOptions['WriteAccess'], ['Active', 'Degraded', 'Probing'])) gLogger.notice('Try specifying the command switches') else: resW = resourceStatus.setElementStatus(se, 'StorageElement', 'WriteAccess', 'Banned', reason, userName) # res = csAPI.setOption( "%s/%s/WriteAccess" % ( storageCFGBase, se ), "InActive" ) if not resW['OK']: gLogger.error("Failed to update %s write access to Banned" % se) else: gLogger.notice("Successfully updated %s write access to Banned" % se) writeBanned.append(se) # Eventually, we will get rid of the notion of InActive, as we always write Banned. if check and 'CheckAccess' in seOptions: if not seOptions['CheckAccess'] in ['Active', 'Degraded', 'Probing']: gLogger.notice('Check option for %s is %s, instead of %s' % (se, seOptions['CheckAccess'], ['Active', 'Degraded', 'Probing'])) gLogger.notice('Try specifying the command switches') else: resC = resourceStatus.setElementStatus(se, 'StorageElement', 'CheckAccess', 'Banned', reason, userName) # res = csAPI.setOption( "%s/%s/CheckAccess" % ( storageCFGBase, se ), "InActive" ) if not resC['OK']: gLogger.error("Failed to update %s check access to Banned" % se) else: gLogger.notice("Successfully updated %s check access to Banned" % se) checkBanned.append(se) # Eventually, we will get rid of the notion of InActive, as we always write Banned. if remove and 'RemoveAccess' in seOptions: if not seOptions['RemoveAccess'] in ['Active', 'Degraded', 'Probing']: gLogger.notice('Remove option for %s is %s, instead of %s' % (se, seOptions['RemoveAccess'], ['Active', 'Degraded', 'Probing'])) gLogger.notice('Try specifying the command switches') else: resC = resourceStatus.setElementStatus(se, 'StorageElement', 'RemoveAccess', 'Banned', reason, userName) # res = csAPI.setOption( "%s/%s/CheckAccess" % ( storageCFGBase, se ), "InActive" ) if not resC['OK']: gLogger.error("Failed to update %s remove access to Banned" % se) else: gLogger.notice("Successfully updated %s remove access to Banned" % se) removeBanned.append(se) if not(resR['OK'] or resW['OK'] or resC['OK']): DIRAC.exit(-1) if not (writeBanned or readBanned or checkBanned or removeBanned): gLogger.notice("No storage elements were banned") DIRAC.exit(-1) if mute: gLogger.notice('Email is muted by script switch') DIRAC.exit(0) subject = '%s storage elements banned for use' % len(writeBanned + readBanned + checkBanned + removeBanned) addressPath = 'EMail/Production' address = Operations().getValue(addressPath, '') body = '' if read: body = "%s\n\nThe following storage elements were banned for reading:" % body for se in readBanned: body = "%s\n%s" % (body, se) if write: body = "%s\n\nThe following storage elements were banned for writing:" % body for se in writeBanned: body = "%s\n%s" % (body, se) if check: body = "%s\n\nThe following storage elements were banned for check access:" % body for se in checkBanned: body = "%s\n%s" % (body, se) if remove: body = "%s\n\nThe following storage elements were banned for remove access:" % body for se in removeBanned: body = "%s\n%s" % (body, se) if not address: gLogger.notice("'%s' not defined in Operations, can not send Mail\n" % addressPath, body) DIRAC.exit(0) res = diracAdmin.sendMail(address, subject, body) gLogger.notice('Notifying %s' % address) if res['OK']: gLogger.notice(res['Value']) else: gLogger.notice(res['Message']) DIRAC.exit(0)
class FTS3ServerPolicy(object): """ This class manages the policy for choosing a server """ def __init__(self, serverDict, serverPolicy="Random"): """ Call the init of the parent, and initialize the list of FTS3 servers """ self.log = gLogger.getSubLogger("FTS3ServerPolicy") self._serverDict = serverDict self._serverList = serverDict.keys() self._maxAttempts = len(self._serverList) self._nextServerID = 0 self._resourceStatus = ResourceStatus() methName = "_%sServerPolicy" % serverPolicy.lower() if not hasattr(self, methName): self.log.error('Unknown server policy %s. Using Random instead' % serverPolicy) methName = "_randomServerPolicy" self._policyMethod = getattr(self, methName) def _failoverServerPolicy(self, _attempt): """ Returns always the server at a given position (normally the first one) :param attempt: position of the server in the list """ if _attempt >= len(self._serverList): raise Exception( "FTS3ServerPolicy.__failoverServerPolicy: attempt to reach non existing server index" ) return self._serverList[_attempt] def _sequenceServerPolicy(self, _attempt): """ Every time the this policy is called, return the next server on the list """ fts3server = self._serverList[self._nextServerID] self._nextServerID = (self._nextServerID + 1) % len(self._serverList) return fts3server def _randomServerPolicy(self, _attempt): """ return a server from shuffledServerList """ if getattr(threadLocal, 'shuffledServerList', None) is None: threadLocal.shuffledServerList = self._serverList[:] random.shuffle(threadLocal.shuffledServerList) fts3Server = threadLocal.shuffledServerList[_attempt] if _attempt == self._maxAttempts - 1: random.shuffle(threadLocal.shuffledServerList) return fts3Server def _getFTSServerStatus(self, ftsServer): """ Fetch the status of the FTS server from RSS """ res = self._resourceStatus.getElementStatus(ftsServer, 'FTS') if not res['OK']: return res result = res['Value'] if ftsServer not in result: return S_ERROR("No FTS Server %s known to RSS" % ftsServer) if result[ftsServer]['all'] == 'Active': return S_OK(True) return S_OK(False) def chooseFTS3Server(self): """ Choose the appropriate FTS3 server depending on the policy """ fts3Server = None attempt = 0 while not fts3Server and attempt < self._maxAttempts: fts3Server = self._policyMethod(attempt) res = self._getFTSServerStatus(fts3Server) if not res['OK']: self.log.warn("Error getting the RSS status for %s: %s" % (fts3Server, res)) fts3Server = None attempt += 1 continue ftsServerStatus = res['Value'] if not ftsServerStatus: self.log.warn( 'FTS server %s is not in good shape. Choose another one' % fts3Server) fts3Server = None attempt += 1 if fts3Server: return S_OK(self._serverDict[fts3Server]) return S_ERROR("Could not find an FTS3 server (max attempt reached)")
gLogger.error('Failed to get Site mask information') DIRACExit(-1) siteMaskList = resultMask.get('Value', []) rssClient = ResourceStatus() fields = ('Site', 'CE', 'Queue', 'Status', 'Match', 'Reason') records = [] for queue, queueInfo in queueDict.iteritems(): site = queueInfo['Site'] ce = queueInfo['CEName'] siteStatus = "Active" if site in siteMaskList else "InActive" ceStatus = siteStatus if rssClient.rssFlag: result = rssClient.getElementStatus(ce, "ComputingElement") if result['OK']: ceStatus = result['Value'][ce]['all'] result = matchQueue(jdl, queueInfo, fullMatch=fullMatch) if not result['OK']: gLogger.error('Failed in getting match data', result['Message']) DIRACExit(-1) status = "Active" if siteStatus is "Active" and ceStatus is "Active" else "Inactive" if result['Value']['Match']: records.append((site, ce, queueInfo['Queue'], status, 'Yes', '')) else: records.append((site, ce, queueInfo['Queue'], status, 'No', result['Value']['Reason'])) gLogger.notice(
def main(): read = True write = True check = True remove = True sites = [] mute = False Script.registerSwitch("r", "BanRead", " Ban only reading from the storage element") Script.registerSwitch("w", "BanWrite", " Ban writing to the storage element") Script.registerSwitch("k", "BanCheck", " Ban check access to the storage element") Script.registerSwitch("v", "BanRemove", " Ban remove access to the storage element") Script.registerSwitch("a", "All", " Ban all access to the storage element") Script.registerSwitch("m", "Mute", " Do not send email") Script.registerSwitch( "S:", "Site=", " Ban all SEs associate to site (note that if writing is allowed, check is always allowed)" ) # Registering arguments will automatically add their description to the help menu Script.registerArgument( ["seGroupList: list of SEs or comma-separated SEs"]) switches, ses = Script.parseCommandLine(ignoreErrors=True) for switch in switches: if switch[0].lower() in ("r", "banread"): write = False check = False remove = False if switch[0].lower() in ("w", "banwrite"): read = False check = False remove = False if switch[0].lower() in ("k", "bancheck"): read = False write = False remove = False if switch[0].lower() in ("v", "banremove"): read = False write = False check = False if switch[0].lower() in ("a", "all"): pass if switch[0].lower() in ("m", "mute"): mute = True if switch[0].lower() in ("s", "site"): sites = switch[1].split(",") # from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI from DIRAC import gConfig, gLogger from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations from DIRAC.Core.Security.ProxyInfo import getProxyInfo from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin from DIRAC.ResourceStatusSystem.Client.ResourceStatus import ResourceStatus from DIRAC.DataManagementSystem.Utilities.DMSHelpers import resolveSEGroup, DMSHelpers ses = resolveSEGroup(ses) diracAdmin = DiracAdmin() setup = gConfig.getValue("/DIRAC/Setup", "") if not setup: print("ERROR: Could not contact Configuration Service") DIRAC.exit(2) res = getProxyInfo() if not res["OK"]: gLogger.error("Failed to get proxy information", res["Message"]) DIRAC.exit(2) userName = res["Value"].get("username") if not userName: gLogger.error("Failed to get username for proxy") DIRAC.exit(2) for site in sites: res = DMSHelpers().getSEsForSite(site) if not res["OK"]: gLogger.error(res["Message"], site) DIRAC.exit(-1) ses.extend(res["Value"]) if not ses: gLogger.error("There were no SEs provided") DIRAC.exit(-1) readBanned = [] writeBanned = [] checkBanned = [] removeBanned = [] resourceStatus = ResourceStatus() res = resourceStatus.getElementStatus(ses, "StorageElement") if not res["OK"]: gLogger.error("Storage Element %s does not exist" % ses) DIRAC.exit(-1) reason = "Forced with dirac-admin-ban-se by %s" % userName for se, seOptions in res["Value"].items(): resW = resC = resR = {"OK": False} # Eventually, we will get rid of the notion of InActive, as we always write Banned. if read and "ReadAccess" in seOptions: if seOptions["ReadAccess"] == "Banned": gLogger.notice("Read access already banned", se) resR["OK"] = True elif not seOptions["ReadAccess"] in [ "Active", "Degraded", "Probing" ]: gLogger.notice("Read option for %s is %s, instead of %s" % (se, seOptions["ReadAccess"], ["Active", "Degraded", "Probing"])) gLogger.notice("Try specifying the command switches") else: resR = resourceStatus.setElementStatus(se, "StorageElement", "ReadAccess", "Banned", reason, userName) # res = csAPI.setOption( "%s/%s/ReadAccess" % ( storageCFGBase, se ), "InActive" ) if not resR["OK"]: gLogger.error("Failed to update %s read access to Banned" % se) else: gLogger.notice( "Successfully updated %s read access to Banned" % se) readBanned.append(se) # Eventually, we will get rid of the notion of InActive, as we always write Banned. if write and "WriteAccess" in seOptions: if seOptions["WriteAccess"] == "Banned": gLogger.notice("Write access already banned", se) resW["OK"] = True elif not seOptions["WriteAccess"] in [ "Active", "Degraded", "Probing" ]: gLogger.notice("Write option for %s is %s, instead of %s" % (se, seOptions["WriteAccess"], ["Active", "Degraded", "Probing"])) gLogger.notice("Try specifying the command switches") else: resW = resourceStatus.setElementStatus(se, "StorageElement", "WriteAccess", "Banned", reason, userName) # res = csAPI.setOption( "%s/%s/WriteAccess" % ( storageCFGBase, se ), "InActive" ) if not resW["OK"]: gLogger.error( "Failed to update %s write access to Banned" % se) else: gLogger.notice( "Successfully updated %s write access to Banned" % se) writeBanned.append(se) # Eventually, we will get rid of the notion of InActive, as we always write Banned. if check and "CheckAccess" in seOptions: if seOptions["CheckAccess"] == "Banned": gLogger.notice("Check access already banned", se) resC["OK"] = True elif not seOptions["CheckAccess"] in [ "Active", "Degraded", "Probing" ]: gLogger.notice("Check option for %s is %s, instead of %s" % (se, seOptions["CheckAccess"], ["Active", "Degraded", "Probing"])) gLogger.notice("Try specifying the command switches") else: resC = resourceStatus.setElementStatus(se, "StorageElement", "CheckAccess", "Banned", reason, userName) # res = csAPI.setOption( "%s/%s/CheckAccess" % ( storageCFGBase, se ), "InActive" ) if not resC["OK"]: gLogger.error( "Failed to update %s check access to Banned" % se) else: gLogger.notice( "Successfully updated %s check access to Banned" % se) checkBanned.append(se) # Eventually, we will get rid of the notion of InActive, as we always write Banned. if remove and "RemoveAccess" in seOptions: if seOptions["RemoveAccess"] == "Banned": gLogger.notice("Remove access already banned", se) resC["OK"] = True elif not seOptions["RemoveAccess"] in [ "Active", "Degraded", "Probing" ]: gLogger.notice("Remove option for %s is %s, instead of %s" % (se, seOptions["RemoveAccess"], ["Active", "Degraded", "Probing"])) gLogger.notice("Try specifying the command switches") else: resC = resourceStatus.setElementStatus(se, "StorageElement", "RemoveAccess", "Banned", reason, userName) # res = csAPI.setOption( "%s/%s/CheckAccess" % ( storageCFGBase, se ), "InActive" ) if not resC["OK"]: gLogger.error( "Failed to update %s remove access to Banned" % se) else: gLogger.notice( "Successfully updated %s remove access to Banned" % se) removeBanned.append(se) if not (resR["OK"] or resW["OK"] or resC["OK"]): DIRAC.exit(-1) if not (writeBanned or readBanned or checkBanned or removeBanned): gLogger.notice("No storage elements were banned") DIRAC.exit(-1) if mute: gLogger.notice("Email is muted by script switch") DIRAC.exit(0) subject = "%s storage elements banned for use" % len( writeBanned + readBanned + checkBanned + removeBanned) addressPath = "EMail/Production" address = Operations().getValue(addressPath, "") body = "" if read: body = "%s\n\nThe following storage elements were banned for reading:" % body for se in readBanned: body = "%s\n%s" % (body, se) if write: body = "%s\n\nThe following storage elements were banned for writing:" % body for se in writeBanned: body = "%s\n%s" % (body, se) if check: body = "%s\n\nThe following storage elements were banned for check access:" % body for se in checkBanned: body = "%s\n%s" % (body, se) if remove: body = "%s\n\nThe following storage elements were banned for remove access:" % body for se in removeBanned: body = "%s\n%s" % (body, se) if not address: gLogger.notice( "'%s' not defined in Operations, can not send Mail\n" % addressPath, body) DIRAC.exit(0) res = diracAdmin.sendMail(address, subject, body) gLogger.notice("Notifying %s" % address) if res["OK"]: gLogger.notice(res["Value"]) else: gLogger.notice(res["Message"]) DIRAC.exit(0)
class StorageFactory(object): def __init__(self, useProxy=False, vo=None): self.proxy = False self.proxy = useProxy self.resourceStatus = ResourceStatus() self.vo = vo if self.vo is None: result = getVOfromProxyGroup() if result['OK']: self.vo = result['Value'] else: RuntimeError("Can not get the current VO context") self.remotePlugins = [] self.localPlugins = [] self.name = '' self.options = {} self.protocols = {} self.storages = [] ########################################################################################### # # Below are public methods for obtaining storage objects # def getStorageName(self, initialName): return self._getConfigStorageName(initialName, 'Alias') def getStorage(self, parameterDict, hideExceptions=False): """ This instantiates a single storage for the details provided and doesn't check the CS. """ # The storage name must be supplied. if 'StorageName' in parameterDict: storageName = parameterDict['StorageName'] else: errStr = "StorageFactory.getStorage: StorageName must be supplied" gLogger.error(errStr) return S_ERROR(errStr) # PluginName must be supplied otherwise nothing with work. pluginName = parameterDict.get('PluginName') if not pluginName: errStr = "StorageFactory.getStorage: PluginName must be supplied" gLogger.error(errStr) return S_ERROR(errStr) return self.__generateStorageObject(storageName, pluginName, parameterDict, hideExceptions=hideExceptions) def getStorages(self, storageName, pluginList=None, hideExceptions=False): """ Get an instance of a Storage based on the DIRAC SE name based on the CS entries CS :param storageName: is the DIRAC SE name i.e. 'CERN-RAW' :param pluginList: is an optional list of protocols if a sub-set is desired i.e ['SRM2','SRM1'] :return: dictionary containing storage elements and information about them """ self.remotePlugins = [] self.localPlugins = [] self.name = '' self.options = {} self.protocols = {} self.storages = [] if pluginList is None: pluginList = [] elif isinstance(pluginList, six.string_types): pluginList = [pluginList] if not self.vo: gLogger.warn('No VO information available') # Get the name of the storage provided res = self._getConfigStorageName(storageName, 'Alias') if not res['OK']: return res storageName = res['Value'] self.name = storageName # In case the storage is made from a base SE, get this information res = self._getConfigStorageName(storageName, 'BaseSE') if not res['OK']: return res # If the storage is derived frmo another one, keep the information # We initialize the seConfigPath to SE_BASE_CONFIG_PATH if there is a derivedSE, SE_CONFIG_PATH if not if res['Value'] != storageName: derivedStorageName = storageName storageName = res['Value'] seConfigPath = SE_BASE_CONFIG_PATH else: derivedStorageName = None seConfigPath = SE_CONFIG_PATH # Get the options defined in the CS for this storage res = self._getConfigStorageOptions(storageName, derivedStorageName=derivedStorageName, seConfigPath=seConfigPath) if not res['OK']: # This is for the backward compatibility and to invite developer to move their BaseSE in the correct section gLogger.warn("Deprecated configuration, you can ignore the error message above." " Please move the baseSE in the correct section: ", SE_BASE_CONFIG_PATH) # We change the value of seConfigPath to avoid other errors due to the bad SE_BASE_CONFIG_PATH seConfigPath = SE_CONFIG_PATH res = self._getConfigStorageOptions(storageName, derivedStorageName=derivedStorageName, seConfigPath=seConfigPath) if not res['OK']: return res self.options = res['Value'] # Get the protocol specific details res = self._getConfigStorageProtocols(storageName, derivedStorageName=derivedStorageName, seConfigPath=seConfigPath) if not res['OK']: return res self.protocols = res['Value'] requestedLocalPlugins = [] requestedRemotePlugins = [] requestedProtocolDetails = [] turlProtocols = [] # Generate the protocol specific plug-ins for protocolSection, protocolDetails in self.protocols.iteritems(): pluginName = protocolDetails.get('PluginName', protocolSection) if pluginList and pluginName not in pluginList: continue protocol = protocolDetails['Protocol'] result = self.__generateStorageObject(storageName, pluginName, protocolDetails, hideExceptions=hideExceptions) if result['OK']: self.storages.append(result['Value']) if pluginName in self.localPlugins: turlProtocols.append(protocol) requestedLocalPlugins.append(pluginName) if pluginName in self.remotePlugins: requestedRemotePlugins.append(pluginName) requestedProtocolDetails.append(protocolDetails) else: gLogger.info(result['Message']) if self.storages: resDict = {} resDict['StorageName'] = self.name resDict['StorageOptions'] = self.options resDict['StorageObjects'] = self.storages resDict['LocalPlugins'] = requestedLocalPlugins resDict['RemotePlugins'] = requestedRemotePlugins resDict['ProtocolOptions'] = requestedProtocolDetails resDict['TurlProtocols'] = turlProtocols return S_OK(resDict) else: errStr = "StorageFactory.getStorages: Failed to instantiate any storage protocols." gLogger.error(errStr, self.name) return S_ERROR(errStr) ########################################################################################### # # Below are internal methods for obtaining section/option/value configuration # def _getConfigStorageName(self, storageName, referenceType, seConfigPath=SE_CONFIG_PATH): """ This gets the name of the storage the configuration service. If the storage is a reference to another SE the resolution is performed. :param storageName: is the storage section to check in the CS :param referenceType: corresponds to an option inside the storage section :param seConfigPath: the path of the storage section. It can be /Resources/StorageElements or StorageElementBases :return: the name of the storage """ configPath = '%s/%s' % (seConfigPath, storageName) res = gConfig.getOptions(configPath) if not res['OK']: errStr = "StorageFactory._getConfigStorageName: Failed to get storage options" gLogger.error(errStr, res['Message']) return S_ERROR(errStr) if not res['Value']: errStr = "StorageFactory._getConfigStorageName: Supplied storage doesn't exist." gLogger.error(errStr, configPath) return S_ERROR(errStr) if referenceType in res['Value']: configPath = cfgPath(seConfigPath, storageName, referenceType) referenceName = gConfig.getValue(configPath) result = self._getConfigStorageName(referenceName, 'Alias', seConfigPath=SE_BASE_CONFIG_PATH) if not result['OK']: # This is for the backward compatibility and to invite developer to move their BaseSE in the correct section gLogger.warn("Deprecated configuration, you can ignore the error message above." " Please move the baseSE in the correct section: ", SE_BASE_CONFIG_PATH) result = self._getConfigStorageName(referenceName, 'Alias', seConfigPath=SE_CONFIG_PATH) if not result['OK']: return result resolvedName = result['Value'] else: resolvedName = storageName return S_OK(resolvedName) def _getConfigStorageOptions(self, storageName, derivedStorageName=None, seConfigPath=SE_CONFIG_PATH): """ Get the options associated to the StorageElement as defined in the CS :param storageName: is the storage section to check in the CS :param seConfigPath: the path of the storage section. It can be /Resources/StorageElements or StorageElementBases :param derivedStorageName: is the storage section of a derived storage if it inherits from a base :return: options associated to the StorageElement as defined in the CS """ optionsDict = {} # We first get the options of the baseSE, and then overwrite with the derivedSE for seName in (storageName, derivedStorageName) if derivedStorageName else (storageName, ): storageConfigPath = cfgPath(seConfigPath, seName) res = gConfig.getOptions(storageConfigPath) if not res['OK']: errStr = "StorageFactory._getStorageOptions: Failed to get storage options." gLogger.error(errStr, "%s: %s" % (seName, res['Message'])) return S_ERROR(errStr) for option in set(res['Value']) - set(('ReadAccess', 'WriteAccess', 'CheckAccess', 'RemoveAccess')): optionConfigPath = cfgPath(storageConfigPath, option) default = [] if option in ['VO', 'AccessProtocols', 'WriteProtocols'] else '' optionsDict[option] = gConfig.getValue(optionConfigPath, default) # We update the seConfigPath in order to find option in derivedSE now seConfigPath = SE_CONFIG_PATH # The status is that of the derived SE only seName = derivedStorageName if derivedStorageName else storageName res = self.resourceStatus.getElementStatus(seName, "StorageElement") if not res['OK']: errStr = "StorageFactory._getStorageOptions: Failed to get storage status" gLogger.error(errStr, "%s: %s" % (seName, res['Message'])) return S_ERROR(errStr) # For safety, we did not add the ${statusType}Access keys # this requires modifications in the StorageElement class # We add the dictionary with the statusTypes and values # { 'statusType1' : 'status1', 'statusType2' : 'status2' ... } optionsDict.update(res['Value'][seName]) return S_OK(optionsDict) def __getProtocolsSections(self, storageName, seConfigPath=SE_CONFIG_PATH): """ Get the protocols of a specific storage section :param storageName: is the storage section to check in the CS :param seConfigPath: the path of the storage section. It can be /Resources/StorageElements or StorageElementBases :return: list of protocol section names """ storageConfigPath = cfgPath(seConfigPath, storageName) res = gConfig.getSections(storageConfigPath) if not res['OK']: errStr = "StorageFactory._getConfigStorageProtocols: Failed to get storage sections" gLogger.error(errStr, "%s: %s" % (storageName, res['Message'])) return S_ERROR(errStr) protocolSections = res['Value'] return S_OK(protocolSections) def _getConfigStorageProtocols(self, storageName, derivedStorageName=None, seConfigPath=SE_CONFIG_PATH): """ Make a dictionary of protocols with the information associated. Merge with a base SE if it exists :param storageName: is the storage section to check in the CS :param seConfigPath: the path of the storage section. It can be /Resources/StorageElements or StorageElementBases :param derivedStorageName: is the storage section of a derived storage if it inherits from a base :return: dictionary of protocols like {protocolSection: {protocolOptions}} """ # Get the sections res = self.__getProtocolsSections(storageName, seConfigPath=seConfigPath) if not res['OK']: return res protocolSections = res['Value'] sortedProtocolSections = sorted(protocolSections) # Get the details for each section in a dictionary for protocolSection in sortedProtocolSections: res = self._getConfigStorageProtocolDetails(storageName, protocolSection, seConfigPath=seConfigPath) if not res['OK']: return res self.protocols[protocolSection] = res['Value'] if derivedStorageName: # We may have parameters overwriting the baseSE protocols res = self.__getProtocolsSections(derivedStorageName, seConfigPath=SE_CONFIG_PATH) if not res['OK']: return res for protocolSection in res['Value']: res = self._getConfigStorageProtocolDetails( derivedStorageName, protocolSection, seConfigPath=SE_CONFIG_PATH, checkAccess=False) if not res['OK']: return res detail = res['Value'] # If we found the plugin section from which we inherit inheritanceMatched = False for baseStorageProtocolSection in protocolSections: if protocolSection == baseStorageProtocolSection: inheritanceMatched = True for key, val in detail.iteritems(): if val: self.protocols[protocolSection][key] = val break # If not matched, consider it a new protocol if not inheritanceMatched: self.protocols[protocolSection] = detail return S_OK(self.protocols) def _getConfigStorageProtocolDetails(self, storageName, protocolSection, seConfigPath=SE_CONFIG_PATH, checkAccess=True): """ Parse the contents of the protocol block :param storageName: is the storage section to check in the CS :param protocolSection: name of the protocol section to find information :param seConfigPath: the path of the storage section. It can be /Resources/StorageElements or StorageElementBases :param checkAccess: if not set, don't complain if "Access" is not in the section :return: dictionary of the protocol options """ # First obtain the options that are available protocolConfigPath = cfgPath(seConfigPath, storageName, protocolSection) res = gConfig.getOptions(protocolConfigPath) if not res['OK']: errStr = "StorageFactory.__getProtocolDetails: Failed to get protocol options." gLogger.error(errStr, "%s: %s" % (storageName, protocolSection)) return S_ERROR(errStr) options = res['Value'] # We must have certain values internally even if not supplied in CS protocolDict = {'Access': '', 'Host': '', 'Path': '', 'Port': '', 'Protocol': '', 'SpaceToken': '', 'WSUrl': ''} for option in options: configPath = cfgPath(protocolConfigPath, option) optionValue = gConfig.getValue(configPath, '') protocolDict[option] = optionValue # Evaluate the base path taking into account possible VO specific setting if self.vo: result = gConfig.getOptionsDict(cfgPath(protocolConfigPath, 'VOPath')) voPath = '' if result['OK']: voPath = result['Value'].get(self.vo, '') if voPath: protocolDict['Path'] = voPath # Now update the local and remote protocol lists. # A warning will be given if the Access option is not set and the plugin is not already in remote or local. plugin = protocolDict.get('PluginName', protocolSection) if protocolDict['Access'].lower() == 'remote': self.remotePlugins.append(plugin) elif protocolDict['Access'].lower() == 'local': self.localPlugins.append(plugin) # If it is a derived SE, this is normal, no warning elif checkAccess and protocolSection not in self.protocols: errStr = "StorageFactory.__getProtocolDetails: The 'Access' option \ for %s:%s is neither 'local' or 'remote'." % (storageName, protocolSection) gLogger.warn(errStr) return S_OK(protocolDict) ########################################################################################### # # Below is the method for obtaining the object instantiated for a provided storage configuration # def __generateStorageObject(self, storageName, pluginName, parameters, hideExceptions=False): """ Generate a Storage Element from parameters collected :param storageName: is the storage section to check in the CS :param pluginName: name of the plugin used. Example: GFAL2_XROOT, GFAL2_SRM2... :param parameters: dictionary of protocol details. """ storageType = pluginName if self.proxy: storageType = 'Proxy' objectLoader = ObjectLoader() result = objectLoader.loadObject('Resources.Storage.%sStorage' % storageType, storageType + 'Storage', hideExceptions=hideExceptions) if not result['OK']: gLogger.error('Failed to load storage object: %s' % result['Message']) return result storageClass = result['Value'] try: storage = storageClass(storageName, parameters) except Exception as x: errStr = "StorageFactory._generateStorageObject: Failed to instantiate %s: %s" % (storageName, x) gLogger.exception(errStr) return S_ERROR(errStr) return S_OK(storage)
STATUS_TYPES = [ "ReadAccess", "WriteAccess", "CheckAccess", "RemoveAccess" ] ALLOWED_STATUSES = [ "Unknown", "InActive", "Banned", "Probing", "Degraded" ] statusAllowedDict = {} for statusType in STATUS_TYPES: statusAllowedDict[statusType] = [] statusFlagDict = {} statusFlagDict['ReadAccess'] = read statusFlagDict['WriteAccess'] = write statusFlagDict['CheckAccess'] = check statusFlagDict['RemoveAccess'] = remove resourceStatus = ResourceStatus() res = resourceStatus.getElementStatus( ses, "StorageElement" ) if not res[ 'OK' ]: gLogger.error( 'Storage Element %s does not exist' % ses ) DIRAC.exit( -1 ) reason = 'Forced with dirac-admin-allow-se by %s' % userName for se, seOptions in res[ 'Value' ].iteritems(): # InActive is used on the CS model, Banned is the equivalent in RSS for statusType in STATUS_TYPES: if statusFlagDict[statusType]: if seOptions.get( statusType ) == "Active": gLogger.notice( '%s status of %s is already Active' % ( statusType, se ) ) continue if statusType in seOptions:
def executeForVO(self, vo): """ Perform the synchronisation for one VO. :param vo: VO name :return: S_OK or S_ERROR """ rSS = ResourceStatus() try: try: self.log.info( "Login to Rucio as privileged user with host cert/key") certKeyTuple = Locations.getHostCertificateAndKeyLocation() if not certKeyTuple: self.log.error("Hostcert/key location not set") return S_ERROR("Hostcert/key location not set") hostcert, hostkey = certKeyTuple self.log.info("Logging in with a host cert/key pair:") self.log.debug("account: ", self.clientConfig[vo]["privilegedAccount"]) self.log.debug("rucio host: ", self.clientConfig[vo]["rucioHost"]) self.log.debug("auth host: ", self.clientConfig[vo]["authHost"]) self.log.debug("CA cert path: ", self.caCertPath) self.log.debug("Cert location: ", hostcert) self.log.debug("Key location: ", hostkey) self.log.debug("VO: ", vo) client = Client( account=self.clientConfig[vo]["privilegedAccount"], rucio_host=self.clientConfig[vo]["rucioHost"], auth_host=self.clientConfig[vo]["authHost"], ca_cert=self.caCertPath, auth_type="x509", creds={ "client_cert": hostcert, "client_key": hostkey }, timeout=600, user_agent="rucio-clients", vo=vo, ) except Exception as err: self.log.info( "Login to Rucio as privileged user with host cert/key failed. Try username/password" ) client = Client(account="root", auth_type="userpass") except Exception as exc: # login exception, skip this VO self.log.exception("Login for VO failed. VO skipped ", "VO=%s" % vo, lException=exc) return S_ERROR(str(format_exc())) self.log.info( " Rucio login successful - continue with the RSS synchronisation") # return S_OK() try: for rse in client.list_rses(): thisSe = rse["rse"] self.log.info("Checking Dirac SE status for %s" % thisSe) resStatus = rSS.getElementStatus(thisSe, "StorageElement", vO=vo) dictSe = client.get_rse(thisSe) if resStatus["OK"]: self.log.debug("SE status ", resStatus["Value"]) seAccessValue = resStatus["Value"][thisSe] availabilityRead = True if seAccessValue["ReadAccess"] in [ "Active", "Degraded" ] else False availabilityWrite = True if seAccessValue[ "WriteAccess"] in ["Active", "Degraded"] else False availabilityDelete = True if seAccessValue[ "RemoveAccess"] in ["Active", "Degraded"] else False isUpdated = False if dictSe["availability_read"] != availabilityRead: self.log.info( "Set availability_read for RSE", "RSE: %s, availability: %s" % (thisSe, availabilityRead)) client.update_rse( thisSe, {"availability_read": availabilityRead}) isUpdated = True if dictSe["availability_write"] != availabilityWrite: self.log.info( "Set availability_write for RSE", "RSE: %s, availability: %s" % (thisSe, availabilityWrite)) client.update_rse( thisSe, {"availability_write": availabilityWrite}) isUpdated = True if dictSe["availability_delete"] != availabilityDelete: self.log.info( "Set availability_delete for RSE", "RSE: %s, availability: %s" % (thisSe, availabilityDelete), ) client.update_rse( thisSe, {"availability_delete": availabilityDelete}) isUpdated = True except Exception as err: return S_ERROR(str(err)) return S_OK()
class FTS3Placement(FTSAbstractPlacement): """ This class manages all the FTS strategies, routes and what not """ __serverPolicy = "Random" __nextServerID = 0 __serverList = None __maxAttempts = 0 def __init__(self, csPath=None, ftsHistoryViews=None): """ Call the init of the parent, and initialize the list of FTS3 servers """ self.log = gLogger.getSubLogger("FTS3Placement") super(FTS3Placement, self).__init__(csPath=csPath, ftsHistoryViews=ftsHistoryViews) srvList = getFTS3Servers() if not srvList['OK']: self.log.error(srvList['Message']) self.__serverList = srvList.get('Value', []) self.maxAttempts = len(self.__serverList) self.rssClient = ResourceStatus() def getReplicationTree(self, sourceSEs, targetSEs, size, strategy=None): """ For multiple source to multiple destination, find the optimal replication strategy. :param sourceSEs : list of source SE :param targetSEs : list of destination SE :param size : size of the File :param strategy : which strategy to use :returns S_OK(dict) < route name : { dict with key Ancestor, SourceSE, TargetSEtargetSE, Strategy } > For the time being, we are waiting for FTS3 to provide advisory mechanisms. So we just use simple techniques """ # We will use a single random source sourceSE = random.choice(sourceSEs) tree = {} for targetSE in targetSEs: tree["%s#%s" % (sourceSE, targetSE)] = { "Ancestor": False, "SourceSE": sourceSE, "TargetSE": targetSE, "Strategy": "FTS3Simple" } return S_OK(tree) def refresh(self, ftsHistoryViews): """ Refresh, whatever that means... recalculate all what you need, fetches the latest conf and what not. """ return super(FTS3Placement, self).refresh(ftsHistoryViews=ftsHistoryViews) def __failoverServerPolicy(self, attempt=0): """ Returns always the server at a given position (normally the first one) :param attempt: position of the server in the list """ if attempt >= len(self.__serverList): raise Exception( "FTS3Placement.__failoverServerPolicy: attempt to reach non existing server index" ) return self.__serverList[attempt] def __sequenceServerPolicy(self): """ Every time the this policy is called, return the next server on the list """ fts3server = self.__serverList[self.__nextServerID] self.__nextServerID = (self.__nextServerID + 1) % len( self.__serverList) return fts3server def __randomServerPolicy(self): """ return a random server from the list """ return random.choice(self.__serverList) def __chooseFTS3Server(self): """ Choose the appropriate FTS3 server depending on the policy """ fts3Server = None attempt = 0 # FIXME : need to get real valeu from RSS ftsServerStatus = True while not fts3Server and attempt < self.maxAttempts: if self.__serverPolicy == 'Random': fts3Server = self.__randomServerPolicy() elif self.__serverPolicy == 'Sequence': fts3Server = self.__sequenceServerPolicy() elif self.__serverPolicy == 'Failover': fts3Server = self.__failoverServerPolicy(attempt=attempt) else: self.log.error( 'Unknown server policy %s. Using Random instead' % self.__serverPolicy) fts3Server = self.__randomServerPolicy() if not ftsServerStatus: self.log.warn( 'FTS server %s is not in good shape. Choose another one' % fts3Server) fts3Server = None attempt += 1 # FIXME : I need to get the FTS server status from RSS # ftsStatusFromRss = rss.ftsStatusOrSomethingLikeThat if fts3Server: return S_OK(fts3Server) return S_ERROR("Could not find an FTS3 server (max attempt reached)") def findRoute(self, sourceSE, targetSE): """ Find the appropriate route from point A to B :param sourceSE : source SE :param targetSE : destination SE :returns: S_OK(FTSRoute) """ fts3server = self.__chooseFTS3Server() if not fts3server['OK']: return fts3server fts3server = fts3server['Value'] route = FTSRoute(sourceSE, targetSE, fts3server) return S_OK(route) def isRouteValid(self, route): """ FIXME: until RSS is ready, I check manually the status In FTS3, all routes are valid a priori. If a route was not valid for some reason, then FTS would know it thanks to the blacklist sent by RSS, and would deal with it itself. :param route : FTSRoute :returns: S_OK or S_ERROR(reason) """ rAccess = self.rssClient.getElementStatus(route.sourceSE, "StorageElement", "ReadAccess") self.log.debug("se read %s %s" % (route.sourceSE, rAccess)) if not rAccess["OK"]: self.log.error(rAccess["Message"]) return rAccess if rAccess["Value"][route.sourceSE]["ReadAccess"] not in ("Active", "Degraded"): return S_ERROR("Source SE is not readable") wAccess = self.rssClient.getElementStatus(route.targetSE, "StorageElement", "WriteAccess") self.log.debug("se write %s %s" % (route.targetSE, wAccess)) if not wAccess["OK"]: self.log.error(wAccess["Message"]) return wAccess if wAccess["Value"][route.targetSE]["WriteAccess"] not in ("Active", "Degraded"): return S_ERROR("Target SE is not writable") return S_OK()
class FTS3ServerPolicy(object): """ This class manages the policy for choosing a server """ def __init__(self, serverDict, serverPolicy="Random"): """ Call the init of the parent, and initialize the list of FTS3 servers """ self.log = gLogger.getSubLogger("FTS3ServerPolicy") self._serverDict = serverDict self._serverList = serverDict.keys() self._maxAttempts = len(self._serverList) self._nextServerID = 0 self._resourceStatus = ResourceStatus() methName = "_%sServerPolicy" % serverPolicy.lower() if not hasattr(self, methName): self.log.error('Unknown server policy %s. Using Random instead' % serverPolicy) methName = "_randomServerPolicy" self._policyMethod = getattr(self, methName) def _failoverServerPolicy(self, _attempt): """ Returns always the server at a given position (normally the first one) :param attempt: position of the server in the list """ if _attempt >= len(self._serverList): raise Exception( "FTS3ServerPolicy.__failoverServerPolicy: attempt to reach non existing server index") return self._serverList[_attempt] def _sequenceServerPolicy(self, _attempt): """ Every time the this policy is called, return the next server on the list """ fts3server = self._serverList[self._nextServerID] self._nextServerID = (self._nextServerID + 1) % len(self._serverList) return fts3server def _randomServerPolicy(self, _attempt): """ return a server from shuffledServerList """ if getattr(threadLocal, 'shuffledServerList', None) is None: threadLocal.shuffledServerList = self._serverList[:] random.shuffle(threadLocal.shuffledServerList) fts3Server = threadLocal.shuffledServerList[_attempt] if _attempt == self._maxAttempts - 1: random.shuffle(threadLocal.shuffledServerList) return fts3Server def _getFTSServerStatus(self, ftsServer): """ Fetch the status of the FTS server from RSS """ res = self._resourceStatus.getElementStatus(ftsServer, 'FTS') if not res['OK']: return res result = res['Value'] if ftsServer not in result: return S_ERROR("No FTS Server %s known to RSS" % ftsServer) if result[ftsServer]['all'] == 'Active': return S_OK(True) return S_OK(False) def chooseFTS3Server(self): """ Choose the appropriate FTS3 server depending on the policy """ fts3Server = None attempt = 0 while not fts3Server and attempt < self._maxAttempts: fts3Server = self._policyMethod(attempt) res = self._getFTSServerStatus(fts3Server) if not res['OK']: self.log.warn("Error getting the RSS status for %s: %s" % (fts3Server, res)) fts3Server = None attempt += 1 continue ftsServerStatus = res['Value'] if not ftsServerStatus: self.log.warn('FTS server %s is not in good shape. Choose another one' % fts3Server) fts3Server = None attempt += 1 if fts3Server: return S_OK(self._serverDict[fts3Server]) return S_ERROR("Could not find an FTS3 server (max attempt reached)")
class FTS3Placement( FTSAbstractPlacement ): """ This class manages all the FTS strategies, routes and what not """ __serverPolicy = "Random" __nextServerID = 0 __serverList = None __maxAttempts = 0 def __init__( self, csPath = None, ftsHistoryViews = None ): """ Call the init of the parent, and initialize the list of FTS3 servers """ self.log = gLogger.getSubLogger( "FTS3Placement" ) super( FTS3Placement, self ).__init__( csPath = csPath, ftsHistoryViews = ftsHistoryViews ) srvList = getFTS3Servers() if not srvList['OK']: self.log.error( srvList['Message'] ) self.__serverList = srvList.get( 'Value', [] ) self.maxAttempts = len( self.__serverList ) self.rssClient = ResourceStatus() def getReplicationTree( self, sourceSEs, targetSEs, size, strategy = None ): """ For multiple source to multiple destination, find the optimal replication strategy. :param sourceSEs : list of source SE :param targetSEs : list of destination SE :param size : size of the File :param strategy : which strategy to use :returns S_OK(dict) < route name : { dict with key Ancestor, SourceSE, TargetSEtargetSE, Strategy } > For the time being, we are waiting for FTS3 to provide advisory mechanisms. So we just use simple techniques """ # We will use a single random source sourceSE = random.choice( sourceSEs ) tree = {} for targetSE in targetSEs: tree["%s#%s" % ( sourceSE, targetSE )] = { "Ancestor" : False, "SourceSE" : sourceSE, "TargetSE" : targetSE, "Strategy" : "FTS3Simple" } return S_OK( tree ) def refresh( self, ftsHistoryViews ): """ Refresh, whatever that means... recalculate all what you need, fetches the latest conf and what not. """ return super( FTS3Placement, self ).refresh( ftsHistoryViews = ftsHistoryViews ) def __failoverServerPolicy(self, attempt = 0): """ Returns always the server at a given position (normally the first one) :param attempt: position of the server in the list """ if attempt >= len( self.__serverList ): raise Exception( "FTS3Placement.__failoverServerPolicy: attempt to reach non existing server index" ) return self.__serverList[attempt] def __sequenceServerPolicy( self ): """ Every time the this policy is called, return the next server on the list """ fts3server = self.__serverList[self.__nextServerID] self.__nextServerID = ( self.__nextServerID + 1 ) % len( self.__serverList ) return fts3server def __randomServerPolicy(self): """ return a random server from the list """ return random.choice( self.__serverList ) def __chooseFTS3Server( self ): """ Choose the appropriate FTS3 server depending on the policy """ fts3Server = None attempt = 0 # FIXME : need to get real valeu from RSS ftsServerStatus = True while not fts3Server and attempt < self.maxAttempts: if self.__serverPolicy == 'Random': fts3Server = self.__randomServerPolicy() elif self.__serverPolicy == 'Sequence': fts3Server = self.__sequenceServerPolicy() elif self.__serverPolicy == 'Failover': fts3Server = self.__failoverServerPolicy( attempt = attempt ) else: self.log.error( 'Unknown server policy %s. Using Random instead' % self.__serverPolicy ) fts3Server = self.__randomServerPolicy() if not ftsServerStatus: self.log.warn( 'FTS server %s is not in good shape. Choose another one' % fts3Server ) fts3Server = None attempt += 1 # FIXME : I need to get the FTS server status from RSS # ftsStatusFromRss = rss.ftsStatusOrSomethingLikeThat if fts3Server: return S_OK( fts3Server ) return S_ERROR ( "Could not find an FTS3 server (max attempt reached)" ) def findRoute( self, sourceSE, targetSE ): """ Find the appropriate route from point A to B :param sourceSE : source SE :param targetSE : destination SE :returns: S_OK(FTSRoute) """ fts3server = self.__chooseFTS3Server() if not fts3server['OK']: return fts3server fts3server = fts3server['Value'] route = FTSRoute( sourceSE, targetSE, fts3server ) return S_OK( route ) def isRouteValid( self, route ): """ FIXME: until RSS is ready, I check manually the status In FTS3, all routes are valid a priori. If a route was not valid for some reason, then FTS would know it thanks to the blacklist sent by RSS, and would deal with it itself. :param route : FTSRoute :returns: S_OK or S_ERROR(reason) """ rAccess = self.rssClient.getElementStatus( route.sourceSE, "StorageElement", "ReadAccess" ) self.log.debug( "se read %s %s" % ( route.sourceSE, rAccess ) ) if not rAccess["OK"]: self.log.error( rAccess["Message"] ) return rAccess if rAccess["Value"][route.sourceSE]["ReadAccess"] not in ( "Active", "Degraded" ): return S_ERROR( "Source SE is not readable" ) wAccess = self.rssClient.getElementStatus( route.targetSE, "StorageElement", "WriteAccess" ) self.log.debug( "se write %s %s" % ( route.targetSE, wAccess ) ) if not wAccess["OK"]: self.log.error( wAccess["Message"] ) return wAccess if wAccess["Value"][route.targetSE]["WriteAccess"] not in ( "Active", "Degraded" ): return S_ERROR( "Target SE is not writable" ) return S_OK()
def main(): global fullMatch global sites Script.registerSwitch("F", "full-match", "Check all the matching criteria", setFullMatch) Script.registerSwitch( "S:", "site=", "Check matching for these sites (comma separated list)", setSites) Script.parseCommandLine(ignoreErrors=True) args = Script.getPositionalArgs() if len(args) == 0: gLogger.error("Error: No job description provided") Script.showHelp(exitCode=1) from DIRAC.Core.Security.ProxyInfo import getVOfromProxyGroup from DIRAC.ConfigurationSystem.Client.Helpers import Resources from DIRAC.Core.Utilities.PrettyPrint import printTable from DIRAC.ResourceStatusSystem.Client.ResourceStatus import ResourceStatus from DIRAC.ResourceStatusSystem.Client.SiteStatus import SiteStatus from DIRAC.WorkloadManagementSystem.Utilities.QueueUtilities import getQueuesResolved, matchQueue with open(args[0]) as f: jdl = f.read() # Get the current VO result = getVOfromProxyGroup() if not result['OK']: gLogger.error('No proxy found, please login') DIRACExit(-1) voName = result['Value'] resultQueues = Resources.getQueues(siteList=sites, community=voName) if not resultQueues['OK']: gLogger.error('Failed to get CE information') DIRACExit(-1) siteDict = resultQueues['Value'] result = getQueuesResolved(siteDict) if not resultQueues['OK']: gLogger.error('Failed to get CE information') DIRACExit(-1) queueDict = result['Value'] # get list of usable sites within this cycle resultMask = SiteStatus().getUsableSites() if not resultMask['OK']: gLogger.error('Failed to get Site mask information') DIRACExit(-1) siteMaskList = resultMask.get('Value', []) rssClient = ResourceStatus() fields = ('Site', 'CE', 'Queue', 'Status', 'Match', 'Reason') records = [] for queue, queueInfo in queueDict.items(): site = queueInfo['Site'] ce = queueInfo['CEName'] siteStatus = "Active" if site in siteMaskList else "InActive" ceStatus = siteStatus if rssClient.rssFlag: result = rssClient.getElementStatus(ce, "ComputingElement") if result['OK']: ceStatus = result['Value'][ce]['all'] result = matchQueue(jdl, queueInfo, fullMatch=fullMatch) if not result['OK']: gLogger.error('Failed in getting match data', result['Message']) DIRACExit(-1) status = "Active" if siteStatus == "Active" and ceStatus == "Active" else "Inactive" if result['Value']['Match']: records.append((site, ce, queueInfo['Queue'], status, 'Yes', '')) else: records.append((site, ce, queueInfo['Queue'], status, 'No', result['Value']['Reason'])) gLogger.notice( printTable(fields, records, sortField='Site', columnSeparator=' ', printOut=False))
def main(): global vo global noVOFlag global allVOsFlag Script.registerSwitch("V:", "vo=", "Virtual Organization", setVO) Script.registerSwitch("a", "all", "All Virtual Organizations flag", setAllVO) Script.registerSwitch("n", "noVO", "No Virtual Organizations assigned flag", setNoVO) Script.parseCommandLine() from DIRAC import gConfig, gLogger from DIRAC.ResourceStatusSystem.Client.ResourceStatus import ResourceStatus from DIRAC.Core.Utilities.PrettyPrint import printTable from DIRAC.Core.Security.ProxyInfo import getVOfromProxyGroup storageCFGBase = "/Resources/StorageElements" res = gConfig.getSections(storageCFGBase, True) if not res['OK']: gLogger.error('Failed to get storage element info') gLogger.error(res['Message']) DIRACexit(1) gLogger.info("%s %s %s" % ('Storage Element'.ljust(25), 'Read Status'.rjust(15), 'Write Status'.rjust(15))) seList = sorted(res['Value']) resourceStatus = ResourceStatus() res = resourceStatus.getElementStatus(seList, "StorageElement") if not res['OK']: gLogger.error("Failed to get StorageElement status for %s" % str(seList)) DIRACexit(1) fields = ['SE', 'ReadAccess', 'WriteAccess', 'RemoveAccess', 'CheckAccess'] records = [] if vo is None and not allVOsFlag: result = getVOfromProxyGroup() if not result['OK']: gLogger.error('Failed to determine the user VO') DIRACexit(1) vo = result['Value'] for se, statusDict in res['Value'].items(): # Check if the SE is allowed for the user VO if not allVOsFlag: voList = gConfig.getValue('/Resources/StorageElements/%s/VO' % se, []) if noVOFlag and voList: continue if voList and vo not in voList: continue record = [se] for status in fields[1:]: value = statusDict.get(status, 'Unknown') record.append(value) records.append(record) printTable(fields, records, numbering=False, sortField='SE') DIRACexit(0)
def main(): read = False write = False check = False remove = False site = '' mute = False Script.registerSwitch("r", "AllowRead", " Allow only reading from the storage element") Script.registerSwitch("w", "AllowWrite", " Allow only writing to the storage element") Script.registerSwitch( "k", "AllowCheck", " Allow only check access to the storage element") Script.registerSwitch( "v", "AllowRemove", " Allow only remove access to the storage element") Script.registerSwitch("a", "All", " Allow all access to the storage element") Script.registerSwitch("m", "Mute", " Do not send email") Script.registerSwitch("S:", "Site=", " Allow all SEs associated to site") Script.parseCommandLine(ignoreErrors=True) ses = Script.getPositionalArgs() for switch in Script.getUnprocessedSwitches(): if switch[0].lower() in ("r", "allowread"): read = True if switch[0].lower() in ("w", "allowwrite"): write = True if switch[0].lower() in ("k", "allowcheck"): check = True if switch[0].lower() in ("v", "allowremove"): remove = True if switch[0].lower() in ("a", "all"): read = True write = True check = True remove = True if switch[0].lower() in ("m", "mute"): mute = True if switch[0].lower() in ("s", "site"): site = switch[1] # imports from DIRAC import gConfig, gLogger from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getSites from DIRAC.Core.Security.ProxyInfo import getProxyInfo from DIRAC.DataManagementSystem.Utilities.DMSHelpers import resolveSEGroup from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin from DIRAC.ResourceStatusSystem.Client.ResourceStatus import ResourceStatus if not (read or write or check or remove): # No switch was specified, means we need all of them gLogger.notice( "No option given, all accesses will be allowed if they were not") read = True write = True check = True remove = True ses = resolveSEGroup(ses) diracAdmin = DiracAdmin() errorList = [] setup = gConfig.getValue('/DIRAC/Setup', '') if not setup: print('ERROR: Could not contact Configuration Service') DIRAC.exit(2) res = getProxyInfo() if not res['OK']: gLogger.error('Failed to get proxy information', res['Message']) DIRAC.exit(2) userName = res['Value'].get('username') if not userName: gLogger.error('Failed to get username for proxy') DIRAC.exit(2) if site: res = getSites() if not res['OK']: gLogger.error(res['Message']) DIRAC.exit(-1) if site not in res['Value']: gLogger.error('The provided site (%s) is not known.' % site) DIRAC.exit(-1) ses.extend(res['Value']['SE'].replace(' ', '').split(',')) if not ses: gLogger.error('There were no SEs provided') DIRAC.exit() STATUS_TYPES = ["ReadAccess", "WriteAccess", "CheckAccess", "RemoveAccess"] ALLOWED_STATUSES = ["Unknown", "InActive", "Banned", "Probing", "Degraded"] statusAllowedDict = {} for statusType in STATUS_TYPES: statusAllowedDict[statusType] = [] statusFlagDict = {} statusFlagDict['ReadAccess'] = read statusFlagDict['WriteAccess'] = write statusFlagDict['CheckAccess'] = check statusFlagDict['RemoveAccess'] = remove resourceStatus = ResourceStatus() res = resourceStatus.getElementStatus(ses, "StorageElement") if not res['OK']: gLogger.error('Storage Element %s does not exist' % ses) DIRAC.exit(-1) reason = 'Forced with dirac-admin-allow-se by %s' % userName for se, seOptions in res['Value'].items(): # InActive is used on the CS model, Banned is the equivalent in RSS for statusType in STATUS_TYPES: if statusFlagDict[statusType]: if seOptions.get(statusType) == "Active": gLogger.notice('%s status of %s is already Active' % (statusType, se)) continue if statusType in seOptions: if not seOptions[statusType] in ALLOWED_STATUSES: gLogger.notice( '%s option for %s is %s, instead of %s' % (statusType, se, seOptions['ReadAccess'], ALLOWED_STATUSES)) gLogger.notice('Try specifying the command switches') else: resR = resourceStatus.setElementStatus( se, "StorageElement", statusType, 'Active', reason, userName) if not resR['OK']: gLogger.fatal( "Failed to update %s %s to Active, exit -" % (se, statusType), resR['Message']) DIRAC.exit(-1) else: gLogger.notice( "Successfully updated %s %s to Active" % (se, statusType)) statusAllowedDict[statusType].append(se) totalAllowed = 0 totalAllowedSEs = [] for statusType in STATUS_TYPES: totalAllowed += len(statusAllowedDict[statusType]) totalAllowedSEs += statusAllowedDict[statusType] totalAllowedSEs = list(set(totalAllowedSEs)) if not totalAllowed: gLogger.info("No storage elements were allowed") DIRAC.exit(-1) if mute: gLogger.notice('Email is muted by script switch') DIRAC.exit(0) subject = '%s storage elements allowed for use' % len(totalAllowedSEs) addressPath = 'EMail/Production' address = Operations().getValue(addressPath, '') body = '' if read: body = "%s\n\nThe following storage elements were allowed for reading:" % body for se in statusAllowedDict['ReadAccess']: body = "%s\n%s" % (body, se) if write: body = "%s\n\nThe following storage elements were allowed for writing:" % body for se in statusAllowedDict['WriteAccess']: body = "%s\n%s" % (body, se) if check: body = "%s\n\nThe following storage elements were allowed for checking:" % body for se in statusAllowedDict['CheckAccess']: body = "%s\n%s" % (body, se) if remove: body = "%s\n\nThe following storage elements were allowed for removing:" % body for se in statusAllowedDict['RemoveAccess']: body = "%s\n%s" % (body, se) if not address: gLogger.notice( "'%s' not defined in Operations, can not send Mail\n" % addressPath, body) DIRAC.exit(0) res = diracAdmin.sendMail(address, subject, body) gLogger.notice('Notifying %s' % address) if res['OK']: gLogger.notice(res['Value']) else: gLogger.notice(res['Message']) DIRAC.exit(0)
gLogger.error('The provided site (%s) is not known.' % site) DIRAC.exit(-1) ses.extend(res['Value']['SE'].replace(' ', '').split(',')) if not ses: gLogger.error('There were no SEs provided') DIRAC.exit(-1) readBanned = [] writeBanned = [] checkBanned = [] removeBanned = [] resourceStatus = ResourceStatus() res = resourceStatus.getElementStatus(ses, "StorageElement") if not res['OK']: gLogger.error("Storage Element %s does not exist" % ses) DIRAC.exit(-1) reason = 'Forced with dirac-admin-ban-se by %s' % userName for se, seOptions in res['Value'].items(): resW = resC = resR = {'OK': False} # Eventually, we will get rid of the notion of InActive, as we always write Banned. if read and 'ReadAccess' in seOptions: if not seOptions['ReadAccess'] in ['Active', 'Degraded', 'Probing']: gLogger.notice('Read option for %s is %s, instead of %s' %
def main(): global fullMatch global sites Script.registerSwitch("F", "full-match", "Check all the matching criteria", setFullMatch) Script.registerSwitch( "S:", "site=", "Check matching for these sites (comma separated list)", setSites) Script.registerArgument("job_JDL: file with job JDL description") _, args = Script.parseCommandLine(ignoreErrors=True) from DIRAC.Core.Security.ProxyInfo import getVOfromProxyGroup from DIRAC.ConfigurationSystem.Client.Helpers import Resources from DIRAC.Core.Utilities.PrettyPrint import printTable from DIRAC.ResourceStatusSystem.Client.ResourceStatus import ResourceStatus from DIRAC.ResourceStatusSystem.Client.SiteStatus import SiteStatus from DIRAC.WorkloadManagementSystem.Utilities.QueueUtilities import getQueuesResolved, matchQueue with open(args[0]) as f: jdl = f.read() # Get the current VO result = getVOfromProxyGroup() if not result["OK"]: gLogger.error("No proxy found, please login") DIRACExit(-1) voName = result["Value"] resultQueues = Resources.getQueues(siteList=sites, community=voName) if not resultQueues["OK"]: gLogger.error("Failed to get CE information") DIRACExit(-1) siteDict = resultQueues["Value"] result = getQueuesResolved(siteDict, {}, checkPlatform=True) if not resultQueues["OK"]: gLogger.error("Failed to get CE information") DIRACExit(-1) queueDict = result["Value"] # get list of usable sites within this cycle resultMask = SiteStatus().getUsableSites() if not resultMask["OK"]: gLogger.error("Failed to get Site mask information") DIRACExit(-1) siteMaskList = resultMask.get("Value", []) rssClient = ResourceStatus() fields = ("Site", "CE", "Queue", "Status", "Match", "Reason") records = [] for queue, queueInfo in queueDict.items(): site = queueInfo["Site"] ce = queueInfo["CEName"] siteStatus = "Active" if site in siteMaskList else "InActive" ceStatus = siteStatus if rssClient.rssFlag: result = rssClient.getElementStatus(ce, "ComputingElement") if result["OK"]: ceStatus = result["Value"][ce]["all"] result = matchQueue(jdl, queueInfo["ParametersDict"], fullMatch=fullMatch) if not result["OK"]: gLogger.error("Failed in getting match data", result["Message"]) DIRACExit(-1) status = "Active" if siteStatus == "Active" and ceStatus == "Active" else "Inactive" if result["Value"]["Match"]: records.append( (site, ce, queueInfo["QueueName"], status, "Yes", "")) else: records.append((site, ce, queueInfo["QueueName"], status, "No", result["Value"]["Reason"])) gLogger.notice( printTable(fields, records, sortField="Site", columnSeparator=" ", printOut=False))