def getSESiteMapping( gridName = '', withSiteLocalSEMapping = False ): """ Returns a dictionary of all SEs and their associated site(s), e.g. {'CERN-RAW':'LCG.CERN.ch','CERN-RDST':'LCG.CERN.ch',...]} Although normally one site exists for a given SE, it is possible over all Grid types to have multiple entries. If gridName is specified, result is restricted to that Grid type. Assumes CS structure of: /Resources/Sites/<GRIDNAME>/<SITENAME> """ dmsHelper = DMSHelpers() storageElements = dmsHelper.getStorageElements() return S_OK( dict( ( se, getSitesForSE( se, gridName = gridName, withSiteLocalSEMapping = withSiteLocalSEMapping ).get( 'Value', [] ) ) \ for se in storageElements ) )
def getStorageElements(vo): """ Get configuration of storage elements :param vo: VO name that an SE supports :return: S_OK/S_ERROR, Value dictionary with key SE and value protocol list """ log = gLogger.getLocalSubLogger("RucioSynchronizer/%s" % vo) seProtocols = {} dms = DMSHelpers(vo=vo) for seName in dms.getStorageElements(): se = StorageElement(seName) if not se.valid: log.warn("Storage element is not valid.", seName) continue if vo not in se.options.get("VO", []): log.debug("SE is valid, but it doesn't support the VO. Skipped.", "[SE: %s, VO: %s]" % (seName, vo)) continue log.debug(" Processing a valid SE for VO: ", "[SE:%s, VO:%s]" % (seName, vo)) log.debug("Available SE options ", se.options) seProtocols[seName] = [] all_protocols = [] read_protocols = {} protocols = se.options.get("AccessProtocols") log.debug("Global AccessProtocols:", "[VO: %s, protocols: %s]" % (vo, protocols)) if not protocols: protocols = dms.getAccessProtocols() if not protocols: log.warn( " No global or SE specific access protocols defined for SE ", seName) continue log.debug("AccessProtocols:", "[VO: %s, protocols:%s]" % (vo, protocols)) idx = 1 for prot in protocols: read_protocols[prot] = idx idx += 1 if prot not in all_protocols: all_protocols.append(prot) write_protocols = {} protocols = se.options.get("WriteProtocols") if not protocols: if not protocols: protocols = dms.getWriteProtocols() if not protocols: log.warn( " No global or SE specific write protocols defined for SE ", seName) continue idx = 1 for prot in protocols: write_protocols[prot] = idx idx += 1 if prot not in all_protocols: all_protocols.append(prot) mapping = { "Protocol": "scheme", "Host": "hostname", "Port": "port", "Path": "prefix" } for protocol in all_protocols: space_token = None params = { "hostname": None, "scheme": None, "port": None, "prefix": None, "impl": "rucio.rse.protocols.gfal.Default", "domains": { "lan": { "read": 0, "write": 0, "delete": 0 }, "wan": { "read": 0, "write": 0, "delete": 0, "third_party_copy": 0 }, }, } res = se.getStorageParameters(protocol=protocol) if res["OK"]: values = res["Value"] for key in [ "Protocol", "Host", "Access", "Path", "Port", "WSUrl", "SpaceToken", "WSUrl", "PluginName" ]: value = values.get(key) if key in mapping: params[mapping[key]] = value else: if key == "SpaceToken": space_token = value if params["scheme"] == "srm" and key == "WSUrl": params["extended_attributes"] = { "web_service_path": "%s" % value, "space_token": space_token, } if key == "Protocol": params["domains"]["lan"]["read"] = read_protocols.get( value, 0) params["domains"]["wan"]["read"] = read_protocols.get( value, 0) params["domains"]["lan"][ "write"] = write_protocols.get(value, 0) params["domains"]["wan"][ "write"] = write_protocols.get(value, 0) params["domains"]["lan"][ "delete"] = write_protocols.get(value, 0) params["domains"]["wan"][ "delete"] = write_protocols.get(value, 0) params["domains"]["wan"][ "third_party_copy"] = write_protocols.get( value, 0) seProtocols[seName].append(params) log.debug("Accepted Dirac SEs: ", seProtocols) return S_OK(seProtocols)