def getRawSchema(): """ Load the schema from the CS """ base = "%s/Schema" % (BASECS) schema = [] explore = [("", schema)] while len(explore): parentName, parentData = explore.pop(0) fullName = "%s/%s" % (base, parentName) result = gConfig.getSections(fullName) if not result['OK']: continue sectionsList = result['Value'] for sName in sectionsList: sData = [] parentData.append(("%s/%s" % (parentName, sName), sData)) explore.append((sName, sData)) result = gConfig.getOptions(fullName) if not result['OK']: continue optionsList = result['Value'] for opName in optionsList: opVal = gConfig.getValue("%s/%s" % (fullName, opName)) if opVal.find("link|") == 0: parentData.append(("link", opName, opVal[5:])) else: parentData.append(("app", opName, opVal)) return schema
def _getConfigStorageName(self, storageName): """ This gets the name of the storage the configuration service. If the storage is an alias for another the resolution is performed. 'storageName' is the storage section to check in the CS """ configPath = "%s/%s" % (self.rootConfigPath, storageName) res = gConfig.getOptions(configPath) if not res["OK"]: errStr = "StorageFactory._getConfigStorageName: Failed to get storage options" gLogger.error(errStr, res["Message"]) return S_ERROR(errStr) if not res["Value"]: errStr = "StorageFactory._getConfigStorageName: Supplied storage doesn't exist." gLogger.error(errStr, configPath) return S_ERROR(errStr) if "Alias" in res["Value"]: configPath = "%s/%s/Alias" % (self.rootConfigPath, storageName) aliasName = gConfig.getValue(configPath) result = self._getConfigStorageName(aliasName) if not result["OK"]: errStr = "StorageFactory._getConfigStorageName: Supplied storage doesn't exist." gLogger.error(errStr, configPath) return S_ERROR(errStr) resolvedName = result["Value"] else: resolvedName = storageName return S_OK(resolvedName)
def _getConfigStorageOptions( self, storageName, derivedStorageName = None ): """ Get the options associated to the StorageElement as defined in the CS """ optionsDict = {} # We first get the options of the baseSE, and then overwrite with the derivedSE for seName in ( storageName, derivedStorageName ) if derivedStorageName else ( storageName, ): storageConfigPath = cfgPath( self.rootConfigPath, seName ) res = gConfig.getOptions( storageConfigPath ) if not res['OK']: errStr = "StorageFactory._getStorageOptions: Failed to get storage options." gLogger.error( errStr, "%s: %s" % ( seName, res['Message'] ) ) return S_ERROR( errStr ) for option in set( res['Value'] ) - set( ( 'ReadAccess', 'WriteAccess', 'CheckAccess', 'RemoveAccess' ) ): optionConfigPath = cfgPath( storageConfigPath, option ) default = [] if option in [ 'VO' ] else '' optionsDict[option] = gConfig.getValue( optionConfigPath, default ) # The status is that of the derived SE only seName = derivedStorageName if derivedStorageName else storageName res = self.resourceStatus.getStorageElementStatus( seName ) if not res[ 'OK' ]: errStr = "StorageFactory._getStorageOptions: Failed to get storage status" gLogger.error( errStr, "%s: %s" % ( seName, res['Message'] ) ) return S_ERROR( errStr ) # For safety, we did not add the ${statusType}Access keys # this requires modifications in the StorageElement class # We add the dictionary with the statusTypes and values # { 'statusType1' : 'status1', 'statusType2' : 'status2' ... } optionsDict.update( res[ 'Value' ][ seName ] ) return S_OK( optionsDict )
def _getBaseStorageName( self, storageName ): """ This gets the name of the base SE in case the SE is defined from a base SE. 'storageName' is the storage section to check in the CS """ configPath = '%s/%s' % ( self.rootConfigPath, storageName ) res = gConfig.getOptions( configPath ) if not res['OK']: errStr = "StorageFactory._getConfigStorageName: Failed to get storage options" gLogger.error( errStr, res['Message'] ) return S_ERROR( errStr ) if not res['Value']: errStr = "StorageFactory._getConfigStorageName: Supplied storage doesn't exist." gLogger.error( errStr, configPath ) return S_ERROR( errStr ) if 'BaseSE' in res['Value']: configPath = '%s/%s/BaseSE' % ( self.rootConfigPath, storageName ) baseName = gConfig.getValue( configPath ) # Just in case the base is an alias ;-) result = self._getConfigStorageName( baseName ) if not result['OK']: return result resolvedName = result['Value'] else: resolvedName = storageName return S_OK( resolvedName )
def _getCatalogConfigDetails(self, catalogName): # First obtain the options that are available catalogConfigPath = "%s/%s" % (self.rootConfigPath, catalogName) res = gConfig.getOptions(catalogConfigPath) if not res["OK"]: errStr = "FileCatalog._getCatalogConfigDetails: Failed to get catalog options." gLogger.error(errStr, catalogName) return S_ERROR(errStr) catalogConfig = {} for option in res["Value"]: configPath = "%s/%s" % (catalogConfigPath, option) optionValue = gConfig.getValue(configPath) catalogConfig[option] = optionValue # The 'Status' option should be defined (default = 'Active') if not catalogConfig.has_key("Status"): warnStr = "FileCatalog._getCatalogConfigDetails: 'Status' option not defined." gLogger.warn(warnStr, catalogName) catalogConfig["Status"] = "Active" # The 'AccessType' option must be defined if not catalogConfig.has_key("AccessType"): errStr = "FileCatalog._getCatalogConfigDetails: Required option 'AccessType' not defined." gLogger.error(errStr, catalogName) return S_ERROR(errStr) # Anything other than 'True' in the 'Master' option means it is not if not catalogConfig.has_key("Master"): catalogConfig["Master"] = False elif catalogConfig["Master"] == "True": catalogConfig["Master"] = True else: catalogConfig["Master"] = False return S_OK(catalogConfig)
def _getConfigStorageName(self, storageName, referenceType): """ This gets the name of the storage the configuration service. If the storage is a reference to another SE the resolution is performed. 'storageName' is the storage section to check in the CS """ configPath = '%s/%s' % (self.rootConfigPath, storageName) res = gConfig.getOptions(configPath) if not res['OK']: errStr = "StorageFactory._getConfigStorageName: Failed to get storage options" gLogger.error(errStr, res['Message']) return S_ERROR(errStr) if not res['Value']: errStr = "StorageFactory._getConfigStorageName: Supplied storage doesn't exist." gLogger.error(errStr, configPath) return S_ERROR(errStr) if referenceType in res['Value']: configPath = cfgPath(self.rootConfigPath, storageName, referenceType) referenceName = gConfig.getValue(configPath) result = self._getConfigStorageName(referenceName, 'Alias') if not result['OK']: return result resolvedName = result['Value'] else: resolvedName = storageName return S_OK(resolvedName)
def _getCatalogConfigDetails(self, catalogName): # First obtain the options that are available catalogConfigPath = '%s/%s' % (self.rootConfigPath, catalogName) res = gConfig.getOptions(catalogConfigPath) if not res['OK']: errStr = "FileCatalog._getCatalogConfigDetails: Failed to get catalog options." gLogger.error(errStr, catalogName) return S_ERROR(errStr) catalogConfig = {} for option in res['Value']: configPath = '%s/%s' % (catalogConfigPath, option) optionValue = gConfig.getValue(configPath) catalogConfig[option] = optionValue # The 'Status' option should be defined (default = 'Active') if 'Status' not in catalogConfig: warnStr = "FileCatalog._getCatalogConfigDetails: 'Status' option not defined." gLogger.warn(warnStr, catalogName) catalogConfig['Status'] = 'Active' # The 'AccessType' option must be defined if 'AccessType' not in catalogConfig: errStr = "FileCatalog._getCatalogConfigDetails: Required option 'AccessType' not defined." gLogger.error(errStr, catalogName) return S_ERROR(errStr) # Anything other than 'True' in the 'Master' option means it is not catalogConfig['Master'] = (catalogConfig.setdefault('Master', False) == 'True') return S_OK(catalogConfig)
def __getHistory(self, state): """ Just get the history based on state Return resulting list "state" can be either "Save" or "Load" """ gLogger.info("Running getHistory( %s )" % state) msg = "getHistory() for %s@%s" % (getUsername(), getSelectedGroup()) opt = "/Website/" + USER_PROFILE_NAME + "/ShowHistory" history_length = gConfig.getOptions(opt, 5) upc = UserProfileClient("Default", getRPCClient) group = str(getSelectedGroup()) profile_name = USER_PROFILE_NAME + ".History." + state + "." + group result = upc.retrieveVar(profile_name) gLogger.info(result) if not result["OK"]: if result["Message"].find("No data") < 0: gLogger.error("Result %s: %s" % (msg, result["Message"])) return S_ERROR(result["Message"]) history = list() else: history = result["Value"] if not isinstance(history, list): err = "List expected at: %s" % profile_name gLogger.error("Result %s: %s" % (msg, err)) return S_ERROR(err) if (len(history) > history_length): history = result[history_length] gLogger.info("Result %s: %s" % (msg, history)) return S_OK(history)
def __generateSchema( self, base, path ): """ Generate a menu schema based on the user credentials """ #Calculate schema schema = [] fullName = "%s/%s" % ( base, path ) result = gConfig.getSections( fullName ) if not result[ 'OK' ]: return schema sectionsList = result[ 'Value' ] for sName in sectionsList: subSchema = self.__generateSchema( base, "%s/%s" % ( path, sName ) ) if subSchema: schema.append( ( sName, subSchema ) ) result = gConfig.getOptions( fullName ) if not result[ 'OK' ]: return schema optionsList = result[ 'Value' ] for opName in optionsList: opVal = gConfig.getValue( "%s/%s" % ( fullName, opName ) ) if opVal.find( "link|" ) == 0: schema.append( ( "link", opName, opVal[5:] ) ) continue if self.__isGroupAuthApp( opVal ): schema.append( ( "app", opName, opVal ) ) return schema
def _getCatalogConfigDetails( self, catalogName ): # First obtain the options that are available catalogConfigPath = '%s/%s' % ( self.rootConfigPath, catalogName ) res = gConfig.getOptions( catalogConfigPath ) if not res['OK']: errStr = "FileCatalog._getCatalogConfigDetails: Failed to get catalog options." gLogger.error( errStr, catalogName ) return S_ERROR( errStr ) catalogConfig = {} for option in res['Value']: configPath = '%s/%s' % ( catalogConfigPath, option ) optionValue = gConfig.getValue( configPath ) catalogConfig[option] = optionValue # The 'Status' option should be defined (default = 'Active') if 'Status' not in catalogConfig: warnStr = "FileCatalog._getCatalogConfigDetails: 'Status' option not defined." gLogger.warn( warnStr, catalogName ) catalogConfig['Status'] = 'Active' # The 'AccessType' option must be defined if 'AccessType' not in catalogConfig: errStr = "FileCatalog._getCatalogConfigDetails: Required option 'AccessType' not defined." gLogger.error( errStr, catalogName ) return S_ERROR( errStr ) # Anything other than 'True' in the 'Master' option means it is not catalogConfig['Master'] = ( catalogConfig.setdefault( 'Master', False ) == 'True' ) return S_OK( catalogConfig )
def __generateSchema(self, base, path): """ Generate a menu schema based on the user credentials """ #Calculate schema schema = [] fullName = "%s/%s" % (base, path) result = gConfig.getSections(fullName) if not result['OK']: return schema sectionsList = result['Value'] for sName in sectionsList: subSchema = self.__generateSchema(base, "%s/%s" % (path, sName)) if subSchema: schema.append((sName, subSchema)) result = gConfig.getOptions(fullName) if not result['OK']: return schema optionsList = result['Value'] for opName in optionsList: opVal = gConfig.getValue("%s/%s" % (fullName, opName)) if opVal.find("link|") == 0: schema.append(("link", opName, opVal[5:])) continue if self.__isGroupAuthApp(opVal): schema.append(("app", opName, opVal)) return schema
def _getConfigStorageName( self, storageName, referenceType ): """ This gets the name of the storage the configuration service. If the storage is a reference to another SE the resolution is performed. 'storageName' is the storage section to check in the CS """ configPath = '%s/%s' % ( self.rootConfigPath, storageName ) res = gConfig.getOptions( configPath ) if not res['OK']: errStr = "StorageFactory._getConfigStorageName: Failed to get storage options" gLogger.error( errStr, res['Message'] ) return S_ERROR( errStr ) if not res['Value']: errStr = "StorageFactory._getConfigStorageName: Supplied storage doesn't exist." gLogger.error( errStr, configPath ) return S_ERROR( errStr ) if referenceType in res['Value']: configPath = cfgPath( self.rootConfigPath, storageName, referenceType ) referenceName = gConfig.getValue( configPath ) result = self._getConfigStorageName( referenceName, 'Alias' ) if not result['OK']: return result resolvedName = result['Value'] else: resolvedName = storageName return S_OK( resolvedName )
def __getHistory( self , state ): """ Just get the history based on state Return resulting list "state" can be either "Save" or "Load" """ gLogger.info( "Running getHistory( %s )" % state ) msg = "getHistory() for %s@%s" % ( getUsername() , getSelectedGroup() ) opt = "/Website/" + USER_PROFILE_NAME + "/ShowHistory" history_length = gConfig.getOptions( opt , 5 ) upc = UserProfileClient( "Default" , getRPCClient ) group = str( getSelectedGroup() ) profile_name = USER_PROFILE_NAME + ".History." + state + "." + group result = upc.retrieveVar( profile_name ) gLogger.info( result ) if not result[ "OK" ]: if result[ "Message" ].find( "No data" ) < 0 : gLogger.error( "Result %s: %s" % ( msg , result[ "Message" ] ) ) return S_ERROR( result[ "Message" ] ) history = list() else: history = result[ "Value" ] if not isinstance( history , list ): err = "List expected at: %s" % profile_name gLogger.error( "Result %s: %s" % ( msg , err ) ) return S_ERROR( err ) if( len( history ) > history_length ): history = result[ history_length ] gLogger.info( "Result %s: %s" % ( msg , history ) ) return S_OK( history )
def _getConfigStorageOptions( self, storageName ): """ Get the options associated to the StorageElement as defined in the CS """ storageConfigPath = cfgPath( self.rootConfigPath, storageName ) res = gConfig.getOptions( storageConfigPath ) if not res['OK']: errStr = "StorageFactory._getStorageOptions: Failed to get storage options." gLogger.error( errStr, "%s: %s" % ( storageName, res['Message'] ) ) return S_ERROR( errStr ) options = res['Value'] optionsDict = {} for option in options: if option in [ 'ReadAccess', 'WriteAccess', 'CheckAccess', 'RemoveAccess']: continue optionConfigPath = cfgPath( storageConfigPath, option ) if option in [ 'VO' ]: optionsDict[option] = gConfig.getValue( optionConfigPath, [] ) else: optionsDict[option] = gConfig.getValue( optionConfigPath, '' ) res = self.resourceStatus.getStorageElementStatus( storageName ) if not res[ 'OK' ]: errStr = "StorageFactory._getStorageOptions: Failed to get storage status" gLogger.error( errStr, "%s: %s" % ( storageName, res['Message'] ) ) return S_ERROR( errStr ) # For safety, we did not add the ${statusType}Access keys # this requires modifications in the StorageElement class # We add the dictionary with the statusTypes and values # { 'statusType1' : 'status1', 'statusType2' : 'status2' ... } optionsDict.update( res[ 'Value' ][ storageName ] ) return S_OK( optionsDict )
def getRawSchema(): """ Load the schema from the CS """ base = "%s/Schema" % ( BASECS ) schema = [] explore = [ ( "", schema ) ] while len( explore ): parentName, parentData = explore.pop( 0 ) fullName = "%s/%s" % ( base, parentName ) result = gConfig.getSections( fullName ) if not result[ 'OK' ]: continue sectionsList = result[ 'Value' ] for sName in sectionsList: sData = [] parentData.append( ( "%s/%s" % ( parentName, sName ), sData ) ) explore.append( ( sName, sData ) ) result = gConfig.getOptions( fullName ) if not result[ 'OK' ]: continue optionsList = result[ 'Value' ] for opName in optionsList: opVal = gConfig.getValue( "%s/%s" % ( fullName, opName ) ) if opVal.find( "link|" ) == 0: parentData.append( ( "link", opName, opVal[5:] ) ) else: parentData.append( ( "app", opName, opVal ) ) return schema
def _getConfigStorageOptions( self, storageName ): """ Get the options associated to the StorageElement as defined in the CS """ storageConfigPath = '%s/%s' % ( self.rootConfigPath, storageName ) res = gConfig.getOptions( storageConfigPath ) if not res['OK']: errStr = "StorageFactory._getStorageOptions: Failed to get storage options." gLogger.error( errStr, "%s: %s" % ( storageName, res['Message'] ) ) return S_ERROR( errStr ) options = res['Value'] optionsDict = {} for option in options: if option in [ 'ReadAccess', 'WriteAccess', 'CheckAccess', 'RemoveAccess']: continue optionConfigPath = '%s/%s' % ( storageConfigPath, option ) optionsDict[option] = gConfig.getValue( optionConfigPath, '' ) res = self.resourceStatus.getStorageElementStatus( storageName ) if not res[ 'OK' ]: errStr = "StorageFactory._getStorageOptions: Failed to get storage status" gLogger.error( errStr, "%s: %s" % ( storageName, res['Message'] ) ) return S_ERROR( errStr ) # For safety, we did not add the ${statusType}Access keys # this requires modifications in the StorageElement class # We add the dictionary with the statusTypes and values # { 'statusType1' : 'status1', 'statusType2' : 'status2' ... } optionsDict.update( res[ 'Value' ][ storageName ] ) return S_OK( optionsDict )
def _getTestsList(): result = {} for test in gConfig.getSections('/Systems/Framework/Production/Agents/SAMLauncherAgent')['Value']: result[test] = {} for option in gConfig.getOptions('/Systems/Framework/Production/Agents/SAMLauncherAgent/'+test)['Value']: result[test][option] = gConfig.getOption('/Systems/Framework/Production/Agents/SAMLauncherAgent/'+test+'/'+option)['Value'] return result
def _getConfigStorageProtocolDetails(self, storageName, protocol): """ Parse the contents of the protocol block """ # First obtain the options that are available protocolConfigPath = '%s/%s/%s' % (self.rootConfigPath, storageName, protocol) res = gConfig.getOptions(protocolConfigPath) if not res['OK']: errStr = "StorageFactory.__getProtocolDetails: Failed to get protocol options." gLogger.error(errStr, "%s: %s" % (storageName, protocol)) return S_ERROR(errStr) options = res['Value'] # We must have certain values internally even if not supplied in CS protocolDict = { 'Access': '', 'Host': '', 'Path': '', 'Port': '', 'Protocol': '', 'ProtocolName': '', 'SpaceToken': '', 'WSUrl': '' } for option in options: configPath = '%s/%s' % (protocolConfigPath, option) optionValue = gConfig.getValue(configPath, '') protocolDict[option] = optionValue # Evaluate the base path taking into account possible VO specific setting if self.vo: result = gConfig.getOptionsDict( cfgPath(protocolConfigPath, 'VOPath')) voPath = '' if result['OK']: voPath = result['Value'].get(self.vo, '') if voPath: protocolDict['Path'] = voPath # Now update the local and remote protocol lists. # A warning will be given if the Access option is not set. if protocolDict['Access'] == 'remote': self.remoteProtocols.append(protocolDict['ProtocolName']) elif protocolDict['Access'] == 'local': self.localProtocols.append(protocolDict['ProtocolName']) else: errStr = "StorageFactory.__getProtocolDetails: The 'Access' option for %s:%s is neither 'local' or 'remote'." % ( storageName, protocol) gLogger.warn(errStr) # The ProtocolName option must be defined if not protocolDict['ProtocolName']: errStr = "StorageFactory.__getProtocolDetails: 'ProtocolName' option is not defined." gLogger.error(errStr, "%s: %s" % (storageName, protocol)) return S_ERROR(errStr) return S_OK(protocolDict)
def _getFTSEndpoints(basePath='Resources/FTSEndpoints/FTS3'): """ Gets all FTS endpoints that are in CS """ result = gConfig.getOptions(basePath) if result['OK']: return result['Value'] return []
def getFTS3(): ''' Gets all storage elements from /Resources/FTSEndpoints ''' _basePath = 'Resources/FTSEndpoints/FTS3' ftsEndpoints = gConfig.getOptions(_basePath) return ftsEndpoints
def getOptions(self, path): """ Mock the getOptions call of gConfig It reads from dict_cs """ if 'StorageElements' not in path: return gConfig.getOptions(path) csSection = self.crawlCS(path) options = [opt for opt, val in csSection.iteritems() if not isinstance(val, dict)] return S_OK(options)
def getFTS(): """ Gets all FTS endpoints """ result = gConfig.getOptions("Resources/FTSEndpoints/FTS3") if result["OK"]: return result return S_OK([])
def getFTS(): ''' Gets all storage elements from /Resources/FTSEndpoints ''' _basePath = 'Resources/FTSEndpoints' ftsEndpoints = gConfig.getOptions( _basePath ) return ftsEndpoints
def getFTS(): """ Gets all FTS endpoints """ result = gConfig.getOptions('Resources/FTSEndpoints/FTS3') if result['OK']: return result return S_OK([])
def _getFTSEndpoints(basePath='Resources/FTSEndpoints/FTS3'): ''' Gets all FTS endpoints that are in CS ''' result = gConfig.getOptions(basePath) if result['OK']: return result['Value'] else: # nothing found return []
def getOptions(self, path): """ Mock the getOptions call of gConfig It reads from dict_cs """ if 'StorageElements' not in path and 'StorageElementBases' not in path: return gConfig.getOptions(path) csSection = self.crawlCS(path) if not csSection: return S_ERROR("Not a valid section") options = [opt for opt, val in csSection.iteritems() if not isinstance(val, dict)] return S_OK(options)
def _getConfigStorageOptions(self, storageName, derivedStorageName=None, seConfigPath=SE_CONFIG_PATH): """ Get the options associated to the StorageElement as defined in the CS :param storageName: is the storage section to check in the CS :param seConfigPath: the path of the storage section. It can be /Resources/StorageElements or StorageElementBases :param derivedStorageName: is the storage section of a derived storage if it inherits from a base :return: options associated to the StorageElement as defined in the CS """ optionsDict = {} # We first get the options of the baseSE, and then overwrite with the derivedSE for seName in (storageName, derivedStorageName) if derivedStorageName else ( storageName, ): storageConfigPath = cfgPath(seConfigPath, seName) res = gConfig.getOptions(storageConfigPath) if not res["OK"]: errStr = "StorageFactory._getStorageOptions: Failed to get storage options." gLogger.error(errStr, "%s: %s" % (seName, res["Message"])) return S_ERROR(errStr) for option in set(res["Value"]) - set( ("ReadAccess", "WriteAccess", "CheckAccess", "RemoveAccess")): optionConfigPath = cfgPath(storageConfigPath, option) default = [] if option in [ "VO", "AccessProtocols", "WriteProtocols" ] else "" optionsDict[option] = gConfig.getValue(optionConfigPath, default) # We update the seConfigPath in order to find option in derivedSE now seConfigPath = SE_CONFIG_PATH # The status is that of the derived SE only seName = derivedStorageName if derivedStorageName else storageName res = self.resourceStatus.getElementStatus(seName, "StorageElement") if not res["OK"]: errStr = "StorageFactory._getStorageOptions: Failed to get storage status" gLogger.error(errStr, "%s: %s" % (seName, res["Message"])) return S_ERROR(errStr) # For safety, we did not add the ${statusType}Access keys # this requires modifications in the StorageElement class # We add the dictionary with the statusTypes and values # { 'statusType1' : 'status1', 'statusType2' : 'status2' ... } optionsDict.update(res["Value"][seName]) return S_OK(optionsDict)
def _getConfigStorageProtocolDetails(self, storageName, protocol): """ Parse the contents of the protocol block """ # First obtain the options that are available protocolConfigPath = '%s/%s/%s' % (self.rootConfigPath, storageName, protocol) res = gConfig.getOptions(protocolConfigPath) if not res['OK']: errStr = "StorageFactory.__getProtocolDetails: Failed to get protocol options." gLogger.error(errStr, "%s: %s" % (storageName, protocol)) return S_ERROR(errStr) options = res['Value'] # We must have certain values internally even if not supplied in CS protocolDict = { 'Access': '', 'Host': '', 'Path': '', 'Port': '', 'Protocol': '', 'ProtocolName': '', 'SpaceToken': '', 'WSUrl': '' } for option in options: configPath = '%s/%s' % (protocolConfigPath, option) optionValue = gConfig.getValue(configPath, '') protocolDict[option] = optionValue # If the user has requested to use Proxy storage if self.proxy: if protocolDict['ProtocolName'] != 'DIP': protocolDict['ProtocolName'] = 'Proxy' # Now update the local and remote protocol lists. # A warning will be given if the Access option is not set. if protocolDict['Access'] == 'remote': self.remoteProtocols.append(protocolDict['ProtocolName']) elif protocolDict['Access'] == 'local': self.localProtocols.append(protocolDict['ProtocolName']) else: errStr = "StorageFactory.__getProtocolDetails: The 'Access' option for %s:%s is neither 'local' or 'remote'." % ( storageName, protocol) gLogger.warn(errStr) # The ProtocolName option must be defined if not protocolDict['ProtocolName']: errStr = "StorageFactory.__getProtocolDetails: 'ProtocolName' option is not defined." gLogger.error(errStr, "%s: %s" % (storageName, protocol)) return S_ERROR(errStr) return S_OK(protocolDict)
def _getConfigStorageProtocolDetails(self, storageName, protocolSection, seConfigPath=SE_CONFIG_PATH, checkAccess=True): """ Parse the contents of the protocol block :param storageName: is the storage section to check in the CS :param protocolSection: name of the protocol section to find information :param seConfigPath: the path of the storage section. It can be /Resources/StorageElements or StorageElementBases :param checkAccess: if not set, don't complain if "Access" is not in the section :return: dictionary of the protocol options """ # First obtain the options that are available protocolConfigPath = cfgPath(seConfigPath, storageName, protocolSection) res = gConfig.getOptions(protocolConfigPath) if not res['OK']: errStr = "StorageFactory.__getProtocolDetails: Failed to get protocol options." gLogger.error(errStr, "%s: %s" % (storageName, protocolSection)) return S_ERROR(errStr) options = res['Value'] # We must have certain values internally even if not supplied in CS protocolDict = {'Access': '', 'Host': '', 'Path': '', 'Port': '', 'Protocol': '', 'SpaceToken': '', 'WSUrl': ''} for option in options: configPath = cfgPath(protocolConfigPath, option) optionValue = gConfig.getValue(configPath, '') protocolDict[option] = optionValue # Evaluate the base path taking into account possible VO specific setting if self.vo: result = gConfig.getOptionsDict(cfgPath(protocolConfigPath, 'VOPath')) voPath = '' if result['OK']: voPath = result['Value'].get(self.vo, '') if voPath: protocolDict['Path'] = voPath # Now update the local and remote protocol lists. # A warning will be given if the Access option is not set and the plugin is not already in remote or local. plugin = protocolDict.get('PluginName', protocolSection) if protocolDict['Access'].lower() == 'remote': self.remotePlugins.append(plugin) elif protocolDict['Access'].lower() == 'local': self.localPlugins.append(plugin) # If it is a derived SE, this is normal, no warning elif checkAccess and protocolSection not in self.protocols: errStr = "StorageFactory.__getProtocolDetails: The 'Access' option \ for %s:%s is neither 'local' or 'remote'." % (storageName, protocolSection) gLogger.warn(errStr) return S_OK(protocolDict)
def getFTS2(): ''' Gets all storage elements from /Resources/FTSEndpoints ''' _basePath = 'Resources/FTSEndpoints/FTS2' ftsEndpoints = gConfig.getOptions( _basePath ) ftsEndpointDefaultLocation = gConfig.getValue( '/Resources/FTSEndpoints/Default/FTSEndpoint', '' ) if ftsEndpoints['OK'] and ftsEndpointDefaultLocation: ftsEndpoints['Value'].append( ftsEndpointDefaultLocation ) return ftsEndpoints
def getFTS3Servers(): """ get FTSServers for sites """ csPath = cfgPath( gBaseResourcesSection, "FTSEndpoints/FTS3" ) # We do it in two times to keep the order ftsServerNames = gConfig.getOptions( csPath ).get( 'Value', [] ) ftsServers = [] for name in ftsServerNames: ftsServers.append( gConfig.getValue( cfgPath( csPath, name ) ) ) return S_OK( ftsServers )
def getOptions(self, path): """Mock the getOptions call of gConfig It reads from dict_cs """ if "StorageElements" not in path and "StorageElementBases" not in path: return gConfig.getOptions(path) csSection = self.crawlCS(path) if not csSection: return S_ERROR("Not a valid section") options = [ opt for opt, val in csSection.items() if not isinstance(val, dict) ] return S_OK(options)
def _getConfigStorageProtocolDetails( self, storageName, protocolSection ): """ Parse the contents of the protocol block """ # First obtain the options that are available protocolConfigPath = cfgPath( self.rootConfigPath, storageName, protocolSection ) res = gConfig.getOptions( protocolConfigPath ) if not res['OK']: errStr = "StorageFactory.__getProtocolDetails: Failed to get protocol options." gLogger.error( errStr, "%s: %s" % ( storageName, protocolSection ) ) return S_ERROR( errStr ) options = res['Value'] # We must have certain values internally even if not supplied in CS protocolDict = {'Access':'', 'Host':'', 'Path':'', 'Port':'', 'Protocol':'', 'PluginName':'', 'SpaceToken':'', 'WSUrl':''} for option in options: configPath = cfgPath( protocolConfigPath, option ) optionValue = gConfig.getValue( configPath, '' ) protocolDict[option] = optionValue # This is a temporary for backward compatibility if "ProtocolName" in protocolDict and not protocolDict['PluginName']: protocolDict['PluginName'] = protocolDict['ProtocolName'] protocolDict.pop( 'ProtocolName', None ) # Evaluate the base path taking into account possible VO specific setting if self.vo: result = gConfig.getOptionsDict( cfgPath( protocolConfigPath, 'VOPath' ) ) voPath = '' if result['OK']: voPath = result['Value'].get( self.vo, '' ) if voPath: protocolDict['Path'] = voPath # Now update the local and remote protocol lists. # A warning will be given if the Access option is not set. if protocolDict['Access'].lower() == 'remote': self.remotePlugins.append( protocolDict['PluginName'] ) elif protocolDict['Access'].lower() == 'local': self.localPlugins.append( protocolDict['PluginName'] ) else: errStr = "StorageFactory.__getProtocolDetails: The 'Access' option for %s:%s is neither 'local' or 'remote'." % ( storageName, protocolSection ) gLogger.warn( errStr ) # The PluginName option must be defined if not protocolDict['PluginName']: errStr = "StorageFactory.__getProtocolDetails: 'PluginName' option is not defined." gLogger.error( errStr, "%s: %s" % ( storageName, protocolSection ) ) return S_ERROR( errStr ) return S_OK( protocolDict )
def getFTS2(): ''' Gets all storage elements from /Resources/FTSEndpoints ''' _basePath = 'Resources/FTSEndpoints/FTS2' ftsEndpoints = gConfig.getOptions(_basePath) ftsEndpointDefaultLocation = gConfig.getValue( '/Resources/FTSEndpoints/Default/FTSEndpoint', '') if ftsEndpoints['OK'] and ftsEndpointDefaultLocation: ftsEndpoints['Value'].append(ftsEndpointDefaultLocation) return ftsEndpoints
def _getConfigStorageOptions( self, storageName ): """ Get the options associated to the StorageElement as defined in the CS """ storageConfigPath = '%s/%s' % ( self.rootConfigPath, storageName ) res = gConfig.getOptions( storageConfigPath ) if not res['OK']: errStr = "StorageFactory._getStorageOptions: Failed to get storage options." gLogger.error( errStr, "%s: %s" % ( storageName, res['Message'] ) ) return S_ERROR( errStr ) options = res['Value'] optionsDict = {} for option in options: optionConfigPath = '%s/%s' % ( storageConfigPath, option ) optionsDict[option] = gConfig.getValue( optionConfigPath, '' ) return S_OK( optionsDict )
def _getConfigStorageProtocolDetails(self, storageName, protocol): """ Parse the contents of the protocol block """ # First obtain the options that are available protocolConfigPath = "%s/%s/%s" % (self.rootConfigPath, storageName, protocol) res = gConfig.getOptions(protocolConfigPath) if not res["OK"]: errStr = "StorageFactory.__getProtocolDetails: Failed to get protocol options." gLogger.error(errStr, "%s: %s" % (storageName, protocol)) return S_ERROR(errStr) options = res["Value"] # We must have certain values internally even if not supplied in CS protocolDict = { "Access": "", "Host": "", "Path": "", "Port": "", "Protocol": "", "ProtocolName": "", "SpaceToken": "", "WSUrl": "", } for option in options: configPath = "%s/%s" % (protocolConfigPath, option) optionValue = gConfig.getValue(configPath, "") protocolDict[option] = optionValue # Now update the local and remote protocol lists. # A warning will be given if the Access option is not set. if protocolDict["Access"] == "remote": self.remoteProtocols.append(protocolDict["ProtocolName"]) elif protocolDict["Access"] == "local": self.localProtocols.append(protocolDict["ProtocolName"]) else: errStr = ( "StorageFactory.__getProtocolDetails: The 'Access' option for %s:%s is neither 'local' or 'remote'." % (storageName, protocol) ) gLogger.warn(errStr) # The ProtocolName option must be defined if not protocolDict["ProtocolName"]: errStr = "StorageFactory.__getProtocolDetails: 'ProtocolName' option is not defined." gLogger.error(errStr, "%s: %s" % (storageName, protocol)) return S_ERROR(errStr) return S_OK(protocolDict)
def syncStorageElementsInit(): ses = gConfig.getSections('/Resources/StorageElements') if not ses['OK']: gLogger.error(ses['Message']) DIRAC.exit(2) statuses = rsc.getValidStatuses()['Value'] for se in ses['Value']: opts = gConfig.getOptions('/Resources/StorageElements/%s' % se)['Value'] statusTypes = ['Read', 'Write', 'Check', 'Remove'] for opt in opts: if not opt.endswith('Access'): continue status = gConfig.getValue('/Resources/StorageElements/%s/%s' % (se, opt)) if status in ['NotAllowed', 'InActive']: status = 'Banned' if not status in statuses: gLogger.error('%s not a valid status for %s - %s' % (status, se, statusType)) continue statusType = opt.replace('Access', '') statusTypes.remove(statusType) rsc.modifyElementStatus('StorageElement', se, statusType, status=status, reason='Init sync') for sType in statusTypes: # If there is nothing on the CS, we set the statusType to DEFAULT_STATUS DEFAULT_STATUS = 'Active' rsc.modifyElementStatus('StorageElement', se, sType, status=DEFAULT_STATUS, reason='Default status')
def getAppDependencies(self): """ Generate the dependency dictionary :return: Dict """ dependency = {} fullName = "%s/%s" % (Conf.BASECS, self.__dependencySection) result = gConfig.getOptions(fullName) if not result['OK']: gLogger.error(result['Message']) return dependency optionsList = result['Value'] for opName in optionsList: opVal = gConfig.getValue("%s/%s" % (fullName, opName)) dependency[opName] = opVal return dependency
def getFTS3Servers(hostOnly=False): """ get list of FTS3 servers that are in CS :param bool hostOnly: flag for stripping down the protocol and ports """ csPath = cfgPath(gBaseResourcesSection, "FTSEndpoints/FTS3") # We do it in two times to keep the order ftsServerNames = gConfig.getOptions(csPath).get('Value', []) ftsServers = [] for name in ftsServerNames: serverPath = gConfig.getValue(cfgPath(csPath, name)) if hostOnly: serverPath = urlparse.urlparse(serverPath).hostname ftsServers.append(serverPath) return S_OK(ftsServers)
def getAppDependencies(self): """ Generate the dependency dictionary :return: Dict """ dependency = {} fullName = "%s/%s" % ( Conf.BASECS, self.__dependencySection) result = gConfig.getOptions( fullName ) if not result[ 'OK' ]: gLogger.error(result['Message']) return dependency optionsList = result[ 'Value' ] for opName in optionsList: opVal = gConfig.getValue( "%s/%s" % ( fullName, opName ) ) dependency[opName] = opVal return dependency
def _getConfigStorageName(self, storageName, referenceType, seConfigPath=SE_CONFIG_PATH): """ This gets the name of the storage the configuration service. If the storage is a reference to another SE the resolution is performed. :param storageName: is the storage section to check in the CS :param referenceType: corresponds to an option inside the storage section :param seConfigPath: the path of the storage section. It can be /Resources/StorageElements or StorageElementBases :return: the name of the storage """ configPath = "%s/%s" % (seConfigPath, storageName) res = gConfig.getOptions(configPath) if not res["OK"]: errStr = "StorageFactory._getConfigStorageName: Failed to get storage options" gLogger.error(errStr, res["Message"]) return S_ERROR(errStr) if not res["Value"]: errStr = "StorageFactory._getConfigStorageName: Supplied storage doesn't exist." gLogger.error(errStr, configPath) return S_ERROR(errStr) if referenceType in res["Value"]: configPath = cfgPath(seConfigPath, storageName, referenceType) referenceName = gConfig.getValue(configPath) # We first look into the BaseStorageElements section. # If not, we look into the StorageElements section # (contrary to BaseSE, it's OK for an Alias to be in the StorageElements section) result = self._getConfigStorageName( referenceName, "Alias", seConfigPath=SE_BASE_CONFIG_PATH) if not result["OK"]: # Since it is not in the StorageElementBases section, check in the StorageElements section result = self._getConfigStorageName( referenceName, "Alias", seConfigPath=SE_CONFIG_PATH) if not result["OK"]: return result resolvedName = result["Value"] else: resolvedName = storageName return S_OK(resolvedName)
def getAppDependencies(self): """ Generate the dependency dictionary :return: Dict """ if self.__params.name != 'WebAppDIRAC': self._loadWebAppCFGFiles(self.__params.name) dependency = {} fullName = "%s/%s" % ("/WebApp", self.__dependencySection) result = gConfig.getOptions(fullName) if not result['OK']: gLogger.error(result['Message']) return dependency optionsList = result['Value'] for opName in optionsList: opVal = gConfig.getValue("%s/%s" % (fullName, opName)) dependency[opName] = opVal return dependency
def _getConfigStorageName(self, storageName, referenceType, seConfigPath=SE_CONFIG_PATH): """ This gets the name of the storage the configuration service. If the storage is a reference to another SE the resolution is performed. :params storageName: is the storage section to check in the CS :params referenceType: corresponds to an option inside the storage section :params seConfigPath: the path of the storage section. It can be /Resources/StorageElements or StorageElementBases :return: the name of the storage """ configPath = '%s/%s' % (seConfigPath, storageName) res = gConfig.getOptions(configPath) if not res['OK']: errStr = "StorageFactory._getConfigStorageName: Failed to get storage options" gLogger.error(errStr, res['Message']) return S_ERROR(errStr) if not res['Value']: errStr = "StorageFactory._getConfigStorageName: Supplied storage doesn't exist." gLogger.error(errStr, configPath) return S_ERROR(errStr) if referenceType in res['Value']: configPath = cfgPath(seConfigPath, storageName, referenceType) referenceName = gConfig.getValue(configPath) result = self._getConfigStorageName( referenceName, 'Alias', seConfigPath=SE_BASE_CONFIG_PATH) if not result['OK']: # This is for the backward compatibility and to invite developer to move their BaseSE in the correct section gLogger.warn("Deprecated configuration, you can ignore the error message above."\ " Please move the baseSE in the correct section: ", SE_BASE_CONFIG_PATH) result = self._getConfigStorageName( referenceName, 'Alias', seConfigPath=SE_CONFIG_PATH) if not result['OK']: return result resolvedName = result['Value'] else: resolvedName = storageName return S_OK(resolvedName)
def __deleteHistory( self , name , state ): """ Deleting item from Load and Save history list Return resulting list "name" is a string "state" can be either "Save" or "Load" """ gLogger.info( "Running deleteHistory( %s )" % name ) msg = "deleteHistory() for %s@%s" % ( getUsername() , getSelectedGroup() ) opt = "/Website/" + USER_PROFILE_NAME + "/ShowHistory" history_length = gConfig.getOptions( opt , 5 ) upc = UserProfileClient( "Default" , getRPCClient ) group = str( getSelectedGroup() ) profile_name = USER_PROFILE_NAME + ".History." + state + "." + group result = upc.retrieveVar( profile_name ) gLogger.info( result ) if not result[ "OK" ]: if result[ "Message" ].find( "No data" ) < 0 : gLogger.error( "Result %s: %s" % ( msg , result[ "Message" ] ) ) return S_ERROR( result[ "Message" ] ) gLogger.info( "Result %s: %s" % ( msg , result[ "Message" ] ) ) return S_OK( list() ) # Nothing to delete, return an empty list else: result = result[ "Value" ] if not isinstance( result , list ): err = "List expected at: %s" % profile_name gLogger.error( "Result %s: %s" % ( msg , err ) ) return S_ERROR( err ) history = list() for i in result: if i.has_key( "name" ) and not i["name"] == name: history.append( i ) if( len( history ) > history_length ): history = result[ history_length ] gLogger.error( "History: %s" % history ) result = upc.storeVar( profile_name , history ) gLogger.info( result ) if not result[ "OK" ]: gLogger.error( "Result %s: %s" % ( msg , result[ "Message" ] ) ) return S_ERROR( result[ "Message" ] ) gLogger.info( "Result %s: %s" % ( msg , history ) ) return S_OK( history )
def __deleteHistory(self, name, state): """ Deleting item from Load and Save history list Return resulting list "name" is a string "state" can be either "Save" or "Load" """ gLogger.info("Running deleteHistory( %s )" % name) msg = "deleteHistory() for %s@%s" % (getUsername(), getSelectedGroup()) opt = "/Website/" + USER_PROFILE_NAME + "/ShowHistory" history_length = gConfig.getOptions(opt, 5) upc = UserProfileClient("Default", getRPCClient) group = str(getSelectedGroup()) profile_name = USER_PROFILE_NAME + ".History." + state + "." + group result = upc.retrieveVar(profile_name) gLogger.info(result) if not result["OK"]: if result["Message"].find("No data") < 0: gLogger.error("Result %s: %s" % (msg, result["Message"])) return S_ERROR(result["Message"]) gLogger.info("Result %s: %s" % (msg, result["Message"])) return S_OK(list()) # Nothing to delete, return an empty list else: result = result["Value"] if not isinstance(result, list): err = "List expected at: %s" % profile_name gLogger.error("Result %s: %s" % (msg, err)) return S_ERROR(err) history = list() for i in result: if i.has_key("name") and not i["name"] == name: history.append(i) if (len(history) > history_length): history = result[history_length] gLogger.error("History: %s" % history) result = upc.storeVar(profile_name, history) gLogger.info(result) if not result["OK"]: gLogger.error("Result %s: %s" % (msg, result["Message"])) return S_ERROR(result["Message"]) gLogger.info("Result %s: %s" % (msg, history)) return S_OK(history)
def _getConfigStorageProtocolDetails( self, storageName, protocol ): """ Parse the contents of the protocol block """ # First obtain the options that are available protocolConfigPath = '%s/%s/%s' % ( self.rootConfigPath, storageName, protocol ) res = gConfig.getOptions( protocolConfigPath ) if not res['OK']: errStr = "StorageFactory.__getProtocolDetails: Failed to get protocol options." gLogger.error( errStr, "%s: %s" % ( storageName, protocol ) ) return S_ERROR( errStr ) options = res['Value'] # We must have certain values internally even if not supplied in CS protocolDict = {'Access':'', 'Host':'', 'Path':'', 'Port':'', 'Protocol':'', 'ProtocolName':'', 'SpaceToken':'', 'WSUrl':''} for option in options: configPath = '%s/%s' % ( protocolConfigPath, option ) optionValue = gConfig.getValue( configPath, '' ) protocolDict[option] = optionValue # If the user has requested to use Proxy storage if self.proxy: if protocolDict['ProtocolName'] != 'DIP': protocolDict['ProtocolName'] = 'Proxy' # Now update the local and remote protocol lists. # A warning will be given if the Access option is not set. if protocolDict['Access'] == 'remote': self.remoteProtocols.append( protocolDict['ProtocolName'] ) elif protocolDict['Access'] == 'local': self.localProtocols.append( protocolDict['ProtocolName'] ) else: errStr = "StorageFactory.__getProtocolDetails: The 'Access' option for %s:%s is neither 'local' or 'remote'." % ( storageName, protocol ) gLogger.warn( errStr ) # The ProtocolName option must be defined if not protocolDict['ProtocolName']: errStr = "StorageFactory.__getProtocolDetails: 'ProtocolName' option is not defined." gLogger.error( errStr, "%s: %s" % ( storageName, protocol ) ) return S_ERROR( errStr ) return S_OK( protocolDict )
def __getLaunchpadOpts(self): options = gConfig.getOptions("/Website/Launchpad/Options", False) if options and options["OK"]: options = self.__getDataFromCS() else: options = "false" override = gConfig.getOption("/Website/Launchpad/OptionsOverride") if override["OK"]: override = str(override["Value"]).lower() if override == "true": pass else: override = "false" else: override = "false" separator = gConfig.getOption("/Website/Launchpad/ListSeparator") if separator["OK"]: separator = separator["Value"] else: separator = "false" return {"success":"true","result":options,"override":override,"separator":separator}
def __setHistory( self , item , state ): """ Insert item to Load or Save history list in first position and checking for duplications. Return resulting list "item" is a dict "state" should be either "Save" or "Load" but can be any other value """ gLogger.info( "Running setHistory( %s , %s )" % ( item , state ) ) msg = "setHistory() for %s@%s" % ( getUsername() , getSelectedGroup() ) opt = "/Website/" + USER_PROFILE_NAME + "/ShowHistory" history_length = gConfig.getOptions( opt , 5 ) upc = UserProfileClient( "Default" , getRPCClient ) group = str( getSelectedGroup() ) profile_name = USER_PROFILE_NAME + ".History." + state + "." + group result = upc.retrieveVar( profile_name ) gLogger.info( result ) if not result[ "OK" ]: if result[ "Message" ].find( "No data" ) < 0 : gLogger.error( "Result %s: %s" % ( msg , result[ "Message" ] ) ) return S_ERROR( result[ "Message" ] ) history = list() else: history = result[ "Value" ] if not isinstance( history , list ): err = "List expected at: %s" % profile_name gLogger.error( "Result %s: %s" % ( msg , err ) ) return S_ERROR( err ) if( len( history ) > history_length ): history = result[ history_length ] history.insert( 0 , item ) history = uniqueElements( history ) gLogger.error( "History: %s" % history ) result = upc.storeVar( profile_name , history ) gLogger.info( result ) if not result[ "OK" ]: gLogger.error( "Result %s: %s" % ( msg , result[ "Message" ] ) ) return S_ERROR( result[ "Message" ] ) gLogger.info( "Result %s: %s" % ( msg , history ) ) return S_OK( history )
def __setHistory(self, item, state): """ Insert item to Load or Save history list in first position and checking for duplications. Return resulting list "item" is a dict "state" should be either "Save" or "Load" but can be any other value """ gLogger.info("Running setHistory( %s , %s )" % (item, state)) msg = "setHistory() for %s@%s" % (getUsername(), getSelectedGroup()) opt = "/Website/" + USER_PROFILE_NAME + "/ShowHistory" history_length = gConfig.getOptions(opt, 5) upc = UserProfileClient("Default", getRPCClient) group = str(getSelectedGroup()) profile_name = USER_PROFILE_NAME + ".History." + state + "." + group result = upc.retrieveVar(profile_name) gLogger.info(result) if not result["OK"]: if result["Message"].find("No data") < 0: gLogger.error("Result %s: %s" % (msg, result["Message"])) return S_ERROR(result["Message"]) history = list() else: history = result["Value"] if not isinstance(history, list): err = "List expected at: %s" % profile_name gLogger.error("Result %s: %s" % (msg, err)) return S_ERROR(err) if (len(history) > history_length): history = result[history_length] history.insert(0, item) history = uniqueElements(history) gLogger.error("History: %s" % history) result = upc.storeVar(profile_name, history) gLogger.info(result) if not result["OK"]: gLogger.error("Result %s: %s" % (msg, result["Message"])) return S_ERROR(result["Message"]) gLogger.info("Result %s: %s" % (msg, history)) return S_OK(history)
def syncStorageElementsInit(): ses = gConfig.getSections( '/Resources/StorageElements' ) if not ses[ 'OK' ]: gLogger.error( ses[ 'Message' ] ) DIRAC.exit( 2 ) statuses = rsc.getValidStatuses()[ 'Value' ] for se in ses[ 'Value' ]: opts = gConfig.getOptions( '/Resources/StorageElements/%s' % se )[ 'Value' ] statusTypes = [ 'Read', 'Write', 'Check', 'Remove' ] for opt in opts: if not opt.endswith( 'Access' ): continue status = gConfig.getValue( '/Resources/StorageElements/%s/%s' % ( se, opt ) ) if status in [ 'NotAllowed', 'InActive' ]: status = 'Banned' if not status in statuses: gLogger.error( '%s not a valid status for %s - %s' % ( status, se, statusType ) ) continue statusType = opt.replace( 'Access', '' ) statusTypes.remove( statusType ) rsc.modifyElementStatus( 'StorageElement', se, statusType, status = status, reason = 'Init sync' ) for sType in statusTypes: # If there is nothing on the CS, we set the statusType to DEFAULT_STATUS DEFAULT_STATUS = 'Active' rsc.modifyElementStatus( 'StorageElement', se, sType, status = DEFAULT_STATUS, reason = 'Default status' )
def getFTSSites(): FTS = gConfig.getOptions("%s/FTSEndpoints" %g_BaseResourcesSection) return FTS
def getVOMSAttributes( self, proxy, switch = "all" ): """ Return VOMS proxy attributes as list elements if switch="all" (default) OR return the string prepared to be stored in DB if switch="db" OR return the string of elements to be used as the option string in voms-proxy-init if switch="option". If a given proxy is a grid proxy, then function will return an empty list. """ # Get all possible info from voms proxy result = self.getVOMSProxyInfo( proxy, "all" ) if not result["OK"]: return S_ERROR( 'Failed to extract info from proxy: %s' % result[ 'Message' ] ) vomsInfoOutput = List.fromChar( result["Value"], "\n" ) #Get a list of known VOMS attributes validVOMSAttrs = [] result = gConfig.getOptions( "/Registry/VOMS/Mapping" ) if result[ 'OK' ]: for group in result[ 'Value' ]: vA = gConfig.getValue( "/Registry/VOMS/Mapping/%s" % group, "" ) if vA and vA not in validVOMSAttrs: validVOMSAttrs.append( vA ) result = gConfig.getSections( "/Registry/Groups" ) if result[ 'OK' ]: for group in result[ 'Value' ]: vA = gConfig.getValue( "/Registry/Groups/%s/VOMSRole" % group, "" ) if vA and vA not in validVOMSAttrs: validVOMSAttrs.append( vA ) # Parse output of voms-proxy-info command attributes = [] voName = '' nickName = '' for line in vomsInfoOutput: fields = List.fromChar( line, ":" ) key = fields[0] value = " ".join( fields[1:] ) if key == "VO": voName = value elif key == "attribute": # Cut off unsupported Capability selection part if value.find( "nickname" ) == 0: nickName = "=".join( List.fromChar( value, "=" )[ 1: ] ) else: value = value.replace( "/Capability=NULL" , "" ) value = value.replace( "/Role=NULL" , "" ) if value and value not in attributes and value in validVOMSAttrs: attributes.append( value ) # Sorting and joining attributes if switch == "db": returnValue = ":".join( attributes ) elif switch == "option": if len( attributes ) > 1: returnValue = voName + " -order " + ' -order '.join( attributes ) elif attributes: returnValue = voName + ":" + attributes[0] else: returnValue = voName elif switch == 'nickname': returnValue = nickName elif switch == 'all': returnValue = attributes return S_OK( returnValue )
def getCSOptions( opt ): return gConfig.getOptions( "%s/%s" % ( BASECS, opt ) )
def getDestinationSEList(outputSE, site, outputmode='Any'): """ Evaluate the output SE list from a workflow and return the concrete list of SEs to upload output data. """ # Add output SE defined in the job description gLogger.info('Resolving workflow output SE description: %s' % outputSE) # Check if the SE is defined explicitly for the site prefix = site.split('.')[0] country = site.split('.')[-1] # Concrete SE name result = gConfig.getOptions('/Resources/StorageElements/' + outputSE) if result['OK']: gLogger.info('Found concrete SE %s' % outputSE) return S_OK([outputSE]) # There is an alias defined for this Site alias_se = gConfig.getValue('/Resources/Sites/%s/%s/AssociatedSEs/%s' % (prefix, site, outputSE), []) if alias_se: gLogger.info('Found associated SE for site %s' % (alias_se)) return S_OK(alias_se) localSEs = getSEsForSite(site)['Value'] gLogger.verbose('Local SE list is: %s' % (localSEs)) groupSEs = gConfig.getValue('/Resources/StorageElementGroups/' + outputSE, []) gLogger.verbose('Group SE list is: %s' % (groupSEs)) if not groupSEs: return S_ERROR('Failed to resolve SE ' + outputSE) if outputmode.lower() == "local": for se in localSEs: if se in groupSEs: gLogger.info('Found eligible local SE: %s' % (se)) return S_OK([se]) #check if country is already one with associated SEs associatedSE = gConfig.getValue('/Resources/Countries/%s/AssociatedSEs/%s' % (country, outputSE), '') if associatedSE: gLogger.info('Found associated SE %s in /Resources/Countries/%s/AssociatedSEs/%s' % (associatedSE, country, outputSE)) return S_OK([associatedSE]) # Final check for country associated SE count = 0 assignedCountry = country while count < 10: gLogger.verbose('Loop count = %s' % (count)) gLogger.verbose("/Resources/Countries/%s/AssignedTo" % assignedCountry) opt = gConfig.getOption("/Resources/Countries/%s/AssignedTo" % assignedCountry) if opt['OK'] and opt['Value']: assignedCountry = opt['Value'] gLogger.verbose('/Resources/Countries/%s/AssociatedSEs' % assignedCountry) assocCheck = gConfig.getOption('/Resources/Countries/%s/AssociatedSEs' % assignedCountry) if assocCheck['OK'] and assocCheck['Value']: break count += 1 if not assignedCountry: return S_ERROR('Could not determine associated SE list for %s' % country) alias_se = gConfig.getValue('/Resources/Countries/%s/AssociatedSEs/%s' % (assignedCountry, outputSE), []) if alias_se: gLogger.info('Found alias SE for site: %s' % alias_se) return S_OK(alias_se) else: gLogger.error('Could not establish alias SE for country %s from section: /Resources/Countries/%s/AssociatedSEs/%s' % (country, assignedCountry, outputSE)) return S_ERROR('Failed to resolve SE ' + outputSE) # For collective Any and All modes return the whole group # Make sure that local SEs are passing first newSEList = [] for se in groupSEs: if se in localSEs: newSEList.append(se) uniqueSEs = uniqueElements(newSEList + groupSEs) gLogger.verbose('Found unique SEs: %s' % (uniqueSEs)) return S_OK(uniqueSEs)