def getRawSchema():
"""
Load the schema from the CS
"""
base = "%s/Schema" % (BASECS)
schema = []
explore = [("", schema)]
while len(explore):
parentName, parentData = explore.pop(0)
fullName = "%s/%s" % (base, parentName)
result = gConfig.getSections(fullName)
if not result['OK']:
continue
sectionsList = result['Value']
for sName in sectionsList:
sData = []
parentData.append(("%s/%s" % (parentName, sName), sData))
explore.append((sName, sData))
result = gConfig.getOptions(fullName)
if not result['OK']:
continue
optionsList = result['Value']
for opName in optionsList:
opVal = gConfig.getValue("%s/%s" % (fullName, opName))
if opVal.find("link|") == 0:
parentData.append(("link", opName, opVal[5:]))
else:
parentData.append(("app", opName, opVal))
return schema
def _getConfigStorageName(self, storageName):
"""
This gets the name of the storage the configuration service.
If the storage is an alias for another the resolution is performed.
'storageName' is the storage section to check in the CS
"""
configPath = "%s/%s" % (self.rootConfigPath, storageName)
res = gConfig.getOptions(configPath)
if not res["OK"]:
errStr = "StorageFactory._getConfigStorageName: Failed to get storage options"
gLogger.error(errStr, res["Message"])
return S_ERROR(errStr)
if not res["Value"]:
errStr = "StorageFactory._getConfigStorageName: Supplied storage doesn't exist."
gLogger.error(errStr, configPath)
return S_ERROR(errStr)
if "Alias" in res["Value"]:
configPath = "%s/%s/Alias" % (self.rootConfigPath, storageName)
aliasName = gConfig.getValue(configPath)
result = self._getConfigStorageName(aliasName)
if not result["OK"]:
errStr = "StorageFactory._getConfigStorageName: Supplied storage doesn't exist."
gLogger.error(errStr, configPath)
return S_ERROR(errStr)
resolvedName = result["Value"]
else:
resolvedName = storageName
return S_OK(resolvedName)
def _getConfigStorageOptions( self, storageName, derivedStorageName = None ):
""" Get the options associated to the StorageElement as defined in the CS
"""
optionsDict = {}
# We first get the options of the baseSE, and then overwrite with the derivedSE
for seName in ( storageName, derivedStorageName ) if derivedStorageName else ( storageName, ):
storageConfigPath = cfgPath( self.rootConfigPath, seName )
res = gConfig.getOptions( storageConfigPath )
if not res['OK']:
errStr = "StorageFactory._getStorageOptions: Failed to get storage options."
gLogger.error( errStr, "%s: %s" % ( seName, res['Message'] ) )
return S_ERROR( errStr )
for option in set( res['Value'] ) - set( ( 'ReadAccess', 'WriteAccess', 'CheckAccess', 'RemoveAccess' ) ):
optionConfigPath = cfgPath( storageConfigPath, option )
default = [] if option in [ 'VO' ] else ''
optionsDict[option] = gConfig.getValue( optionConfigPath, default )
# The status is that of the derived SE only
seName = derivedStorageName if derivedStorageName else storageName
res = self.resourceStatus.getStorageElementStatus( seName )
if not res[ 'OK' ]:
errStr = "StorageFactory._getStorageOptions: Failed to get storage status"
gLogger.error( errStr, "%s: %s" % ( seName, res['Message'] ) )
return S_ERROR( errStr )
# For safety, we did not add the ${statusType}Access keys
# this requires modifications in the StorageElement class
# We add the dictionary with the statusTypes and values
# { 'statusType1' : 'status1', 'statusType2' : 'status2' ... }
optionsDict.update( res[ 'Value' ][ seName ] )
return S_OK( optionsDict )
def _getBaseStorageName( self, storageName ):
"""
This gets the name of the base SE in case the SE is defined from a base SE.
'storageName' is the storage section to check in the CS
"""
configPath = '%s/%s' % ( self.rootConfigPath, storageName )
res = gConfig.getOptions( configPath )
if not res['OK']:
errStr = "StorageFactory._getConfigStorageName: Failed to get storage options"
gLogger.error( errStr, res['Message'] )
return S_ERROR( errStr )
if not res['Value']:
errStr = "StorageFactory._getConfigStorageName: Supplied storage doesn't exist."
gLogger.error( errStr, configPath )
return S_ERROR( errStr )
if 'BaseSE' in res['Value']:
configPath = '%s/%s/BaseSE' % ( self.rootConfigPath, storageName )
baseName = gConfig.getValue( configPath )
# Just in case the base is an alias ;-)
result = self._getConfigStorageName( baseName )
if not result['OK']:
return result
resolvedName = result['Value']
else:
resolvedName = storageName
return S_OK( resolvedName )
def _getCatalogConfigDetails(self, catalogName):
# First obtain the options that are available
catalogConfigPath = "%s/%s" % (self.rootConfigPath, catalogName)
res = gConfig.getOptions(catalogConfigPath)
if not res["OK"]:
errStr = "FileCatalog._getCatalogConfigDetails: Failed to get catalog options."
gLogger.error(errStr, catalogName)
return S_ERROR(errStr)
catalogConfig = {}
for option in res["Value"]:
configPath = "%s/%s" % (catalogConfigPath, option)
optionValue = gConfig.getValue(configPath)
catalogConfig[option] = optionValue
# The 'Status' option should be defined (default = 'Active')
if not catalogConfig.has_key("Status"):
warnStr = "FileCatalog._getCatalogConfigDetails: 'Status' option not defined."
gLogger.warn(warnStr, catalogName)
catalogConfig["Status"] = "Active"
# The 'AccessType' option must be defined
if not catalogConfig.has_key("AccessType"):
errStr = "FileCatalog._getCatalogConfigDetails: Required option 'AccessType' not defined."
gLogger.error(errStr, catalogName)
return S_ERROR(errStr)
# Anything other than 'True' in the 'Master' option means it is not
if not catalogConfig.has_key("Master"):
catalogConfig["Master"] = False
elif catalogConfig["Master"] == "True":
catalogConfig["Master"] = True
else:
catalogConfig["Master"] = False
return S_OK(catalogConfig)
def _getConfigStorageName(self, storageName, referenceType):
"""
This gets the name of the storage the configuration service.
If the storage is a reference to another SE the resolution is performed.
'storageName' is the storage section to check in the CS
"""
configPath = '%s/%s' % (self.rootConfigPath, storageName)
res = gConfig.getOptions(configPath)
if not res['OK']:
errStr = "StorageFactory._getConfigStorageName: Failed to get storage options"
gLogger.error(errStr, res['Message'])
return S_ERROR(errStr)
if not res['Value']:
errStr = "StorageFactory._getConfigStorageName: Supplied storage doesn't exist."
gLogger.error(errStr, configPath)
return S_ERROR(errStr)
if referenceType in res['Value']:
configPath = cfgPath(self.rootConfigPath, storageName,
referenceType)
referenceName = gConfig.getValue(configPath)
result = self._getConfigStorageName(referenceName, 'Alias')
if not result['OK']:
return result
resolvedName = result['Value']
else:
resolvedName = storageName
return S_OK(resolvedName)
def _getCatalogConfigDetails(self, catalogName):
# First obtain the options that are available
catalogConfigPath = '%s/%s' % (self.rootConfigPath, catalogName)
res = gConfig.getOptions(catalogConfigPath)
if not res['OK']:
errStr = "FileCatalog._getCatalogConfigDetails: Failed to get catalog options."
gLogger.error(errStr, catalogName)
return S_ERROR(errStr)
catalogConfig = {}
for option in res['Value']:
configPath = '%s/%s' % (catalogConfigPath, option)
optionValue = gConfig.getValue(configPath)
catalogConfig[option] = optionValue
# The 'Status' option should be defined (default = 'Active')
if 'Status' not in catalogConfig:
warnStr = "FileCatalog._getCatalogConfigDetails: 'Status' option not defined."
gLogger.warn(warnStr, catalogName)
catalogConfig['Status'] = 'Active'
# The 'AccessType' option must be defined
if 'AccessType' not in catalogConfig:
errStr = "FileCatalog._getCatalogConfigDetails: Required option 'AccessType' not defined."
gLogger.error(errStr, catalogName)
return S_ERROR(errStr)
# Anything other than 'True' in the 'Master' option means it is not
catalogConfig['Master'] = (catalogConfig.setdefault('Master',
False) == 'True')
return S_OK(catalogConfig)
def __getHistory(self, state):
"""
Just get the history based on state
Return resulting list
"state" can be either "Save" or "Load"
"""
gLogger.info("Running getHistory( %s )" % state)
msg = "getHistory() for %s@%s" % (getUsername(), getSelectedGroup())
opt = "/Website/" + USER_PROFILE_NAME + "/ShowHistory"
history_length = gConfig.getOptions(opt, 5)
upc = UserProfileClient("Default", getRPCClient)
group = str(getSelectedGroup())
profile_name = USER_PROFILE_NAME + ".History." + state + "." + group
result = upc.retrieveVar(profile_name)
gLogger.info(result)
if not result["OK"]:
if result["Message"].find("No data") < 0:
gLogger.error("Result %s: %s" % (msg, result["Message"]))
return S_ERROR(result["Message"])
history = list()
else:
history = result["Value"]
if not isinstance(history, list):
err = "List expected at: %s" % profile_name
gLogger.error("Result %s: %s" % (msg, err))
return S_ERROR(err)
if (len(history) > history_length):
history = result[history_length]
gLogger.info("Result %s: %s" % (msg, history))
return S_OK(history)
def __generateSchema( self, base, path ):
"""
Generate a menu schema based on the user credentials
"""
#Calculate schema
schema = []
fullName = "%s/%s" % ( base, path )
result = gConfig.getSections( fullName )
if not result[ 'OK' ]:
return schema
sectionsList = result[ 'Value' ]
for sName in sectionsList:
subSchema = self.__generateSchema( base, "%s/%s" % ( path, sName ) )
if subSchema:
schema.append( ( sName, subSchema ) )
result = gConfig.getOptions( fullName )
if not result[ 'OK' ]:
return schema
optionsList = result[ 'Value' ]
for opName in optionsList:
opVal = gConfig.getValue( "%s/%s" % ( fullName, opName ) )
if opVal.find( "link|" ) == 0:
schema.append( ( "link", opName, opVal[5:] ) )
continue
if self.__isGroupAuthApp( opVal ):
schema.append( ( "app", opName, opVal ) )
return schema
def _getCatalogConfigDetails( self, catalogName ):
# First obtain the options that are available
catalogConfigPath = '%s/%s' % ( self.rootConfigPath, catalogName )
res = gConfig.getOptions( catalogConfigPath )
if not res['OK']:
errStr = "FileCatalog._getCatalogConfigDetails: Failed to get catalog options."
gLogger.error( errStr, catalogName )
return S_ERROR( errStr )
catalogConfig = {}
for option in res['Value']:
configPath = '%s/%s' % ( catalogConfigPath, option )
optionValue = gConfig.getValue( configPath )
catalogConfig[option] = optionValue
# The 'Status' option should be defined (default = 'Active')
if 'Status' not in catalogConfig:
warnStr = "FileCatalog._getCatalogConfigDetails: 'Status' option not defined."
gLogger.warn( warnStr, catalogName )
catalogConfig['Status'] = 'Active'
# The 'AccessType' option must be defined
if 'AccessType' not in catalogConfig:
errStr = "FileCatalog._getCatalogConfigDetails: Required option 'AccessType' not defined."
gLogger.error( errStr, catalogName )
return S_ERROR( errStr )
# Anything other than 'True' in the 'Master' option means it is not
catalogConfig['Master'] = ( catalogConfig.setdefault( 'Master', False ) == 'True' )
return S_OK( catalogConfig )
def __generateSchema(self, base, path):
"""
Generate a menu schema based on the user credentials
"""
#Calculate schema
schema = []
fullName = "%s/%s" % (base, path)
result = gConfig.getSections(fullName)
if not result['OK']:
return schema
sectionsList = result['Value']
for sName in sectionsList:
subSchema = self.__generateSchema(base, "%s/%s" % (path, sName))
if subSchema:
schema.append((sName, subSchema))
result = gConfig.getOptions(fullName)
if not result['OK']:
return schema
optionsList = result['Value']
for opName in optionsList:
opVal = gConfig.getValue("%s/%s" % (fullName, opName))
if opVal.find("link|") == 0:
schema.append(("link", opName, opVal[5:]))
continue
if self.__isGroupAuthApp(opVal):
schema.append(("app", opName, opVal))
return schema
def _getConfigStorageName( self, storageName, referenceType ):
"""
This gets the name of the storage the configuration service.
If the storage is a reference to another SE the resolution is performed.
'storageName' is the storage section to check in the CS
"""
configPath = '%s/%s' % ( self.rootConfigPath, storageName )
res = gConfig.getOptions( configPath )
if not res['OK']:
errStr = "StorageFactory._getConfigStorageName: Failed to get storage options"
gLogger.error( errStr, res['Message'] )
return S_ERROR( errStr )
if not res['Value']:
errStr = "StorageFactory._getConfigStorageName: Supplied storage doesn't exist."
gLogger.error( errStr, configPath )
return S_ERROR( errStr )
if referenceType in res['Value']:
configPath = cfgPath( self.rootConfigPath, storageName, referenceType )
referenceName = gConfig.getValue( configPath )
result = self._getConfigStorageName( referenceName, 'Alias' )
if not result['OK']:
return result
resolvedName = result['Value']
else:
resolvedName = storageName
return S_OK( resolvedName )
def __getHistory( self , state ):
"""
Just get the history based on state
Return resulting list
"state" can be either "Save" or "Load"
"""
gLogger.info( "Running getHistory( %s )" % state )
msg = "getHistory() for %s@%s" % ( getUsername() , getSelectedGroup() )
opt = "/Website/" + USER_PROFILE_NAME + "/ShowHistory"
history_length = gConfig.getOptions( opt , 5 )
upc = UserProfileClient( "Default" , getRPCClient )
group = str( getSelectedGroup() )
profile_name = USER_PROFILE_NAME + ".History." + state + "." + group
result = upc.retrieveVar( profile_name )
gLogger.info( result )
if not result[ "OK" ]:
if result[ "Message" ].find( "No data" ) < 0 :
gLogger.error( "Result %s: %s" % ( msg , result[ "Message" ] ) )
return S_ERROR( result[ "Message" ] )
history = list()
else:
history = result[ "Value" ]
if not isinstance( history , list ):
err = "List expected at: %s" % profile_name
gLogger.error( "Result %s: %s" % ( msg , err ) )
return S_ERROR( err )
if( len( history ) > history_length ):
history = result[ history_length ]
gLogger.info( "Result %s: %s" % ( msg , history ) )
return S_OK( history )
def _getConfigStorageOptions( self, storageName ):
""" Get the options associated to the StorageElement as defined in the CS
"""
storageConfigPath = cfgPath( self.rootConfigPath, storageName )
res = gConfig.getOptions( storageConfigPath )
if not res['OK']:
errStr = "StorageFactory._getStorageOptions: Failed to get storage options."
gLogger.error( errStr, "%s: %s" % ( storageName, res['Message'] ) )
return S_ERROR( errStr )
options = res['Value']
optionsDict = {}
for option in options:
if option in [ 'ReadAccess', 'WriteAccess', 'CheckAccess', 'RemoveAccess']:
continue
optionConfigPath = cfgPath( storageConfigPath, option )
if option in [ 'VO' ]:
optionsDict[option] = gConfig.getValue( optionConfigPath, [] )
else:
optionsDict[option] = gConfig.getValue( optionConfigPath, '' )
res = self.resourceStatus.getStorageElementStatus( storageName )
if not res[ 'OK' ]:
errStr = "StorageFactory._getStorageOptions: Failed to get storage status"
gLogger.error( errStr, "%s: %s" % ( storageName, res['Message'] ) )
return S_ERROR( errStr )
# For safety, we did not add the ${statusType}Access keys
# this requires modifications in the StorageElement class
# We add the dictionary with the statusTypes and values
# { 'statusType1' : 'status1', 'statusType2' : 'status2' ... }
optionsDict.update( res[ 'Value' ][ storageName ] )
return S_OK( optionsDict )
def getRawSchema():
"""
Load the schema from the CS
"""
base = "%s/Schema" % ( BASECS )
schema = []
explore = [ ( "", schema ) ]
while len( explore ):
parentName, parentData = explore.pop( 0 )
fullName = "%s/%s" % ( base, parentName )
result = gConfig.getSections( fullName )
if not result[ 'OK' ]:
continue
sectionsList = result[ 'Value' ]
for sName in sectionsList:
sData = []
parentData.append( ( "%s/%s" % ( parentName, sName ), sData ) )
explore.append( ( sName, sData ) )
result = gConfig.getOptions( fullName )
if not result[ 'OK' ]:
continue
optionsList = result[ 'Value' ]
for opName in optionsList:
opVal = gConfig.getValue( "%s/%s" % ( fullName, opName ) )
if opVal.find( "link|" ) == 0:
parentData.append( ( "link", opName, opVal[5:] ) )
else:
parentData.append( ( "app", opName, opVal ) )
return schema
def _getConfigStorageOptions( self, storageName ):
""" Get the options associated to the StorageElement as defined in the CS
"""
storageConfigPath = '%s/%s' % ( self.rootConfigPath, storageName )
res = gConfig.getOptions( storageConfigPath )
if not res['OK']:
errStr = "StorageFactory._getStorageOptions: Failed to get storage options."
gLogger.error( errStr, "%s: %s" % ( storageName, res['Message'] ) )
return S_ERROR( errStr )
options = res['Value']
optionsDict = {}
for option in options:
if option in [ 'ReadAccess', 'WriteAccess', 'CheckAccess', 'RemoveAccess']:
continue
optionConfigPath = '%s/%s' % ( storageConfigPath, option )
optionsDict[option] = gConfig.getValue( optionConfigPath, '' )
res = self.resourceStatus.getStorageElementStatus( storageName )
if not res[ 'OK' ]:
errStr = "StorageFactory._getStorageOptions: Failed to get storage status"
gLogger.error( errStr, "%s: %s" % ( storageName, res['Message'] ) )
return S_ERROR( errStr )
# For safety, we did not add the ${statusType}Access keys
# this requires modifications in the StorageElement class
# We add the dictionary with the statusTypes and values
# { 'statusType1' : 'status1', 'statusType2' : 'status2' ... }
optionsDict.update( res[ 'Value' ][ storageName ] )
return S_OK( optionsDict )
def _getTestsList():
result = {}
for test in gConfig.getSections('/Systems/Framework/Production/Agents/SAMLauncherAgent')['Value']:
result[test] = {}
for option in gConfig.getOptions('/Systems/Framework/Production/Agents/SAMLauncherAgent/'+test)['Value']:
result[test][option] = gConfig.getOption('/Systems/Framework/Production/Agents/SAMLauncherAgent/'+test+'/'+option)['Value']
return result
def _getConfigStorageProtocolDetails(self, storageName, protocol):
"""
Parse the contents of the protocol block
"""
# First obtain the options that are available
protocolConfigPath = '%s/%s/%s' % (self.rootConfigPath, storageName,
protocol)
res = gConfig.getOptions(protocolConfigPath)
if not res['OK']:
errStr = "StorageFactory.__getProtocolDetails: Failed to get protocol options."
gLogger.error(errStr, "%s: %s" % (storageName, protocol))
return S_ERROR(errStr)
options = res['Value']
# We must have certain values internally even if not supplied in CS
protocolDict = {
'Access': '',
'Host': '',
'Path': '',
'Port': '',
'Protocol': '',
'ProtocolName': '',
'SpaceToken': '',
'WSUrl': ''
}
for option in options:
configPath = '%s/%s' % (protocolConfigPath, option)
optionValue = gConfig.getValue(configPath, '')
protocolDict[option] = optionValue
# Evaluate the base path taking into account possible VO specific setting
if self.vo:
result = gConfig.getOptionsDict(
cfgPath(protocolConfigPath, 'VOPath'))
voPath = ''
if result['OK']:
voPath = result['Value'].get(self.vo, '')
if voPath:
protocolDict['Path'] = voPath
# Now update the local and remote protocol lists.
# A warning will be given if the Access option is not set.
if protocolDict['Access'] == 'remote':
self.remoteProtocols.append(protocolDict['ProtocolName'])
elif protocolDict['Access'] == 'local':
self.localProtocols.append(protocolDict['ProtocolName'])
else:
errStr = "StorageFactory.__getProtocolDetails: The 'Access' option for %s:%s is neither 'local' or 'remote'." % (
storageName, protocol)
gLogger.warn(errStr)
# The ProtocolName option must be defined
if not protocolDict['ProtocolName']:
errStr = "StorageFactory.__getProtocolDetails: 'ProtocolName' option is not defined."
gLogger.error(errStr, "%s: %s" % (storageName, protocol))
return S_ERROR(errStr)
return S_OK(protocolDict)
def _getFTSEndpoints(basePath='Resources/FTSEndpoints/FTS3'):
"""
Gets all FTS endpoints that are in CS
"""
result = gConfig.getOptions(basePath)
if result['OK']:
return result['Value']
return []
def getFTS3():
'''
Gets all storage elements from /Resources/FTSEndpoints
'''
_basePath = 'Resources/FTSEndpoints/FTS3'
ftsEndpoints = gConfig.getOptions(_basePath)
return ftsEndpoints
def getOptions(self, path):
""" Mock the getOptions call of gConfig
It reads from dict_cs
"""
if 'StorageElements' not in path:
return gConfig.getOptions(path)
csSection = self.crawlCS(path)
options = [opt for opt, val in csSection.iteritems() if not isinstance(val, dict)]
return S_OK(options)
def getFTS():
"""
Gets all FTS endpoints
"""
result = gConfig.getOptions("Resources/FTSEndpoints/FTS3")
if result["OK"]:
return result
return S_OK([])
def getFTS():
'''
Gets all storage elements from /Resources/FTSEndpoints
'''
_basePath = 'Resources/FTSEndpoints'
ftsEndpoints = gConfig.getOptions( _basePath )
return ftsEndpoints
def getFTS():
"""
Gets all FTS endpoints
"""
result = gConfig.getOptions('Resources/FTSEndpoints/FTS3')
if result['OK']:
return result
return S_OK([])
def _getFTSEndpoints(basePath='Resources/FTSEndpoints/FTS3'):
'''
Gets all FTS endpoints that are in CS
'''
result = gConfig.getOptions(basePath)
if result['OK']:
return result['Value']
else: # nothing found
return []
def getOptions(self, path):
""" Mock the getOptions call of gConfig
It reads from dict_cs
"""
if 'StorageElements' not in path and 'StorageElementBases' not in path:
return gConfig.getOptions(path)
csSection = self.crawlCS(path)
if not csSection:
return S_ERROR("Not a valid section")
options = [opt for opt, val in csSection.iteritems() if not isinstance(val, dict)]
return S_OK(options)
def _getConfigStorageOptions(self,
storageName,
derivedStorageName=None,
seConfigPath=SE_CONFIG_PATH):
"""
Get the options associated to the StorageElement as defined in the CS
:param storageName: is the storage section to check in the CS
:param seConfigPath: the path of the storage section.
It can be /Resources/StorageElements or StorageElementBases
:param derivedStorageName: is the storage section of a derived storage if it inherits from a base
:return: options associated to the StorageElement as defined in the CS
"""
optionsDict = {}
# We first get the options of the baseSE, and then overwrite with the derivedSE
for seName in (storageName,
derivedStorageName) if derivedStorageName else (
storageName, ):
storageConfigPath = cfgPath(seConfigPath, seName)
res = gConfig.getOptions(storageConfigPath)
if not res["OK"]:
errStr = "StorageFactory._getStorageOptions: Failed to get storage options."
gLogger.error(errStr, "%s: %s" % (seName, res["Message"]))
return S_ERROR(errStr)
for option in set(res["Value"]) - set(
("ReadAccess", "WriteAccess", "CheckAccess", "RemoveAccess")):
optionConfigPath = cfgPath(storageConfigPath, option)
default = [] if option in [
"VO", "AccessProtocols", "WriteProtocols"
] else ""
optionsDict[option] = gConfig.getValue(optionConfigPath,
default)
# We update the seConfigPath in order to find option in derivedSE now
seConfigPath = SE_CONFIG_PATH
# The status is that of the derived SE only
seName = derivedStorageName if derivedStorageName else storageName
res = self.resourceStatus.getElementStatus(seName, "StorageElement")
if not res["OK"]:
errStr = "StorageFactory._getStorageOptions: Failed to get storage status"
gLogger.error(errStr, "%s: %s" % (seName, res["Message"]))
return S_ERROR(errStr)
# For safety, we did not add the ${statusType}Access keys
# this requires modifications in the StorageElement class
# We add the dictionary with the statusTypes and values
# { 'statusType1' : 'status1', 'statusType2' : 'status2' ... }
optionsDict.update(res["Value"][seName])
return S_OK(optionsDict)
def _getConfigStorageProtocolDetails(self, storageName, protocol):
"""
Parse the contents of the protocol block
"""
# First obtain the options that are available
protocolConfigPath = '%s/%s/%s' % (self.rootConfigPath, storageName,
protocol)
res = gConfig.getOptions(protocolConfigPath)
if not res['OK']:
errStr = "StorageFactory.__getProtocolDetails: Failed to get protocol options."
gLogger.error(errStr, "%s: %s" % (storageName, protocol))
return S_ERROR(errStr)
options = res['Value']
# We must have certain values internally even if not supplied in CS
protocolDict = {
'Access': '',
'Host': '',
'Path': '',
'Port': '',
'Protocol': '',
'ProtocolName': '',
'SpaceToken': '',
'WSUrl': ''
}
for option in options:
configPath = '%s/%s' % (protocolConfigPath, option)
optionValue = gConfig.getValue(configPath, '')
protocolDict[option] = optionValue
# If the user has requested to use Proxy storage
if self.proxy:
if protocolDict['ProtocolName'] != 'DIP':
protocolDict['ProtocolName'] = 'Proxy'
# Now update the local and remote protocol lists.
# A warning will be given if the Access option is not set.
if protocolDict['Access'] == 'remote':
self.remoteProtocols.append(protocolDict['ProtocolName'])
elif protocolDict['Access'] == 'local':
self.localProtocols.append(protocolDict['ProtocolName'])
else:
errStr = "StorageFactory.__getProtocolDetails: The 'Access' option for %s:%s is neither 'local' or 'remote'." % (
storageName, protocol)
gLogger.warn(errStr)
# The ProtocolName option must be defined
if not protocolDict['ProtocolName']:
errStr = "StorageFactory.__getProtocolDetails: 'ProtocolName' option is not defined."
gLogger.error(errStr, "%s: %s" % (storageName, protocol))
return S_ERROR(errStr)
return S_OK(protocolDict)
def _getConfigStorageProtocolDetails(self, storageName, protocolSection,
seConfigPath=SE_CONFIG_PATH, checkAccess=True):
"""
Parse the contents of the protocol block
:param storageName: is the storage section to check in the CS
:param protocolSection: name of the protocol section to find information
:param seConfigPath: the path of the storage section.
It can be /Resources/StorageElements or StorageElementBases
:param checkAccess: if not set, don't complain if "Access" is not in the section
:return: dictionary of the protocol options
"""
# First obtain the options that are available
protocolConfigPath = cfgPath(seConfigPath, storageName, protocolSection)
res = gConfig.getOptions(protocolConfigPath)
if not res['OK']:
errStr = "StorageFactory.__getProtocolDetails: Failed to get protocol options."
gLogger.error(errStr, "%s: %s" % (storageName, protocolSection))
return S_ERROR(errStr)
options = res['Value']
# We must have certain values internally even if not supplied in CS
protocolDict = {'Access': '', 'Host': '', 'Path': '', 'Port': '', 'Protocol': '', 'SpaceToken': '', 'WSUrl': ''}
for option in options:
configPath = cfgPath(protocolConfigPath, option)
optionValue = gConfig.getValue(configPath, '')
protocolDict[option] = optionValue
# Evaluate the base path taking into account possible VO specific setting
if self.vo:
result = gConfig.getOptionsDict(cfgPath(protocolConfigPath, 'VOPath'))
voPath = ''
if result['OK']:
voPath = result['Value'].get(self.vo, '')
if voPath:
protocolDict['Path'] = voPath
# Now update the local and remote protocol lists.
# A warning will be given if the Access option is not set and the plugin is not already in remote or local.
plugin = protocolDict.get('PluginName', protocolSection)
if protocolDict['Access'].lower() == 'remote':
self.remotePlugins.append(plugin)
elif protocolDict['Access'].lower() == 'local':
self.localPlugins.append(plugin)
# If it is a derived SE, this is normal, no warning
elif checkAccess and protocolSection not in self.protocols:
errStr = "StorageFactory.__getProtocolDetails: The 'Access' option \
for %s:%s is neither 'local' or 'remote'." % (storageName, protocolSection)
gLogger.warn(errStr)
return S_OK(protocolDict)
def getFTS2():
'''
Gets all storage elements from /Resources/FTSEndpoints
'''
_basePath = 'Resources/FTSEndpoints/FTS2'
ftsEndpoints = gConfig.getOptions( _basePath )
ftsEndpointDefaultLocation = gConfig.getValue( '/Resources/FTSEndpoints/Default/FTSEndpoint', '' )
if ftsEndpoints['OK'] and ftsEndpointDefaultLocation:
ftsEndpoints['Value'].append( ftsEndpointDefaultLocation )
return ftsEndpoints
def getFTS3Servers():
""" get FTSServers for sites
"""
csPath = cfgPath( gBaseResourcesSection, "FTSEndpoints/FTS3" )
# We do it in two times to keep the order
ftsServerNames = gConfig.getOptions( csPath ).get( 'Value', [] )
ftsServers = []
for name in ftsServerNames:
ftsServers.append( gConfig.getValue( cfgPath( csPath, name ) ) )
return S_OK( ftsServers )
def getFTS3Servers():
""" get FTSServers for sites
"""
csPath = cfgPath( gBaseResourcesSection, "FTSEndpoints/FTS3" )
# We do it in two times to keep the order
ftsServerNames = gConfig.getOptions( csPath ).get( 'Value', [] )
ftsServers = []
for name in ftsServerNames:
ftsServers.append( gConfig.getValue( cfgPath( csPath, name ) ) )
return S_OK( ftsServers )
def getOptions(self, path):
"""Mock the getOptions call of gConfig
It reads from dict_cs
"""
if "StorageElements" not in path and "StorageElementBases" not in path:
return gConfig.getOptions(path)
csSection = self.crawlCS(path)
if not csSection:
return S_ERROR("Not a valid section")
options = [
opt for opt, val in csSection.items() if not isinstance(val, dict)
]
return S_OK(options)
def _getConfigStorageProtocolDetails( self, storageName, protocolSection ):
"""
Parse the contents of the protocol block
"""
# First obtain the options that are available
protocolConfigPath = cfgPath( self.rootConfigPath, storageName, protocolSection )
res = gConfig.getOptions( protocolConfigPath )
if not res['OK']:
errStr = "StorageFactory.__getProtocolDetails: Failed to get protocol options."
gLogger.error( errStr, "%s: %s" % ( storageName, protocolSection ) )
return S_ERROR( errStr )
options = res['Value']
# We must have certain values internally even if not supplied in CS
protocolDict = {'Access':'', 'Host':'', 'Path':'', 'Port':'', 'Protocol':'', 'PluginName':'', 'SpaceToken':'', 'WSUrl':''}
for option in options:
configPath = cfgPath( protocolConfigPath, option )
optionValue = gConfig.getValue( configPath, '' )
protocolDict[option] = optionValue
# This is a temporary for backward compatibility
if "ProtocolName" in protocolDict and not protocolDict['PluginName']:
protocolDict['PluginName'] = protocolDict['ProtocolName']
protocolDict.pop( 'ProtocolName', None )
# Evaluate the base path taking into account possible VO specific setting
if self.vo:
result = gConfig.getOptionsDict( cfgPath( protocolConfigPath, 'VOPath' ) )
voPath = ''
if result['OK']:
voPath = result['Value'].get( self.vo, '' )
if voPath:
protocolDict['Path'] = voPath
# Now update the local and remote protocol lists.
# A warning will be given if the Access option is not set.
if protocolDict['Access'].lower() == 'remote':
self.remotePlugins.append( protocolDict['PluginName'] )
elif protocolDict['Access'].lower() == 'local':
self.localPlugins.append( protocolDict['PluginName'] )
else:
errStr = "StorageFactory.__getProtocolDetails: The 'Access' option for %s:%s is neither 'local' or 'remote'." % ( storageName, protocolSection )
gLogger.warn( errStr )
# The PluginName option must be defined
if not protocolDict['PluginName']:
errStr = "StorageFactory.__getProtocolDetails: 'PluginName' option is not defined."
gLogger.error( errStr, "%s: %s" % ( storageName, protocolSection ) )
return S_ERROR( errStr )
return S_OK( protocolDict )
def getFTS2():
'''
Gets all storage elements from /Resources/FTSEndpoints
'''
_basePath = 'Resources/FTSEndpoints/FTS2'
ftsEndpoints = gConfig.getOptions(_basePath)
ftsEndpointDefaultLocation = gConfig.getValue(
'/Resources/FTSEndpoints/Default/FTSEndpoint', '')
if ftsEndpoints['OK'] and ftsEndpointDefaultLocation:
ftsEndpoints['Value'].append(ftsEndpointDefaultLocation)
return ftsEndpoints
def _getConfigStorageOptions( self, storageName ):
""" Get the options associated to the StorageElement as defined in the CS
"""
storageConfigPath = '%s/%s' % ( self.rootConfigPath, storageName )
res = gConfig.getOptions( storageConfigPath )
if not res['OK']:
errStr = "StorageFactory._getStorageOptions: Failed to get storage options."
gLogger.error( errStr, "%s: %s" % ( storageName, res['Message'] ) )
return S_ERROR( errStr )
options = res['Value']
optionsDict = {}
for option in options:
optionConfigPath = '%s/%s' % ( storageConfigPath, option )
optionsDict[option] = gConfig.getValue( optionConfigPath, '' )
return S_OK( optionsDict )
def _getConfigStorageProtocolDetails(self, storageName, protocol):
"""
Parse the contents of the protocol block
"""
# First obtain the options that are available
protocolConfigPath = "%s/%s/%s" % (self.rootConfigPath, storageName, protocol)
res = gConfig.getOptions(protocolConfigPath)
if not res["OK"]:
errStr = "StorageFactory.__getProtocolDetails: Failed to get protocol options."
gLogger.error(errStr, "%s: %s" % (storageName, protocol))
return S_ERROR(errStr)
options = res["Value"]
# We must have certain values internally even if not supplied in CS
protocolDict = {
"Access": "",
"Host": "",
"Path": "",
"Port": "",
"Protocol": "",
"ProtocolName": "",
"SpaceToken": "",
"WSUrl": "",
}
for option in options:
configPath = "%s/%s" % (protocolConfigPath, option)
optionValue = gConfig.getValue(configPath, "")
protocolDict[option] = optionValue
# Now update the local and remote protocol lists.
# A warning will be given if the Access option is not set.
if protocolDict["Access"] == "remote":
self.remoteProtocols.append(protocolDict["ProtocolName"])
elif protocolDict["Access"] == "local":
self.localProtocols.append(protocolDict["ProtocolName"])
else:
errStr = (
"StorageFactory.__getProtocolDetails: The 'Access' option for %s:%s is neither 'local' or 'remote'."
% (storageName, protocol)
)
gLogger.warn(errStr)
# The ProtocolName option must be defined
if not protocolDict["ProtocolName"]:
errStr = "StorageFactory.__getProtocolDetails: 'ProtocolName' option is not defined."
gLogger.error(errStr, "%s: %s" % (storageName, protocol))
return S_ERROR(errStr)
return S_OK(protocolDict)
def syncStorageElementsInit():
ses = gConfig.getSections('/Resources/StorageElements')
if not ses['OK']:
gLogger.error(ses['Message'])
DIRAC.exit(2)
statuses = rsc.getValidStatuses()['Value']
for se in ses['Value']:
opts = gConfig.getOptions('/Resources/StorageElements/%s' %
se)['Value']
statusTypes = ['Read', 'Write', 'Check', 'Remove']
for opt in opts:
if not opt.endswith('Access'):
continue
status = gConfig.getValue('/Resources/StorageElements/%s/%s' %
(se, opt))
if status in ['NotAllowed', 'InActive']:
status = 'Banned'
if not status in statuses:
gLogger.error('%s not a valid status for %s - %s' %
(status, se, statusType))
continue
statusType = opt.replace('Access', '')
statusTypes.remove(statusType)
rsc.modifyElementStatus('StorageElement',
se,
statusType,
status=status,
reason='Init sync')
for sType in statusTypes:
# If there is nothing on the CS, we set the statusType to DEFAULT_STATUS
DEFAULT_STATUS = 'Active'
rsc.modifyElementStatus('StorageElement',
se,
sType,
status=DEFAULT_STATUS,
reason='Default status')
def getAppDependencies(self):
"""
Generate the dependency dictionary
:return: Dict
"""
dependency = {}
fullName = "%s/%s" % (Conf.BASECS, self.__dependencySection)
result = gConfig.getOptions(fullName)
if not result['OK']:
gLogger.error(result['Message'])
return dependency
optionsList = result['Value']
for opName in optionsList:
opVal = gConfig.getValue("%s/%s" % (fullName, opName))
dependency[opName] = opVal
return dependency
def getFTS3Servers(hostOnly=False):
""" get list of FTS3 servers that are in CS
:param bool hostOnly: flag for stripping down the protocol and ports
"""
csPath = cfgPath(gBaseResourcesSection, "FTSEndpoints/FTS3")
# We do it in two times to keep the order
ftsServerNames = gConfig.getOptions(csPath).get('Value', [])
ftsServers = []
for name in ftsServerNames:
serverPath = gConfig.getValue(cfgPath(csPath, name))
if hostOnly:
serverPath = urlparse.urlparse(serverPath).hostname
ftsServers.append(serverPath)
return S_OK(ftsServers)
def getAppDependencies(self):
"""
Generate the dependency dictionary
:return: Dict
"""
dependency = {}
fullName = "%s/%s" % ( Conf.BASECS, self.__dependencySection)
result = gConfig.getOptions( fullName )
if not result[ 'OK' ]:
gLogger.error(result['Message'])
return dependency
optionsList = result[ 'Value' ]
for opName in optionsList:
opVal = gConfig.getValue( "%s/%s" % ( fullName, opName ) )
dependency[opName] = opVal
return dependency
def _getConfigStorageName(self,
storageName,
referenceType,
seConfigPath=SE_CONFIG_PATH):
"""
This gets the name of the storage the configuration service.
If the storage is a reference to another SE the resolution is performed.
:param storageName: is the storage section to check in the CS
:param referenceType: corresponds to an option inside the storage section
:param seConfigPath: the path of the storage section.
It can be /Resources/StorageElements or StorageElementBases
:return: the name of the storage
"""
configPath = "%s/%s" % (seConfigPath, storageName)
res = gConfig.getOptions(configPath)
if not res["OK"]:
errStr = "StorageFactory._getConfigStorageName: Failed to get storage options"
gLogger.error(errStr, res["Message"])
return S_ERROR(errStr)
if not res["Value"]:
errStr = "StorageFactory._getConfigStorageName: Supplied storage doesn't exist."
gLogger.error(errStr, configPath)
return S_ERROR(errStr)
if referenceType in res["Value"]:
configPath = cfgPath(seConfigPath, storageName, referenceType)
referenceName = gConfig.getValue(configPath)
# We first look into the BaseStorageElements section.
# If not, we look into the StorageElements section
# (contrary to BaseSE, it's OK for an Alias to be in the StorageElements section)
result = self._getConfigStorageName(
referenceName, "Alias", seConfigPath=SE_BASE_CONFIG_PATH)
if not result["OK"]:
# Since it is not in the StorageElementBases section, check in the StorageElements section
result = self._getConfigStorageName(
referenceName, "Alias", seConfigPath=SE_CONFIG_PATH)
if not result["OK"]:
return result
resolvedName = result["Value"]
else:
resolvedName = storageName
return S_OK(resolvedName)
def getAppDependencies(self):
"""
Generate the dependency dictionary
:return: Dict
"""
if self.__params.name != 'WebAppDIRAC':
self._loadWebAppCFGFiles(self.__params.name)
dependency = {}
fullName = "%s/%s" % ("/WebApp", self.__dependencySection)
result = gConfig.getOptions(fullName)
if not result['OK']:
gLogger.error(result['Message'])
return dependency
optionsList = result['Value']
for opName in optionsList:
opVal = gConfig.getValue("%s/%s" % (fullName, opName))
dependency[opName] = opVal
return dependency
def getAppDependencies(self):
"""
Generate the dependency dictionary
:return: Dict
"""
if self.__params.name != 'WebAppDIRAC':
self._loadWebAppCFGFiles(self.__params.name)
dependency = {}
fullName = "%s/%s" % ("/WebApp", self.__dependencySection)
result = gConfig.getOptions(fullName)
if not result['OK']:
gLogger.error(result['Message'])
return dependency
optionsList = result['Value']
for opName in optionsList:
opVal = gConfig.getValue("%s/%s" % (fullName, opName))
dependency[opName] = opVal
return dependency
def _getConfigStorageName(self,
storageName,
referenceType,
seConfigPath=SE_CONFIG_PATH):
"""
This gets the name of the storage the configuration service.
If the storage is a reference to another SE the resolution is performed.
:params storageName: is the storage section to check in the CS
:params referenceType: corresponds to an option inside the storage section
:params seConfigPath: the path of the storage section.
It can be /Resources/StorageElements or StorageElementBases
:return: the name of the storage
"""
configPath = '%s/%s' % (seConfigPath, storageName)
res = gConfig.getOptions(configPath)
if not res['OK']:
errStr = "StorageFactory._getConfigStorageName: Failed to get storage options"
gLogger.error(errStr, res['Message'])
return S_ERROR(errStr)
if not res['Value']:
errStr = "StorageFactory._getConfigStorageName: Supplied storage doesn't exist."
gLogger.error(errStr, configPath)
return S_ERROR(errStr)
if referenceType in res['Value']:
configPath = cfgPath(seConfigPath, storageName, referenceType)
referenceName = gConfig.getValue(configPath)
result = self._getConfigStorageName(
referenceName, 'Alias', seConfigPath=SE_BASE_CONFIG_PATH)
if not result['OK']:
# This is for the backward compatibility and to invite developer to move their BaseSE in the correct section
gLogger.warn("Deprecated configuration, you can ignore the error message above."\
" Please move the baseSE in the correct section: ", SE_BASE_CONFIG_PATH)
result = self._getConfigStorageName(
referenceName, 'Alias', seConfigPath=SE_CONFIG_PATH)
if not result['OK']:
return result
resolvedName = result['Value']
else:
resolvedName = storageName
return S_OK(resolvedName)
def __deleteHistory( self , name , state ):
"""
Deleting item from Load and Save history list
Return resulting list
"name" is a string
"state" can be either "Save" or "Load"
"""
gLogger.info( "Running deleteHistory( %s )" % name )
msg = "deleteHistory() for %s@%s" % ( getUsername() , getSelectedGroup() )
opt = "/Website/" + USER_PROFILE_NAME + "/ShowHistory"
history_length = gConfig.getOptions( opt , 5 )
upc = UserProfileClient( "Default" , getRPCClient )
group = str( getSelectedGroup() )
profile_name = USER_PROFILE_NAME + ".History." + state + "." + group
result = upc.retrieveVar( profile_name )
gLogger.info( result )
if not result[ "OK" ]:
if result[ "Message" ].find( "No data" ) < 0 :
gLogger.error( "Result %s: %s" % ( msg , result[ "Message" ] ) )
return S_ERROR( result[ "Message" ] )
gLogger.info( "Result %s: %s" % ( msg , result[ "Message" ] ) )
return S_OK( list() ) # Nothing to delete, return an empty list
else:
result = result[ "Value" ]
if not isinstance( result , list ):
err = "List expected at: %s" % profile_name
gLogger.error( "Result %s: %s" % ( msg , err ) )
return S_ERROR( err )
history = list()
for i in result:
if i.has_key( "name" ) and not i["name"] == name:
history.append( i )
if( len( history ) > history_length ):
history = result[ history_length ]
gLogger.error( "History: %s" % history )
result = upc.storeVar( profile_name , history )
gLogger.info( result )
if not result[ "OK" ]:
gLogger.error( "Result %s: %s" % ( msg , result[ "Message" ] ) )
return S_ERROR( result[ "Message" ] )
gLogger.info( "Result %s: %s" % ( msg , history ) )
return S_OK( history )
def __deleteHistory(self, name, state):
"""
Deleting item from Load and Save history list
Return resulting list
"name" is a string
"state" can be either "Save" or "Load"
"""
gLogger.info("Running deleteHistory( %s )" % name)
msg = "deleteHistory() for %s@%s" % (getUsername(), getSelectedGroup())
opt = "/Website/" + USER_PROFILE_NAME + "/ShowHistory"
history_length = gConfig.getOptions(opt, 5)
upc = UserProfileClient("Default", getRPCClient)
group = str(getSelectedGroup())
profile_name = USER_PROFILE_NAME + ".History." + state + "." + group
result = upc.retrieveVar(profile_name)
gLogger.info(result)
if not result["OK"]:
if result["Message"].find("No data") < 0:
gLogger.error("Result %s: %s" % (msg, result["Message"]))
return S_ERROR(result["Message"])
gLogger.info("Result %s: %s" % (msg, result["Message"]))
return S_OK(list()) # Nothing to delete, return an empty list
else:
result = result["Value"]
if not isinstance(result, list):
err = "List expected at: %s" % profile_name
gLogger.error("Result %s: %s" % (msg, err))
return S_ERROR(err)
history = list()
for i in result:
if i.has_key("name") and not i["name"] == name:
history.append(i)
if (len(history) > history_length):
history = result[history_length]
gLogger.error("History: %s" % history)
result = upc.storeVar(profile_name, history)
gLogger.info(result)
if not result["OK"]:
gLogger.error("Result %s: %s" % (msg, result["Message"]))
return S_ERROR(result["Message"])
gLogger.info("Result %s: %s" % (msg, history))
return S_OK(history)
def _getConfigStorageProtocolDetails( self, storageName, protocol ):
"""
Parse the contents of the protocol block
"""
# First obtain the options that are available
protocolConfigPath = '%s/%s/%s' % ( self.rootConfigPath, storageName, protocol )
res = gConfig.getOptions( protocolConfigPath )
if not res['OK']:
errStr = "StorageFactory.__getProtocolDetails: Failed to get protocol options."
gLogger.error( errStr, "%s: %s" % ( storageName, protocol ) )
return S_ERROR( errStr )
options = res['Value']
# We must have certain values internally even if not supplied in CS
protocolDict = {'Access':'', 'Host':'', 'Path':'', 'Port':'', 'Protocol':'', 'ProtocolName':'', 'SpaceToken':'', 'WSUrl':''}
for option in options:
configPath = '%s/%s' % ( protocolConfigPath, option )
optionValue = gConfig.getValue( configPath, '' )
protocolDict[option] = optionValue
# If the user has requested to use Proxy storage
if self.proxy:
if protocolDict['ProtocolName'] != 'DIP':
protocolDict['ProtocolName'] = 'Proxy'
# Now update the local and remote protocol lists.
# A warning will be given if the Access option is not set.
if protocolDict['Access'] == 'remote':
self.remoteProtocols.append( protocolDict['ProtocolName'] )
elif protocolDict['Access'] == 'local':
self.localProtocols.append( protocolDict['ProtocolName'] )
else:
errStr = "StorageFactory.__getProtocolDetails: The 'Access' option for %s:%s is neither 'local' or 'remote'." % ( storageName, protocol )
gLogger.warn( errStr )
# The ProtocolName option must be defined
if not protocolDict['ProtocolName']:
errStr = "StorageFactory.__getProtocolDetails: 'ProtocolName' option is not defined."
gLogger.error( errStr, "%s: %s" % ( storageName, protocol ) )
return S_ERROR( errStr )
return S_OK( protocolDict )
def __getLaunchpadOpts(self):
options = gConfig.getOptions("/Website/Launchpad/Options", False)
if options and options["OK"]:
options = self.__getDataFromCS()
else:
options = "false"
override = gConfig.getOption("/Website/Launchpad/OptionsOverride")
if override["OK"]:
override = str(override["Value"]).lower()
if override == "true":
pass
else:
override = "false"
else:
override = "false"
separator = gConfig.getOption("/Website/Launchpad/ListSeparator")
if separator["OK"]:
separator = separator["Value"]
else:
separator = "false"
return {"success":"true","result":options,"override":override,"separator":separator}
def __setHistory( self , item , state ):
"""
Insert item to Load or Save history list in first position and checking for
duplications.
Return resulting list
"item" is a dict
"state" should be either "Save" or "Load" but can be any other value
"""
gLogger.info( "Running setHistory( %s , %s )" % ( item , state ) )
msg = "setHistory() for %s@%s" % ( getUsername() , getSelectedGroup() )
opt = "/Website/" + USER_PROFILE_NAME + "/ShowHistory"
history_length = gConfig.getOptions( opt , 5 )
upc = UserProfileClient( "Default" , getRPCClient )
group = str( getSelectedGroup() )
profile_name = USER_PROFILE_NAME + ".History." + state + "." + group
result = upc.retrieveVar( profile_name )
gLogger.info( result )
if not result[ "OK" ]:
if result[ "Message" ].find( "No data" ) < 0 :
gLogger.error( "Result %s: %s" % ( msg , result[ "Message" ] ) )
return S_ERROR( result[ "Message" ] )
history = list()
else:
history = result[ "Value" ]
if not isinstance( history , list ):
err = "List expected at: %s" % profile_name
gLogger.error( "Result %s: %s" % ( msg , err ) )
return S_ERROR( err )
if( len( history ) > history_length ):
history = result[ history_length ]
history.insert( 0 , item )
history = uniqueElements( history )
gLogger.error( "History: %s" % history )
result = upc.storeVar( profile_name , history )
gLogger.info( result )
if not result[ "OK" ]:
gLogger.error( "Result %s: %s" % ( msg , result[ "Message" ] ) )
return S_ERROR( result[ "Message" ] )
gLogger.info( "Result %s: %s" % ( msg , history ) )
return S_OK( history )
def __setHistory(self, item, state):
"""
Insert item to Load or Save history list in first position and checking for
duplications.
Return resulting list
"item" is a dict
"state" should be either "Save" or "Load" but can be any other value
"""
gLogger.info("Running setHistory( %s , %s )" % (item, state))
msg = "setHistory() for %s@%s" % (getUsername(), getSelectedGroup())
opt = "/Website/" + USER_PROFILE_NAME + "/ShowHistory"
history_length = gConfig.getOptions(opt, 5)
upc = UserProfileClient("Default", getRPCClient)
group = str(getSelectedGroup())
profile_name = USER_PROFILE_NAME + ".History." + state + "." + group
result = upc.retrieveVar(profile_name)
gLogger.info(result)
if not result["OK"]:
if result["Message"].find("No data") < 0:
gLogger.error("Result %s: %s" % (msg, result["Message"]))
return S_ERROR(result["Message"])
history = list()
else:
history = result["Value"]
if not isinstance(history, list):
err = "List expected at: %s" % profile_name
gLogger.error("Result %s: %s" % (msg, err))
return S_ERROR(err)
if (len(history) > history_length):
history = result[history_length]
history.insert(0, item)
history = uniqueElements(history)
gLogger.error("History: %s" % history)
result = upc.storeVar(profile_name, history)
gLogger.info(result)
if not result["OK"]:
gLogger.error("Result %s: %s" % (msg, result["Message"]))
return S_ERROR(result["Message"])
gLogger.info("Result %s: %s" % (msg, history))
return S_OK(history)
def syncStorageElementsInit():
ses = gConfig.getSections( '/Resources/StorageElements' )
if not ses[ 'OK' ]:
gLogger.error( ses[ 'Message' ] )
DIRAC.exit( 2 )
statuses = rsc.getValidStatuses()[ 'Value' ]
for se in ses[ 'Value' ]:
opts = gConfig.getOptions( '/Resources/StorageElements/%s' % se )[ 'Value' ]
statusTypes = [ 'Read', 'Write', 'Check', 'Remove' ]
for opt in opts:
if not opt.endswith( 'Access' ):
continue
status = gConfig.getValue( '/Resources/StorageElements/%s/%s' % ( se, opt ) )
if status in [ 'NotAllowed', 'InActive' ]:
status = 'Banned'
if not status in statuses:
gLogger.error( '%s not a valid status for %s - %s' % ( status, se, statusType ) )
continue
statusType = opt.replace( 'Access', '' )
statusTypes.remove( statusType )
rsc.modifyElementStatus( 'StorageElement', se, statusType,
status = status, reason = 'Init sync' )
for sType in statusTypes:
# If there is nothing on the CS, we set the statusType to DEFAULT_STATUS
DEFAULT_STATUS = 'Active'
rsc.modifyElementStatus( 'StorageElement', se, sType,
status = DEFAULT_STATUS, reason = 'Default status' )
def getFTSSites():
FTS = gConfig.getOptions("%s/FTSEndpoints" %g_BaseResourcesSection)
return FTS
def getVOMSAttributes( self, proxy, switch = "all" ):
"""
Return VOMS proxy attributes as list elements if switch="all" (default) OR
return the string prepared to be stored in DB if switch="db" OR
return the string of elements to be used as the option string in voms-proxy-init
if switch="option".
If a given proxy is a grid proxy, then function will return an empty list.
"""
# Get all possible info from voms proxy
result = self.getVOMSProxyInfo( proxy, "all" )
if not result["OK"]:
return S_ERROR( 'Failed to extract info from proxy: %s' % result[ 'Message' ] )
vomsInfoOutput = List.fromChar( result["Value"], "\n" )
#Get a list of known VOMS attributes
validVOMSAttrs = []
result = gConfig.getOptions( "/Registry/VOMS/Mapping" )
if result[ 'OK' ]:
for group in result[ 'Value' ]:
vA = gConfig.getValue( "/Registry/VOMS/Mapping/%s" % group, "" )
if vA and vA not in validVOMSAttrs:
validVOMSAttrs.append( vA )
result = gConfig.getSections( "/Registry/Groups" )
if result[ 'OK' ]:
for group in result[ 'Value' ]:
vA = gConfig.getValue( "/Registry/Groups/%s/VOMSRole" % group, "" )
if vA and vA not in validVOMSAttrs:
validVOMSAttrs.append( vA )
# Parse output of voms-proxy-info command
attributes = []
voName = ''
nickName = ''
for line in vomsInfoOutput:
fields = List.fromChar( line, ":" )
key = fields[0]
value = " ".join( fields[1:] )
if key == "VO":
voName = value
elif key == "attribute":
# Cut off unsupported Capability selection part
if value.find( "nickname" ) == 0:
nickName = "=".join( List.fromChar( value, "=" )[ 1: ] )
else:
value = value.replace( "/Capability=NULL" , "" )
value = value.replace( "/Role=NULL" , "" )
if value and value not in attributes and value in validVOMSAttrs:
attributes.append( value )
# Sorting and joining attributes
if switch == "db":
returnValue = ":".join( attributes )
elif switch == "option":
if len( attributes ) > 1:
returnValue = voName + " -order " + ' -order '.join( attributes )
elif attributes:
returnValue = voName + ":" + attributes[0]
else:
returnValue = voName
elif switch == 'nickname':
returnValue = nickName
elif switch == 'all':
returnValue = attributes
return S_OK( returnValue )
def getCSOptions( opt ): return gConfig.getOptions( "%s/%s" % ( BASECS, opt ) )
def getDestinationSEList(outputSE, site, outputmode='Any'):
""" Evaluate the output SE list from a workflow and return the concrete list
of SEs to upload output data.
"""
# Add output SE defined in the job description
gLogger.info('Resolving workflow output SE description: %s' % outputSE)
# Check if the SE is defined explicitly for the site
prefix = site.split('.')[0]
country = site.split('.')[-1]
# Concrete SE name
result = gConfig.getOptions('/Resources/StorageElements/' + outputSE)
if result['OK']:
gLogger.info('Found concrete SE %s' % outputSE)
return S_OK([outputSE])
# There is an alias defined for this Site
alias_se = gConfig.getValue('/Resources/Sites/%s/%s/AssociatedSEs/%s' % (prefix, site, outputSE), [])
if alias_se:
gLogger.info('Found associated SE for site %s' % (alias_se))
return S_OK(alias_se)
localSEs = getSEsForSite(site)['Value']
gLogger.verbose('Local SE list is: %s' % (localSEs))
groupSEs = gConfig.getValue('/Resources/StorageElementGroups/' + outputSE, [])
gLogger.verbose('Group SE list is: %s' % (groupSEs))
if not groupSEs:
return S_ERROR('Failed to resolve SE ' + outputSE)
if outputmode.lower() == "local":
for se in localSEs:
if se in groupSEs:
gLogger.info('Found eligible local SE: %s' % (se))
return S_OK([se])
#check if country is already one with associated SEs
associatedSE = gConfig.getValue('/Resources/Countries/%s/AssociatedSEs/%s' % (country, outputSE), '')
if associatedSE:
gLogger.info('Found associated SE %s in /Resources/Countries/%s/AssociatedSEs/%s' % (associatedSE, country, outputSE))
return S_OK([associatedSE])
# Final check for country associated SE
count = 0
assignedCountry = country
while count < 10:
gLogger.verbose('Loop count = %s' % (count))
gLogger.verbose("/Resources/Countries/%s/AssignedTo" % assignedCountry)
opt = gConfig.getOption("/Resources/Countries/%s/AssignedTo" % assignedCountry)
if opt['OK'] and opt['Value']:
assignedCountry = opt['Value']
gLogger.verbose('/Resources/Countries/%s/AssociatedSEs' % assignedCountry)
assocCheck = gConfig.getOption('/Resources/Countries/%s/AssociatedSEs' % assignedCountry)
if assocCheck['OK'] and assocCheck['Value']:
break
count += 1
if not assignedCountry:
return S_ERROR('Could not determine associated SE list for %s' % country)
alias_se = gConfig.getValue('/Resources/Countries/%s/AssociatedSEs/%s' % (assignedCountry, outputSE), [])
if alias_se:
gLogger.info('Found alias SE for site: %s' % alias_se)
return S_OK(alias_se)
else:
gLogger.error('Could not establish alias SE for country %s from section: /Resources/Countries/%s/AssociatedSEs/%s' % (country, assignedCountry, outputSE))
return S_ERROR('Failed to resolve SE ' + outputSE)
# For collective Any and All modes return the whole group
# Make sure that local SEs are passing first
newSEList = []
for se in groupSEs:
if se in localSEs:
newSEList.append(se)
uniqueSEs = uniqueElements(newSEList + groupSEs)
gLogger.verbose('Found unique SEs: %s' % (uniqueSEs))
return S_OK(uniqueSEs)
