def export_getSiteSummarySelectors(self):
""" Get all the distinct selector values for the site summary web portal page
"""
resultDict = {}
statusList = ['Good', 'Fair', 'Poor', 'Bad', 'Idle']
resultDict['Status'] = statusList
maskStatus = ['Active', 'Banned', 'NoMask', 'Reduced']
resultDict['MaskStatus'] = maskStatus
resources = Resources()
result = resources.getSites()
if not result['OK']:
return result
siteList = result['Value']
countryList = []
for site in siteList:
if site.find('.') != -1:
country = site.split('.')[1]
country = country.lower()
if country not in countryList:
countryList.append(country)
countryList.sort()
resultDict['Country'] = countryList
siteList.sort()
resultDict['Site'] = siteList
return S_OK(resultDict)
def __init__(self, catalogs=[], vo=None):
""" Default constructor
"""
self.valid = True
self.timeout = 180
self.readCatalogs = []
self.writeCatalogs = []
self.vo = vo
if not vo:
result = getVOfromProxyGroup()
if not result['OK']:
return result
self.vo = result['Value']
self.opHelper = Operations(vo=self.vo)
self.reHelper = Resources(vo=self.vo)
if type(catalogs) in types.StringTypes:
catalogs = [catalogs]
if catalogs:
res = self._getSelectedCatalogs(catalogs)
else:
res = self._getCatalogs()
if not res['OK']:
self.valid = False
elif (len(self.readCatalogs) == 0) and (len(self.writeCatalogs) == 0):
self.valid = False
def export_getSiteSummarySelectors(self):
""" Get all the distinct selector values for the site summary web portal page
"""
resultDict = {}
statusList = ['Good','Fair','Poor','Bad','Idle']
resultDict['Status'] = statusList
maskStatus = ['Active','Banned','NoMask','Reduced']
resultDict['MaskStatus'] = maskStatus
resources = Resources()
result = resources.getSites()
if not result['OK']:
return result
siteList = result['Value']
countryList = []
for site in siteList:
if site.find('.') != -1:
country = site.split('.')[1]
country = country.lower()
if country not in countryList:
countryList.append(country)
countryList.sort()
resultDict['Country'] = countryList
siteList.sort()
resultDict['Site'] = siteList
return S_OK(resultDict)
def __init__(self, useProxy=False, vo=None):
self.valid = True
self.proxy = False
self.proxy = useProxy
self.resourceStatus = ResourceStatus()
self.resourcesHelper = Resources(vo=vo)
def getSiteCEMapping():
""" Returns a dictionary of all sites and their CEs as a list, e.g.
{'LCG.CERN.ch':['ce101.cern.ch',...]}
If gridName is specified, result is restricted to that Grid type.
"""
siteCEMapping = {}
resourceHelper = Resources()
result = resourceHelper.getEligibleSites()
if not result['OK']:
return result
sites = result['Value']
for site in sites:
result = resourceHelper.getEligibleResources( 'Computing', {'Site':site} )
if not result['OK']:
continue
ceList = result['Value']
result = getSiteFullNames( site )
if not result['OK']:
continue
for sName in result['Value']:
siteCEMapping[sName] = ceList
return S_OK( siteCEMapping )
def getSiteCEMapping():
""" Returns a dictionary of all sites and their CEs as a list, e.g.
{'LCG.CERN.ch':['ce101.cern.ch',...]}
If gridName is specified, result is restricted to that Grid type.
"""
siteCEMapping = {}
resourceHelper = Resources()
result = resourceHelper.getEligibleSites()
if not result['OK']:
return result
sites = result['Value']
for site in sites:
result = resourceHelper.getEligibleResources('Computing',
{'Site': site})
if not result['OK']:
continue
ceList = result['Value']
result = getSiteFullNames(site)
if not result['OK']:
continue
for sName in result['Value']:
siteCEMapping[sName] = ceList
return S_OK(siteCEMapping)
def createCatalog(self,
catalogName,
useProxy=False,
vo=None,
catalogConfig={}):
""" Create a file catalog object from its name and CS description
"""
if useProxy:
catalog = FileCatalogProxyClient(catalogName)
return S_OK(catalog)
# get the CS description first
catConfig = catalogConfig
if not catConfig:
if not vo:
result = getVOfromProxyGroup()
if not result['OK']:
return result
vo = result['Value']
reHelper = Resources(vo=vo)
result = reHelper.getCatalogOptionsDict(catalogName)
if not result['OK']:
return result
catConfig = result['Value']
catalogType = catConfig.get('CatalogType', catalogName)
catalogURL = catConfig.get('CatalogURL', '')
self.log.verbose('Creating %s client' % catalogName)
moduleRootPaths = getInstalledExtensions()
for moduleRootPath in moduleRootPaths:
gLogger.verbose("Trying to load from root path %s" %
moduleRootPath)
#moduleFile = os.path.join( rootPath, moduleRootPath, "Resources", "Catalog", "%sClient.py" % catalogType )
#gLogger.verbose( "Looking for file %s" % moduleFile )
#if not os.path.isfile( moduleFile ):
# continue
try:
# This enforces the convention that the plug in must be named after the file catalog
moduleName = "%sClient" % (catalogType)
catalogModule = __import__(
'%s.Resources.Catalog.%s' % (moduleRootPath, moduleName),
globals(), locals(), [moduleName])
except ImportError, x:
if "No module" in str(x):
gLogger.debug('Catalog module %s not found in %s' %
(catalogType, moduleRootPath))
else:
errStr = "Failed attempt to import %s from the path %s: %s" % (
catalogType, moduleRootPath, x)
gLogger.error(errStr)
continue
except Exception, x:
errStr = "Failed attempt to import %s from the path %s: %s" % (
catalogType, moduleRootPath, x)
gLogger.error(errStr)
continue
def createCatalog(self,
catalogName,
useProxy=False,
vo=None,
catalogConfig={}):
""" Create a file catalog object from its name and CS description
"""
if useProxy:
catalog = FileCatalogProxyClient(catalogName)
return S_OK(catalog)
# get the CS description first
catConfig = catalogConfig
if not catConfig:
if not vo:
result = getVOfromProxyGroup()
if not result['OK']:
return result
vo = result['Value']
reHelper = Resources(vo=vo)
result = reHelper.getCatalogOptionsDict(catalogName)
if not result['OK']:
return result
catConfig = result['Value']
catalogType = catConfig.get('CatalogType', catalogName)
catalogURL = catConfig.get('CatalogURL', '')
self.log.verbose('Creating %s client' % catalogName)
objectLoader = ObjectLoader()
result = objectLoader.loadObject(
'Resources.Catalog.%sClient' % catalogType, catalogType + 'Client')
if not result['OK']:
gLogger.error('Failed to load catalog object: %s' %
result['Message'])
return result
catalogClass = result['Value']
try:
if catalogType in ['LcgFileCatalogCombined', 'LcgFileCatalog']:
# The LFC special case
infoSys = catConfig.get('LcgGfalInfosys', '')
host = catConfig.get('MasterHost', '')
catalog = catalogClass(infoSys, host)
else:
if catalogURL:
catalog = catalogClass(url=catalogURL)
else:
catalog = catalogClass()
self.log.debug('Loaded module %sClient' % catalogType)
return S_OK(catalog)
except Exception, x:
errStr = "Failed to instantiate %s()" % (catalogType)
gLogger.exception(errStr, lException=x)
return S_ERROR(errStr)
def getCEsForSite( siteName ):
""" Given a DIRAC site name this method returns a list of corresponding CEs.
"""
resourceHelper = Resources()
result = resourceHelper.getEligibleResources( 'Computing', {'Site':siteName} )
if not result['OK']:
return result
ceList = result['Value']
return S_OK( ceList )
def getCEsForSite(siteName):
""" Given a DIRAC site name this method returns a list of corresponding CEs.
"""
resourceHelper = Resources()
result = resourceHelper.getEligibleResources('Computing',
{'Site': siteName})
if not result['OK']:
return result
ceList = result['Value']
return S_OK(ceList)
def getQueueInfo(ceUniqueID):
"""
Extract information from full CE Name including associate DIRAC Site
"""
try:
subClusterUniqueID = ceUniqueID.split('/')[0].split(':')[0]
queueID = ceUniqueID.split('/')[1]
except:
return S_ERROR('Wrong full queue Name')
result = getSiteForCE(subClusterUniqueID)
if not result['OK']:
return result
diracSiteName = result['Value']
if not diracSiteName:
return S_ERROR('Can not find corresponding Site in CS')
resourceHelper = Resources()
result = getSiteName(diracSiteName)
site = result['Value']
domain = result.get('Domain', 'Unknonw')
country = result.get('Country', 'xx')
result = resourceHelper.getQueueOptionsDict(site, subClusterUniqueID,
queueID)
if not result['OK']:
return result
queueDict = result['Value']
maxCPUTime = queueDict.get('maxCPUTime', 0)
SI00 = queueDict.get('SI00', 0)
if not maxCPUTime or not SI00:
result = resourceHelper.getComputingOptionsDict(
site, subClusterUniqueID)
if not result['OK']:
return result
ceDict = result['Value']
if not maxCPUTime:
maxCPUTime = ceDict.get('maxCPUTime', 0)
if not SI00:
SI00 = ceDict.get('SI00', 0)
resultDict = {
'SubClusterUniqueID': subClusterUniqueID,
'QueueID': queueID,
'SiteName': diracSiteName,
'Domain': domain,
'Country': country,
'maxCPUTime': maxCPUTime,
'SI00': SI00
}
return S_OK(resultDict)
def __checkSEs( self, seList ):
resources = Resources()
res = resources.getEligibleResources( 'Storage' )
if not res['OK']:
return self._errorReport( res, 'Failed to get possible StorageElements' )
missing = []
for se in seList:
if not se in res['Value']:
gLogger.error( "StorageElement %s is not known" % se )
missing.append( se )
if missing:
return S_ERROR( "%d StorageElements not known" % len( missing ) )
return S_OK()
def createCatalog( self, catalogName, useProxy = False, vo = None, catalogConfig = {} ):
""" Create a file catalog object from its name and CS description
"""
if useProxy:
catalog = FileCatalogProxyClient( catalogName )
return S_OK( catalog )
# get the CS description first
catConfig = catalogConfig
if not catConfig:
if not vo:
result = getVOfromProxyGroup()
if not result['OK']:
return result
vo = result['Value']
reHelper = Resources( vo = vo )
result = reHelper.getCatalogOptionsDict( catalogName )
if not result['OK']:
return result
catConfig = result['Value']
catalogType = catConfig.get('CatalogType',catalogName)
catalogURL = catConfig.get('CatalogURL','')
self.log.verbose( 'Creating %s client' % catalogName )
objectLoader = ObjectLoader()
result = objectLoader.loadObject( 'Resources.Catalog.%sClient' % catalogType, catalogType+'Client' )
if not result['OK']:
gLogger.error( 'Failed to load catalog object: %s' % result['Message'] )
return result
catalogClass = result['Value']
try:
if catalogType in ['LcgFileCatalogCombined','LcgFileCatalog']:
# The LFC special case
infoSys = catConfig.get('LcgGfalInfosys','')
host = catConfig.get('MasterHost','')
catalog = catalogClass( infoSys, host )
else:
if catalogURL:
catalog = catalogClass( url = catalogURL )
else:
catalog = catalogClass()
self.log.debug('Loaded module %sClient' % catalogType )
return S_OK( catalog )
except Exception, x:
errStr = "Failed to instantiate %s()" % ( catalogType )
gLogger.exception( errStr, lException = x )
return S_ERROR( errStr )
def getQueueInfo( ceUniqueID ):
"""
Extract information from full CE Name including associate DIRAC Site
"""
try:
subClusterUniqueID = ceUniqueID.split( '/' )[0].split( ':' )[0]
queueID = ceUniqueID.split( '/' )[1]
except:
return S_ERROR( 'Wrong full queue Name' )
result = getSiteForCE( subClusterUniqueID )
if not result['OK']:
return result
diracSiteName = result['Value']
if not diracSiteName:
return S_ERROR( 'Can not find corresponding Site in CS' )
resourceHelper = Resources()
result = getSiteName( diracSiteName )
site = result['Value']
domain = result.get( 'Domain', 'Unknonw' )
country = result.get( 'Country', 'xx' )
result = resourceHelper.getQueueOptionsDict( site, subClusterUniqueID, queueID )
if not result['OK']:
return result
queueDict = result['Value']
maxCPUTime = queueDict.get( 'maxCPUTime', 0 )
SI00 = queueDict.get( 'SI00', 0 )
if not maxCPUTime or not SI00:
result = resourceHelper.getComputingOptionsDict( site, subClusterUniqueID )
if not result['OK']:
return result
ceDict = result['Value']
if not maxCPUTime:
maxCPUTime = ceDict.get( 'maxCPUTime', 0 )
if not SI00:
SI00 = ceDict.get( 'SI00', 0 )
resultDict = { 'SubClusterUniqueID': subClusterUniqueID,
'QueueID': queueID,
'SiteName': diracSiteName,
'Domain': domain,
'Country': country,
'maxCPUTime': maxCPUTime,
'SI00': SI00 }
return S_OK( resultDict )
def __init__(self, useProxy=False, vo=None):
self.valid = True
self.proxy = False
self.proxy = useProxy
self.resourceStatus = ResourceStatus()
self.resourcesHelper = Resources(vo=vo)
def initialize(self):
self.replicaManager = ReplicaManager()
#self.stagerClient = StorageManagerClient()
self.dataIntegrityClient = DataIntegrityClient()
self.storageDB = StorageManagementDB()
# pin lifetime = 1 day
self.pinLifetime = self.am_getOption('PinLifetime', THROTTLING_TIME)
# Resources helper
self.resources = Resources()
# This sets the Default Proxy to used as that defined under
# /Operations/Shifter/DataManager
# the shifterProxy option in the Configuration can be used to change this default.
self.am_setOption('shifterProxy', 'DataManager')
return S_OK()
def __init__(self):
"""
Constructor.
examples:
>>> s = Synchronizer()
"""
self.log = gLogger.getSubLogger(self.__class__.__name__)
self.operations = Operations()
self.resources = Resources()
self.rStatus = ResourceStatusClient.ResourceStatusClient()
self.rssConfig = RssConfiguration()
self.diracAdmin = DiracAdmin()
def __init__(self, database=None):
self.db = None
if database is not None:
self.setDatabase(database)
self.lock = threading.Lock()
self.seUpdatePeriod = 600
self.resourcesHelper = Resources()
self._refreshSEs()
def getCESiteMapping():
""" Returns a dictionary of all CEs and their associated site, e.g.
{'ce101.cern.ch':'LCG.CERN.ch', ...]}
"""
ceSiteMapping = {}
resourceHelper = Resources()
result = resourceHelper.getEligibleResources( 'Computing' )
if not result['OK']:
return result
ceList = result['Value']
for ce in ceList:
result = getSiteForCE( ce )
if not result['OK']:
continue
site = result['Value']
ceSiteMapping[ce] = site
return S_OK( ceSiteMapping )
def getCESiteMapping():
""" Returns a dictionary of all CEs and their associated site, e.g.
{'ce101.cern.ch':'LCG.CERN.ch', ...]}
"""
ceSiteMapping = {}
resourceHelper = Resources()
result = resourceHelper.getEligibleResources('Computing')
if not result['OK']:
return result
ceList = result['Value']
for ce in ceList:
result = getSiteForCE(ce)
if not result['OK']:
continue
site = result['Value']
ceSiteMapping[ce] = site
return S_OK(ceSiteMapping)
def getSESiteMapping( gridName = '' ):
""" Returns a dictionary of all SEs and their associated site(s), e.g.
{'CERN-RAW':'LCG.CERN.ch','CERN-RDST':'LCG.CERN.ch',...]}
Although normally one site exists for a given SE, it is possible over all
Grid types to have multiple entries.
If gridName is specified, result is restricted to that Grid type.
"""
seSiteMapping = {}
resourceHelper = Resources()
result = resourceHelper.getEligibleResources( 'Storage' )
if not result['OK']:
return result
seList = result['Value']
for se in seList:
result = getSitesForSE( se )
if not result['OK']:
continue
site = result['Value']
seSiteMapping[se] = site
return S_OK( seSiteMapping )
def getSESiteMapping(gridName=''):
""" Returns a dictionary of all SEs and their associated site(s), e.g.
{'CERN-RAW':'LCG.CERN.ch','CERN-RDST':'LCG.CERN.ch',...]}
Although normally one site exists for a given SE, it is possible over all
Grid types to have multiple entries.
If gridName is specified, result is restricted to that Grid type.
"""
seSiteMapping = {}
resourceHelper = Resources()
result = resourceHelper.getEligibleResources('Storage')
if not result['OK']:
return result
seList = result['Value']
for se in seList:
result = getSitesForSE(se)
if not result['OK']:
continue
site = result['Value']
seSiteMapping[se] = site
return S_OK(seSiteMapping)
def getSiteSEMapping():
""" Returns a dictionary of all sites and their localSEs as a list, e.g.
{'LCG.CERN.ch':['CERN-RAW','CERN-RDST',...]}
If gridName is specified, result is restricted to that Grid type.
"""
siteSEMapping = {}
resourceHelper = Resources()
result = resourceHelper.getEligibleSites()
if not result['OK']:
return result
sites = result['Value']
for site in sites:
result = resourceHelper.getEligibleResources('Storage', {'Site': site})
if not result['OK']:
continue
seList = result['Value']
result = getSiteFullNames(site)
if not result['OK']:
continue
for sName in result['Value']:
siteSEMapping[sName] = seList
# Add Sites from the SiteToLocalSEMapping in the CS
opsHelper = Operations()
result = opsHelper.getSiteMapping('Storage', 'LocalSE')
if result['OK']:
mapping = result['Value']
for site in mapping:
if site not in siteSEMapping:
siteSEMapping[site] = mapping[site]
else:
for se in mapping[site]:
if se not in siteSEMapping[site]:
siteSEMapping[site].append(se)
return S_OK(siteSEMapping)
def getSiteSEMapping():
""" Returns a dictionary of all sites and their localSEs as a list, e.g.
{'LCG.CERN.ch':['CERN-RAW','CERN-RDST',...]}
If gridName is specified, result is restricted to that Grid type.
"""
siteSEMapping = {}
resourceHelper = Resources()
result = resourceHelper.getEligibleSites()
if not result['OK']:
return result
sites = result['Value']
for site in sites:
result = resourceHelper.getEligibleResources( 'Storage', {'Site':site} )
if not result['OK']:
continue
seList = result['Value']
result = getSiteFullNames( site )
if not result['OK']:
continue
for sName in result['Value']:
siteSEMapping[sName] = seList
# Add Sites from the SiteToLocalSEMapping in the CS
opsHelper = Operations()
result = opsHelper.getSiteMapping( 'Storage', 'LocalSE' )
if result['OK']:
mapping = result['Value']
for site in mapping:
if site not in siteSEMapping:
siteSEMapping[site] = mapping[site]
else:
for se in mapping[site]:
if se not in siteSEMapping[site]:
siteSEMapping[site].append( se )
return S_OK( siteSEMapping )
def __init__(self, args=None, clients=None):
super(DowntimeCommand, self).__init__(args, clients)
self.resources = Resources()
if 'GOCDBClient' in self.apis:
self.gClient = self.apis['GOCDBClient']
else:
self.gClient = GOCDBClient()
if 'ResourceManagementClient' in self.apis:
self.rmClient = self.apis['ResourceManagementClient']
else:
self.rmClient = ResourceManagementClient()
def __init__( self ):
"""
Constructor.
examples:
>>> s = Synchronizer()
"""
self.log = gLogger.getSubLogger( self.__class__.__name__ )
self.operations = Operations()
self.resources = Resources()
self.rStatus = ResourceStatusClient.ResourceStatusClient()
self.rssConfig = RssConfiguration()
self.diracAdmin = DiracAdmin()
def initialize( self ):
self.replicaManager = ReplicaManager()
#self.stagerClient = StorageManagerClient()
self.dataIntegrityClient = DataIntegrityClient()
self.storageDB = StorageManagementDB()
# pin lifetime = 1 day
self.pinLifetime = self.am_getOption( 'PinLifetime', THROTTLING_TIME )
# Resources helper
self.resources = Resources()
# This sets the Default Proxy to used as that defined under
# /Operations/Shifter/DataManager
# the shifterProxy option in the Configuration can be used to change this default.
self.am_setOption( 'shifterProxy', 'DataManager' )
return S_OK()
def export_getSiteMaskSummary(self):
""" Get the mask status for all the configured sites
"""
# Get all the configured site names
result = Resources().getSites()
if not result['OK']:
return result
sites = result['Value']
# Get the current mask status
result = jobDB.getSiteMaskStatus()
siteDict = result['Value']
for site in sites:
if site not in siteDict:
siteDict[site] = 'Unknown'
return S_OK(siteDict)
class InputDataAgent(OptimizerModule):
"""
The specific Optimizer must provide the following methods:
- initializeOptimizer() before each execution cycle
- checkJob() - the main method called for each job
"""
#############################################################################
def initializeOptimizer(self):
"""Initialize specific parameters for JobSanityAgent.
"""
self.failedMinorStatus = self.am_getOption('/FailedJobStatus',
'Input Data Not Available')
#this will ignore failover SE files
self.checkFileMetadata = self.am_getOption('CheckFileMetadata', True)
#Define the shifter proxy needed
# This sets the Default Proxy to used as that defined under
# /Operations/Shifter/ProductionManager
# the shifterProxy option in the Configuration can be used to change this default.
self.am_setOption('shifterProxy', 'ProductionManager')
try:
self.replicaManager = ReplicaManager()
except Exception, e:
msg = 'Failed to create ReplicaManager'
self.log.exception(msg)
return S_ERROR(msg + str(e))
self.resourceStatus = ResourceStatus()
self.resourcesHelper = Resources()
self.seToSiteMapping = {}
self.lastCScheck = 0
self.cacheLength = 600
return S_OK()
def initSEs():
'''
Initializes SEs statuses taking their values from the CS.
'''
subLogger.info('Initializing SEs')
resources = Resources()
ses = resources.getEligibleStorageElements()
if not ses['OK']:
return ses
ses = ses['Value']
statuses = StateMachine.RSSMachine(None).getStates()
statusTypes = RssConfiguration.RssConfiguration().getConfigStatusType(
'StorageElement')
reason = 'dirac-rss-sync'
subLogger.debug(statuses)
subLogger.debug(statusTypes)
rssClient = ResourceStatusClient.ResourceStatusClient()
for se in ses:
subLogger.debug(se)
#opts = gConfig.getOptionsDict( '/Resources/StorageElements/%s' % se )
opts = resources.getStorageElementOptionsDict(se)
if not opts['OK']:
subLogger.warn(opts['Message'])
continue
opts = opts['Value']
subLogger.debug(opts)
# We copy the list into a new object to remove items INSIDE the loop !
statusTypesList = statusTypes[:]
for statusType, status in opts.iteritems():
#Sanity check...
if not statusType in statusTypesList:
continue
#Transforms statuses to RSS terms
if status in ('NotAllowed', 'InActive'):
status = 'Banned'
if not status in statuses:
subLogger.error('%s not a valid status for %s - %s' %
(status, se, statusType))
continue
# We remove from the backtracking
statusTypesList.remove(statusType)
subLogger.debug([se, statusType, status, reason])
result = rssClient.modifyStatusElement('Resource',
'Status',
name=se,
statusType=statusType,
status=status,
reason=reason)
if not result['OK']:
subLogger.error('Failed to modify')
subLogger.error(result['Message'])
continue
#Backtracking: statusTypes not present on CS
for statusType in statusTypesList:
result = rssClient.modifyStatusElement('Resource',
'Status',
name=se,
statusType=statusType,
status=DEFAULT_STATUS,
reason=reason)
if not result['OK']:
subLogger.error('Error in backtracking for %s,%s,%s' %
(se, statusType, status))
subLogger.error(result['Message'])
return S_OK()
return DIRAC.S_OK()
ceFlag = False
def setCEFlag( args ):
global ceFlag
ceFlag = True
return DIRAC.S_OK()
Script.registerSwitch( "V:", "vo=", "choose resources eligible for the given VO", setVO )
Script.registerSwitch( "S", "se", "display storage element information", setSEFlag )
Script.registerSwitch( "C", "ce", "display computing element information", setSEFlag )
Script.setUsageMessage('\n'.join( [ __doc__.split( '\n' )[1],
'Usage:',
' %s [option|cfgfile] ...' % Script.scriptName, ] ) )
Script.parseCommandLine()
from DIRAC.ConfigurationSystem.Client.Helpers.Resources import Resources, getSites
resources = Resources ( vo = 'biomed' )
result = resources.getEligibleSites()
if not result['OK']:
print "ERROR:", result['Message']
siteList = [ resources.getSiteFullName( site )['Value'] for site in result['Value'] ]
print siteList
class StageRequestAgent(AgentModule):
def initialize(self):
self.replicaManager = ReplicaManager()
#self.stagerClient = StorageManagerClient()
self.dataIntegrityClient = DataIntegrityClient()
self.storageDB = StorageManagementDB()
# pin lifetime = 1 day
self.pinLifetime = self.am_getOption('PinLifetime', THROTTLING_TIME)
# Resources helper
self.resources = Resources()
# This sets the Default Proxy to used as that defined under
# /Operations/Shifter/DataManager
# the shifterProxy option in the Configuration can be used to change this default.
self.am_setOption('shifterProxy', 'DataManager')
return S_OK()
def execute(self):
# Get the current submitted stage space and the amount of pinned space for each storage element
res = self.getStorageUsage()
if not res['OK']:
return res
return self.submitStageRequests()
def getStorageUsage(self):
""" Fill the current Status of the SE Caches from the DB
"""
self.storageElementCache = {}
res = self.storageDB.getSubmittedStagePins()
if not res['OK']:
gLogger.fatal(
"StageRequest.getStorageUsage: Failed to obtain submitted requests from StorageManagementDB.",
res['Message'])
return res
self.storageElementUsage = res['Value']
if self.storageElementUsage:
gLogger.info(
"StageRequest.getStorageUsage: Active stage/pin requests found at the following sites:"
)
for storageElement in sortList(self.storageElementUsage.keys()):
seDict = self.storageElementUsage[storageElement]
# Convert to GB for printout
seDict['TotalSize'] = seDict['TotalSize'] / (1000 * 1000 *
1000.0)
gLogger.info(
"StageRequest.getStorageUsage: %s: %s replicas with a size of %.3f GB."
% (storageElement.ljust(15), str(
seDict['Replicas']).rjust(6), seDict['TotalSize']))
if not self.storageElementUsage:
gLogger.info(
"StageRequest.getStorageUsage: No active stage/pin requests found."
)
return S_OK()
def submitStageRequests(self):
""" This manages the following transitions of the Replicas
* Waiting -> Offline (if the file is not found Cached)
* Waiting -> StageSubmitted (if the file is found Cached)
* Offline -> StageSubmitted (if there are not more Waiting replicas)
"""
# Retry Replicas that have not been Staged in a previous attempt
res = self._getMissingReplicas()
if not res['OK']:
gLogger.fatal(
"StageRequest.submitStageRequests: Failed to get replicas from StorageManagementDB.",
res['Message'])
return res
seReplicas = res['Value']['SEReplicas']
allReplicaInfo = res['Value']['AllReplicaInfo']
if seReplicas:
gLogger.info(
"StageRequest.submitStageRequests: Completing partially Staged Tasks"
)
for storageElement, seReplicaIDs in seReplicas.items():
gLogger.debug('Staging at %s:' % storageElement, seReplicaIDs)
self._issuePrestageRequests(storageElement, seReplicaIDs,
allReplicaInfo)
# Check Waiting Replicas and select those found Online and all other Replicas from the same Tasks
res = self._getOnlineReplicas()
if not res['OK']:
gLogger.fatal(
"StageRequest.submitStageRequests: Failed to get replicas from StorageManagementDB.",
res['Message'])
return res
seReplicas = res['Value']['SEReplicas']
allReplicaInfo = res['Value']['AllReplicaInfo']
# Check Offline Replicas that fit in the Cache and all other Replicas from the same Tasks
res = self._getOfflineReplicas()
if not res['OK']:
gLogger.fatal(
"StageRequest.submitStageRequests: Failed to get replicas from StorageManagementDB.",
res['Message'])
return res
# Merge info from both results
for storageElement, seReplicaIDs in res['Value']['SEReplicas'].items():
if storageElement not in seReplicas:
seReplicas[storageElement] = seReplicaIDs
else:
for replicaID in seReplicaIDs:
if replicaID not in seReplicas[storageElement]:
seReplicas[storageElement].append(replicaID)
allReplicaInfo.update(res['Value']['AllReplicaInfo'])
gLogger.info(
"StageRequest.submitStageRequests: Obtained %s replicas for staging."
% len(allReplicaInfo))
for storageElement, seReplicaIDs in seReplicas.items():
gLogger.debug('Staging at %s:' % storageElement, seReplicaIDs)
self._issuePrestageRequests(storageElement, seReplicaIDs,
allReplicaInfo)
return S_OK()
def _getMissingReplicas(self):
""" This recovers Replicas that were not Staged on a previous attempt (the stage request failed or timed out),
while other Replicas of the same task are already Staged. If left behind they can produce a deadlock.
All SEs are considered, even if their Cache is full
"""
# Get Replicas that are in Staged/StageSubmitted
gLogger.info(
'StageRequest._getMissingReplicas: Checking Staged Replicas')
res = self.__getStagedReplicas()
if not res['OK']:
gLogger.fatal(
"StageRequest._getMissingReplicas: Failed to get replicas from StorageManagementDB.",
res['Message'])
return res
seReplicas = {}
allReplicaInfo = res['Value']['AllReplicaInfo']
replicasToStage = []
for storageElement, seReplicaIDs in res['Value']['SEReplicas'].items():
# Consider all SEs
replicasToStage.extend(seReplicaIDs)
# Get Replicas from the same Tasks as those selected
res = self.__addAssociatedReplicas(replicasToStage, seReplicas,
allReplicaInfo)
if not res['OK']:
gLogger.fatal(
"StageRequest._getMissingReplicas: Failed to get associated Replicas.",
res['Message'])
return res
def _getOnlineReplicas(self):
""" This manages the transition
* Waiting -> Offline (if the file is not found Cached)
and returns the list of Cached Replicas for which the pin time has to be extended
SEs for which the cache is currently full are not considered
"""
# Get all Replicas in Waiting Status associated to Staging Tasks
gLogger.verbose(
'StageRequest._getOnlineReplicas: Checking Online Replicas to be handled'
)
res = self.__getWaitingReplicas()
if not res['OK']:
gLogger.fatal(
"StageRequest._getOnlineReplicas: Failed to get replicas from StorageManagementDB.",
res['Message'])
return res
seReplicas = {}
allReplicaInfo = res['Value']['AllReplicaInfo']
if not len(allReplicaInfo):
gLogger.info(
"StageRequest._getOnlineReplicas: There were no Waiting replicas found"
)
return res
gLogger.info(
"StageRequest._getOnlineReplicas: Obtained %s replicas Waiting for staging."
% len(allReplicaInfo))
replicasToStage = []
for storageElement, seReplicaIDs in res['Value']['SEReplicas'].items():
if not self.__usage(storageElement) < self.__cache(storageElement):
gLogger.info(
'StageRequest._getOnlineReplicas: Skipping %s, current usage above limit ( %s GB )'
% (storageElement, self.__cache(storageElement)))
# Do not consider those SE that have the Cache full
continue
# Check if the Replica Metadata is OK and find out if they are Online or Offline
res = self.__checkIntegrity(storageElement, seReplicaIDs,
allReplicaInfo)
if not res['OK']:
gLogger.error(
'StageRequest._getOnlineReplicas: Failed to check Replica Metadata',
'(%s): %s' % (storageElement, res['Message']))
else:
# keep only Online Replicas
seReplicas[storageElement] = res['Value']['Online']
replicasToStage.extend(res['Value']['Online'])
# Get Replicas from the same Tasks as those selected
res = self.__addAssociatedReplicas(replicasToStage, seReplicas,
allReplicaInfo)
if not res['OK']:
gLogger.fatal(
"StageRequest._getOnlineReplicas: Failed to get associated Replicas.",
res['Message'])
return res
def _getOfflineReplicas(self):
""" This checks Replicas in Offline status
and returns the list of Replicas to be Staged
SEs for which the cache is currently full are not considered
"""
# Get all Replicas in Waiting Status associated to Staging Tasks
gLogger.verbose(
'StageRequest._getOfflineReplicas: Checking Offline Replicas to be handled'
)
res = self.__getOfflineReplicas()
if not res['OK']:
gLogger.fatal(
"StageRequest._getOfflineReplicas: Failed to get replicas from StorageManagementDB.",
res['Message'])
return res
seReplicas = {}
allReplicaInfo = res['Value']['AllReplicaInfo']
if not len(allReplicaInfo):
gLogger.info(
"StageRequest._getOfflineReplicas: There were no Offline replicas found"
)
return res
gLogger.info(
"StageRequest._getOfflineReplicas: Obtained %s replicas Offline for staging."
% len(allReplicaInfo))
replicasToStage = []
for storageElement, seReplicaIDs in res['Value']['SEReplicas'].items():
if not self.__usage(storageElement) < self.__cache(storageElement):
gLogger.info(
'StageRequest._getOfflineReplicas: Skipping %s, current usage above limit ( %s GB )'
% (storageElement, self.__cache(storageElement)))
# Do not consider those SE that have the Cache full
continue
seReplicas[storageElement] = []
for replicaID in sorted(seReplicaIDs):
seReplicas[storageElement].append(replicaID)
replicasToStage.append(replicaID)
self.__add(storageElement, allReplicaInfo[replicaID]['Size'])
if not self.__usage(storageElement) < self.__cache(
storageElement):
# Stop adding Replicas when the cache is full
break
# Get Replicas from the same Tasks as those selected
res = self.__addAssociatedReplicas(replicasToStage, seReplicas,
allReplicaInfo)
if not res['OK']:
gLogger.fatal(
"StageRequest._getOfflineReplicas: Failed to get associated Replicas.",
res['Message'])
return res
def __usage(self, storageElement):
""" Retrieve current usage of SE
"""
if not storageElement in self.storageElementUsage:
self.storageElementUsage[storageElement] = {'TotalSize': 0.}
return self.storageElementUsage[storageElement]['TotalSize']
def __cache(self, storageElement):
""" Retrieve cache size for SE
"""
if not storageElement in self.storageElementCache:
diskCache = self.resources.getStorageElementValue(
storageElement, 'DiskCacheTB', 1.)
self.storageElementCache[
storageElement] = diskCache * 1000. / THROTTLING_STEPS
return self.storageElementCache[storageElement]
def __add(self, storageElement, size):
""" Add size (in bytes) to current usage of storageElement (in GB)
"""
if not storageElement in self.storageElementUsage:
self.storageElementUsage[storageElement] = {'TotalSize': 0.}
size = size / (1000 * 1000 * 1000.0)
self.storageElementUsage[storageElement]['TotalSize'] += size
return size
def _issuePrestageRequests(self, storageElement, seReplicaIDs,
allReplicaInfo):
""" Make the request to the SE and update the DB
"""
pfnRepIDs = {}
for replicaID in seReplicaIDs:
pfn = allReplicaInfo[replicaID]['PFN']
pfnRepIDs[pfn] = replicaID
# Now issue the prestage requests for the remaining replicas
stageRequestMetadata = {}
updatedPfnIDs = []
if pfnRepIDs:
gLogger.info(
"StageRequest._issuePrestageRequests: Submitting %s stage requests for %s."
% (len(pfnRepIDs), storageElement))
res = self.replicaManager.prestageStorageFile(
pfnRepIDs.keys(), storageElement, lifetime=self.pinLifetime)
gLogger.debug(
"StageRequest._issuePrestageRequests: replicaManager.prestageStorageFile: res=",
res)
#Daniela: fishy result from ReplicaManager!!! Should NOT return OK
#res= {'OK': True, 'Value': {'Successful': {}, 'Failed': {'srm://srm-lhcb.cern.ch/castor/cern.ch/grid/lhcb/data/2010/RAW/EXPRESS/LHCb/COLLISION10/71476/071476_0000000241.raw': ' SRM2Storage.__gfal_exec: Failed to perform gfal_prestage.[SE][BringOnline][SRM_INVALID_REQUEST] httpg://srm-lhcb.cern.ch:8443/srm/managerv2: User not able to access specified space token\n'}}}
#res= {'OK': True, 'Value': {'Successful': {'srm://gridka-dCache.fzk.de/pnfs/gridka.de/lhcb/data/2009/RAW/FULL/LHCb/COLLISION09/63495/063495_0000000001.raw': '-2083846379'}, 'Failed': {}}}
if not res['OK']:
gLogger.error(
"StageRequest._issuePrestageRequests: Completely failed to submit stage requests for replicas.",
res['Message'])
else:
for pfn, requestID in res['Value']['Successful'].items():
if not stageRequestMetadata.has_key(requestID):
stageRequestMetadata[requestID] = []
stageRequestMetadata[requestID].append(pfnRepIDs[pfn])
updatedPfnIDs.append(pfnRepIDs[pfn])
if stageRequestMetadata:
gLogger.info(
"StageRequest._issuePrestageRequests: %s stage request metadata to be updated."
% len(stageRequestMetadata))
res = self.storageDB.insertStageRequest(stageRequestMetadata,
self.pinLifetime)
if not res['OK']:
gLogger.error(
"StageRequest._issuePrestageRequests: Failed to insert stage request metadata.",
res['Message'])
return res
res = self.storageDB.updateReplicaStatus(updatedPfnIDs,
'StageSubmitted')
if not res['OK']:
gLogger.error(
"StageRequest._issuePrestageRequests: Failed to insert replica status.",
res['Message'])
return
def __sortBySE(self, replicaDict):
seReplicas = {}
replicaIDs = {}
for replicaID, info in replicaDict.items():
lfn = info['LFN']
storageElement = info['SE']
size = info['Size']
pfn = info['PFN']
replicaIDs[replicaID] = {
'LFN': lfn,
'PFN': pfn,
'Size': size,
'StorageElement': storageElement
}
if not seReplicas.has_key(storageElement):
seReplicas[storageElement] = []
seReplicas[storageElement].append(replicaID)
return S_OK({'SEReplicas': seReplicas, 'AllReplicaInfo': replicaIDs})
def __getStagedReplicas(self):
""" This obtains the Staged replicas from the Replicas table and for each LFN the requested storage element """
# First obtain the Waiting replicas from the Replicas table
res = self.storageDB.getStagedReplicas()
if not res['OK']:
gLogger.error(
"StageRequest.__getStagedReplicas: Failed to get replicas with Waiting status.",
res['Message'])
return res
if not res['Value']:
gLogger.debug(
"StageRequest.__getStagedReplicas: No Waiting replicas found to process."
)
else:
gLogger.debug(
"StageRequest.__getStagedReplicas: Obtained %s Waiting replicas(s) to process."
% len(res['Value']))
return self.__sortBySE(res['Value'])
def __getWaitingReplicas(self):
""" This obtains the Waiting replicas from the Replicas table and for each LFN the requested storage element """
# First obtain the Waiting replicas from the Replicas table
res = self.storageDB.getWaitingReplicas()
if not res['OK']:
gLogger.error(
"StageRequest.__getWaitingReplicas: Failed to get replicas with Waiting status.",
res['Message'])
return res
if not res['Value']:
gLogger.debug(
"StageRequest.__getWaitingReplicas: No Waiting replicas found to process."
)
else:
gLogger.debug(
"StageRequest.__getWaitingReplicas: Obtained %s Waiting replicas(s) to process."
% len(res['Value']))
return self.__sortBySE(res['Value'])
def __getOfflineReplicas(self):
""" This obtains the Offline replicas from the Replicas table and for each LFN the requested storage element """
# First obtain the Waiting replicas from the Replicas table
res = self.storageDB.getOfflineReplicas()
if not res['OK']:
gLogger.error(
"StageRequest.__getOfflineReplicas: Failed to get replicas with Waiting status.",
res['Message'])
return res
if not res['Value']:
gLogger.debug(
"StageRequest.__getOfflineReplicas: No Waiting replicas found to process."
)
else:
gLogger.debug(
"StageRequest.__getOfflineReplicas: Obtained %s Waiting replicas(s) to process."
% len(res['Value']))
return self.__sortBySE(res['Value'])
def __addAssociatedReplicas(self, replicasToStage, seReplicas,
allReplicaInfo):
""" Retrieve the list of Replicas that belong to the same Tasks as the provided list
"""
res = self.storageDB.getAssociatedReplicas(replicasToStage)
if not res['OK']:
gLogger.fatal(
"StageRequest.__addAssociatedReplicas: Failed to get associated Replicas.",
res['Message'])
return res
addReplicas = {'Offline': {}, 'Waiting': {}}
replicaIDs = {}
for replicaID, info in res['Value'].items():
lfn = info['LFN']
storageElement = info['SE']
size = info['Size']
pfn = info['PFN']
status = info['Status']
if status not in ['Waiting', 'Offline']:
continue
if not addReplicas[status].has_key(storageElement):
addReplicas[status][storageElement] = []
replicaIDs[replicaID] = {
'LFN': lfn,
'PFN': pfn,
'Size': size,
'StorageElement': storageElement
}
addReplicas[status][storageElement].append(replicaID)
waitingReplicas = addReplicas['Waiting']
offlineReplicas = addReplicas['Offline']
newReplicaInfo = replicaIDs
allReplicaInfo.update(newReplicaInfo)
# First handle Waiting Replicas for which metadata is to be checked
for storageElement, seReplicaIDs in waitingReplicas.items():
for replicaID in list(seReplicaIDs):
if replicaID in replicasToStage:
seReplicaIDs.remove(replicaID)
res = self.__checkIntegrity(storageElement, seReplicaIDs,
allReplicaInfo)
if not res['OK']:
gLogger.error(
'StageRequest.__addAssociatedReplicas: Failed to check Replica Metadata',
'(%s): %s' % (storageElement, res['Message']))
else:
# keep all Replicas (Online and Offline)
if not storageElement in seReplicas:
seReplicas[storageElement] = []
seReplicas[storageElement].extend(res['Value']['Online'])
replicasToStage.extend(res['Value']['Online'])
seReplicas[storageElement].extend(res['Value']['Offline'])
replicasToStage.extend(res['Value']['Offline'])
# Then handle Offline Replicas for which metadata is already checked
for storageElement, seReplicaIDs in offlineReplicas.items():
if not storageElement in seReplicas:
seReplicas[storageElement] = []
for replicaID in sorted(seReplicaIDs):
if replicaID in replicasToStage:
seReplicaIDs.remove(replicaID)
seReplicas[storageElement].extend(seReplicaIDs)
replicasToStage.extend(seReplicaIDs)
for replicaID in allReplicaInfo.keys():
if replicaID not in replicasToStage:
del allReplicaInfo[replicaID]
totalSize = 0
for storageElement in sorted(seReplicas.keys()):
replicaIDs = seReplicas[storageElement]
size = 0
for replicaID in replicaIDs:
size += self.__add(storageElement,
allReplicaInfo[replicaID]['Size'])
gLogger.info(
'StageRequest.__addAssociatedReplicas: Considering %s GB to be staged at %s'
% (size, storageElement))
totalSize += size
gLogger.info(
"StageRequest.__addAssociatedReplicas: Obtained %s GB for staging."
% totalSize)
return S_OK({
'SEReplicas': seReplicas,
'AllReplicaInfo': allReplicaInfo
})
def __checkIntegrity(self, storageElement, seReplicaIDs, allReplicaInfo):
""" Check the integrity of the files to ensure they are available
Updates status of Offline Replicas for a later pass
Return list of Online replicas to be Stage
"""
if not seReplicaIDs:
return S_OK({'Online': [], 'Offline': []})
pfnRepIDs = {}
for replicaID in seReplicaIDs:
pfn = allReplicaInfo[replicaID]['PFN']
pfnRepIDs[pfn] = replicaID
gLogger.info(
"StageRequest.__checkIntegrity: Checking the integrity of %s replicas at %s."
% (len(pfnRepIDs), storageElement))
res = self.replicaManager.getStorageFileMetadata(
pfnRepIDs.keys(), storageElement)
if not res['OK']:
gLogger.error(
"StageRequest.__checkIntegrity: Completely failed to obtain metadata for replicas.",
res['Message'])
return res
terminalReplicaIDs = {}
onlineReplicaIDs = []
offlineReplicaIDs = []
for pfn, metadata in res['Value']['Successful'].items():
if metadata['Size'] != allReplicaInfo[pfnRepIDs[pfn]]['Size']:
gLogger.error(
"StageRequest.__checkIntegrity: PFN StorageElement size does not match FileCatalog",
pfn)
terminalReplicaIDs[pfnRepIDs[
pfn]] = 'PFN StorageElement size does not match FileCatalog'
pfnRepIDs.pop(pfn)
elif metadata['Lost']:
gLogger.error(
"StageRequest.__checkIntegrity: PFN has been Lost by the StorageElement",
pfn)
terminalReplicaIDs[
pfnRepIDs[pfn]] = 'PFN has been Lost by the StorageElement'
pfnRepIDs.pop(pfn)
elif metadata['Unavailable']:
gLogger.error(
"StageRequest.__checkIntegrity: PFN is declared Unavailable by the StorageElement",
pfn)
terminalReplicaIDs[pfnRepIDs[
pfn]] = 'PFN is declared Unavailable by the StorageElement'
pfnRepIDs.pop(pfn)
else:
if metadata['Cached']:
gLogger.verbose(
"StageRequest.__checkIntegrity: Cache hit for file.")
onlineReplicaIDs.append(pfnRepIDs[pfn])
else:
offlineReplicaIDs.append(pfnRepIDs[pfn])
for pfn, reason in res['Value']['Failed'].items():
if re.search('File does not exist', reason):
gLogger.error(
"StageRequest.__checkIntegrity: PFN does not exist in the StorageElement",
pfn)
terminalReplicaIDs[pfnRepIDs[
pfn]] = 'PFN does not exist in the StorageElement'
pfnRepIDs.pop(pfn)
# Update the states of the replicas in the database #TODO Sent status to integrity DB
if terminalReplicaIDs:
gLogger.info(
"StageRequest.__checkIntegrity: %s replicas are terminally failed."
% len(terminalReplicaIDs))
res = self.storageDB.updateReplicaFailure(terminalReplicaIDs)
if not res['OK']:
gLogger.error(
"StageRequest.__checkIntegrity: Failed to update replica failures.",
res['Message'])
if onlineReplicaIDs:
gLogger.info(
"StageRequest.__checkIntegrity: %s replicas found Online." %
len(onlineReplicaIDs))
if offlineReplicaIDs:
gLogger.info(
"StageRequest.__checkIntegrity: %s replicas found Offline." %
len(offlineReplicaIDs))
res = self.storageDB.updateReplicaStatus(offlineReplicaIDs,
'Offline')
return S_OK({'Online': onlineReplicaIDs, 'Offline': offlineReplicaIDs})
def __reportProblematicFiles(self, lfns, reason):
return S_OK()
res = self.dataIntegrityClient.setFileProblematic(
lfns, reason, self.name)
if not res['OK']:
gLogger.error(
"RequestPreparation.__reportProblematicFiles: Failed to report missing files.",
res['Message'])
return res
if res['Value']['Successful']:
gLogger.info(
"RequestPreparation.__reportProblematicFiles: Successfully reported %s missing files."
% len(res['Value']['Successful']))
if res['Value']['Failed']:
gLogger.info(
"RequestPreparation.__reportProblematicFiles: Failed to report %s problematic files."
% len(res['Value']['Failed']))
return res
class StorageElement:
"""
.. class:: StorageElement
common interface to the grid storage element
"""
def __init__( self, name, protocols = None, vo = None ):
""" c'tor
:param str name: SE name
:param list protocols: requested protocols
"""
self.vo = vo
if not vo:
result = getVOfromProxyGroup()
if not result['OK']:
return result
self.vo = result['Value']
self.opHelper = Operations( vo = self.vo )
self.resources = Resources( vo = self.vo )
proxiedProtocols = gConfig.getValue( '/LocalSite/StorageElements/ProxyProtocols', "" ).split( ',' )
result = self.resources.getAccessProtocols( name )
if result['OK']:
ap = result['Value'][0]
useProxy = ( self.resources.getAccessProtocolValue( ap, "Protocol", "UnknownProtocol" )
in proxiedProtocols )
#print "Proxy", name, proxiedProtocols, \
#gConfig.getValue( "/Resources/StorageElements/%s/AccessProtocol.1/Protocol" % name, "xxx" )
if not useProxy:
useProxy = gConfig.getValue( '/LocalSite/StorageElements/%s/UseProxy' % name, False )
if not useProxy:
useProxy = self.opHelper.getValue( '/Services/StorageElements/%s/UseProxy' % name, False )
self.valid = True
if protocols == None:
res = StorageFactory( useProxy ).getStorages( name, protocolList = [] )
else:
res = StorageFactory( useProxy ).getStorages( name, protocolList = protocols )
if not res['OK']:
self.valid = False
self.name = name
self.errorReason = res['Message']
else:
factoryDict = res['Value']
self.name = factoryDict['StorageName']
self.options = factoryDict['StorageOptions']
self.localProtocols = factoryDict['LocalProtocols']
self.remoteProtocols = factoryDict['RemoteProtocols']
self.storages = factoryDict['StorageObjects']
self.protocolOptions = factoryDict['ProtocolOptions']
self.turlProtocols = factoryDict['TurlProtocols']
self.log = gLogger.getSubLogger( "SE[%s]" % self.name )
self.readMethods = [ 'getFile',
'getAccessUrl',
'getTransportURL',
'prestageFile',
'prestageFileStatus',
'getDirectory']
self.writeMethods = [ 'retransferOnlineFile',
'putFile',
'replicateFile',
'pinFile',
'releaseFile',
'createDirectory',
'putDirectory' ]
self.removeMethods = [ 'removeFile', 'removeDirectory' ]
self.checkMethods = [ 'exists',
'getDirectoryMetadata',
'getDirectorySize',
'getFileSize',
'getFileMetadata',
'listDirectory',
'isDirectory',
'isFile',
]
self.okMethods = [ 'getLocalProtocols',
'getPfnForProtocol',
'getPfnForLfn',
'getPfnPath',
'getProtocols',
'getRemoteProtocols',
'getStorageElementName',
'getStorageElementOption',
'getStorageParameters',
'isLocalSE' ]
self.__resourceStatus = ResourceStatus()
def dump( self ):
""" Dump to the logger a summary of the StorageElement items. """
self.log.info( "dump: Preparing dump for StorageElement %s." % self.name )
if not self.valid:
self.log.error( "dump: Failed to create StorageElement plugins.", self.errorReason )
return
i = 1
outStr = "\n\n============ Options ============\n"
for key in sorted( self.options ):
outStr = "%s%s: %s\n" % ( outStr, key.ljust( 15 ), self.options[key] )
for storage in self.storages:
outStr = "%s============Protocol %s ============\n" % ( outStr, i )
res = storage.getParameters()
storageParameters = res['Value']
for key in sorted( storageParameters ):
outStr = "%s%s: %s\n" % ( outStr, key.ljust( 15 ), storageParameters[key] )
i = i + 1
self.log.info( outStr )
#################################################################################################
#
# These are the basic get functions for storage configuration
#
def getStorageElementName( self ):
""" SE name getter """
self.log.verbose( "getStorageElementName: The Storage Element name is %s." % self.name )
return S_OK( self.name )
def getChecksumType( self ):
""" get local /Resources/StorageElements/SEName/ChecksumType option if defined, otherwise
global /Resources/StorageElements/ChecksumType
"""
return S_OK( str( gConfig.getValue( "/Resources/StorageElements/ChecksumType", "ADLER32" ) ).upper()
if "ChecksumType" not in self.options else str( self.options["ChecksumType"] ).upper() )
def getStatus( self ):
"""
Return Status of the SE, a dictionary with:
- Read: True (is allowed), False (it is not allowed)
- Write: True (is allowed), False (it is not allowed)
- Remove: True (is allowed), False (it is not allowed)
- Check: True (is allowed), False (it is not allowed).
NB: Check always allowed IF Read is allowed (regardless of what set in the Check option of the configuration)
- DiskSE: True if TXDY with Y > 0 (defaults to True)
- TapeSE: True if TXDY with X > 0 (defaults to False)
- TotalCapacityTB: float (-1 if not defined)
- DiskCacheTB: float (-1 if not defined)
"""
retDict = {}
if not self.valid:
retDict['Read'] = False
retDict['Write'] = False
retDict['Remove'] = False
retDict['Check'] = False
retDict['DiskSE'] = False
retDict['TapeSE'] = False
retDict['TotalCapacityTB'] = -1
retDict['DiskCacheTB'] = -1
return S_OK( retDict )
# If nothing is defined in the CS Access is allowed
# If something is defined, then it must be set to Active
retDict['Read'] = self.__resourceStatus.isUsableStorage( self.name, 'ReadAccess' )
retDict['Write'] = self.__resourceStatus.isUsableStorage( self.name, 'WriteAccess' )
retDict['Remove'] = self.__resourceStatus.isUsableStorage( self.name, 'RemoveAccess' )
if retDict['Read']:
retDict['Check'] = True
else:
retDict['Check'] = self.__resourceStatus.isUsableStorage( self.name, 'CheckAccess' )
diskSE = True
tapeSE = False
if 'SEType' in self.options:
# Type should follow the convention TXDY
seType = self.options['SEType']
diskSE = re.search( 'D[1-9]', seType ) != None
tapeSE = re.search( 'T[1-9]', seType ) != None
retDict['DiskSE'] = diskSE
retDict['TapeSE'] = tapeSE
try:
retDict['TotalCapacityTB'] = float( self.options['TotalCapacityTB'] )
except Exception:
retDict['TotalCapacityTB'] = -1
try:
retDict['DiskCacheTB'] = float( self.options['DiskCacheTB'] )
except Exception:
retDict['DiskCacheTB'] = -1
return S_OK( retDict )
def isValid( self, operation = '' ):
""" check CS/RSS statuses for :operation:
:param str operation: operation name
"""
self.log.debug( "isValid: Determining whether the StorageElement %s is valid for %s" % ( self.name,
operation ) )
if ( not operation ) or ( operation in self.okMethods ):
return S_OK()
if not self.valid:
self.log.error( "isValid: Failed to create StorageElement plugins.", self.errorReason )
return S_ERROR( self.errorReason )
# Determine whether the StorageElement is valid for checking, reading, writing
res = self.getStatus()
if not res[ 'OK' ]:
self.log.error( "Could not call getStatus" )
return S_ERROR( "StorageElement.isValid could not call the getStatus method" )
checking = res[ 'Value' ][ 'Check' ]
reading = res[ 'Value' ][ 'Read' ]
writing = res[ 'Value' ][ 'Write' ]
removing = res[ 'Value' ][ 'Remove' ]
# Determine whether the requested operation can be fulfilled
if ( not operation ) and ( not reading ) and ( not writing ) and ( not checking ):
self.log.error( "isValid: Read, write and check access not permitted." )
return S_ERROR( "StorageElement.isValid: Read, write and check access not permitted." )
# The supplied operation can be 'Read','Write' or any of the possible StorageElement methods.
if ( operation in self.readMethods ) or ( operation.lower() in ( 'read', 'readaccess' ) ):
operation = 'ReadAccess'
elif operation in self.writeMethods or ( operation.lower() in ( 'write', 'writeaccess' ) ):
operation = 'WriteAccess'
elif operation in self.removeMethods or ( operation.lower() in ( 'remove', 'removeaccess' ) ):
operation = 'RemoveAccess'
elif operation in self.checkMethods or ( operation.lower() in ( 'check', 'checkaccess' ) ):
operation = 'CheckAccess'
else:
self.log.error( "isValid: The supplied operation is not known.", operation )
return S_ERROR( "StorageElement.isValid: The supplied operation is not known." )
self.log.debug( "in isValid check the operation: %s " % operation )
# Check if the operation is valid
if operation == 'CheckAccess':
if not reading:
if not checking:
self.log.error( "isValid: Check access not currently permitted." )
return S_ERROR( "StorageElement.isValid: Check access not currently permitted." )
if operation == 'ReadAccess':
if not reading:
self.log.error( "isValid: Read access not currently permitted." )
return S_ERROR( "StorageElement.isValid: Read access not currently permitted." )
if operation == 'WriteAccess':
if not writing:
self.log.error( "isValid: Write access not currently permitted." )
return S_ERROR( "StorageElement.isValid: Write access not currently permitted." )
if operation == 'RemoveAccess':
if not removing:
self.log.error( "isValid: Remove access not currently permitted." )
return S_ERROR( "StorageElement.isValid: Remove access not currently permitted." )
return S_OK()
def getProtocols( self ):
""" Get the list of all the protocols defined for this Storage Element
"""
if not self.valid:
return S_ERROR( self.errorReason )
self.log.verbose( "getProtocols: Obtaining all protocols." )
allProtocols = self.localProtocols + self.remoteProtocols
return S_OK( allProtocols )
def getRemoteProtocols( self ):
""" Get the list of all the remote access protocols defined for this Storage Element
"""
if not self.valid:
return S_ERROR( self.errorReason )
self.log.verbose( "getRemoteProtocols: Obtaining remote protocols for %s." % self.name )
return S_OK( self.remoteProtocols )
def getLocalProtocols( self ):
""" Get the list of all the local access protocols defined for this Storage Element
"""
if not self.valid:
return S_ERROR( self.errorReason )
self.log.verbose( "getLocalProtocols: Obtaining local protocols for %s." % self.name )
return S_OK( self.localProtocols )
def getStorageElementOption( self, option ):
""" Get the value for the option supplied from self.options
"""
if not self.valid:
return S_ERROR( self.errorReason )
self.log.verbose( "getStorageElementOption: Obtaining %s option for Storage Element %s." % ( option,
self.name ) )
if option in self.options:
optionValue = self.options[option]
return S_OK( optionValue )
else:
errStr = "getStorageElementOption: Option not defined for SE."
self.log.error( errStr, "%s for %s" % ( option, self.name ) )
return S_ERROR( errStr )
def getStorageParameters( self, protocol ):
""" Get protocol specific options
"""
self.log.verbose( "getStorageParameters: Obtaining storage parameters for %s protocol %s." % ( self.name,
protocol ) )
res = self.getProtocols()
if not res['OK']:
return res
availableProtocols = res['Value']
if not protocol in availableProtocols:
errStr = "getStorageParameters: Requested protocol not available for SE."
self.log.warn( errStr, '%s for %s' % ( protocol, self.name ) )
return S_ERROR( errStr )
for storage in self.storages:
res = storage.getParameters()
storageParameters = res['Value']
if storageParameters['ProtocolName'] == protocol:
return S_OK( storageParameters )
errStr = "getStorageParameters: Requested protocol supported but no object found."
self.log.error( errStr, "%s for %s" % ( protocol, self.name ) )
return S_ERROR( errStr )
def isLocalSE( self ):
""" Test if the Storage Element is local in the current context
"""
import DIRAC
self.log.verbose( "isLocalSE: Determining whether %s is a local SE." % self.name )
localSEs = getSEsForSite( DIRAC.siteName() )['Value']
if self.name in localSEs:
return S_OK( True )
else:
return S_OK( False )
#################################################################################################
#
# These are the basic get functions for pfn manipulation
#
def getPfnForProtocol( self, pfn, protocol, withPort = True ):
""" Transform the input pfn into another with the given protocol for the Storage Element.
"""
res = self.getProtocols()
if not res['OK']:
return res
if type( protocol ) == StringType:
protocols = [protocol]
elif type( protocol ) == ListType:
protocols = protocol
else:
errStr = "getPfnForProtocol: Supplied protocol must be string or list of strings."
self.log.error( errStr, "%s %s" % ( protocol, self.name ) )
return S_ERROR( errStr )
availableProtocols = res['Value']
protocolsToTry = []
for protocol in protocols:
if protocol in availableProtocols:
protocolsToTry.append( protocol )
else:
errStr = "getPfnForProtocol: Requested protocol not available for SE."
self.log.debug( errStr, '%s for %s' % ( protocol, self.name ) )
if not protocolsToTry:
errStr = "getPfnForProtocol: None of the requested protocols were available for SE."
self.log.error( errStr, '%s for %s' % ( protocol, self.name ) )
return S_ERROR( errStr )
# Check all available storages for required protocol then contruct the PFN
for storage in self.storages:
res = storage.getParameters()
if res['Value']['ProtocolName'] in protocolsToTry:
res = pfnparse( pfn )
if res['OK']:
res = storage.getProtocolPfn( res['Value'], withPort )
if res['OK']:
return res
errStr = "getPfnForProtocol: Failed to get PFN for requested protocols."
self.log.error( errStr, "%s for %s" % ( protocols, self.name ) )
return S_ERROR( errStr )
def getPfnPath( self, pfn ):
""" Get the part of the PFN path below the basic storage path.
This path must coincide with the LFN of the file in order to be compliant with the LHCb conventions.
"""
if not self.valid:
return S_ERROR( self.errorReason )
res = pfnparse( pfn )
if not res['OK']:
return res
fullPfnPath = '%s/%s' % ( res['Value']['Path'], res['Value']['FileName'] )
# Check all available storages and check whether the pfn is for that protocol
pfnPath = ''
for storage in self.storages:
res = storage.isPfnForProtocol( pfn )
if res['OK']:
if res['Value']:
res = storage.getParameters()
saPath = res['Value']['Path']
if not saPath:
# If the sa path doesn't exist then the pfn path is the entire string
pfnPath = fullPfnPath
else:
if re.search( saPath, fullPfnPath ):
# Remove the sa path from the fullPfnPath
pfnPath = fullPfnPath.replace( saPath, '' )
if pfnPath:
return S_OK( pfnPath )
# This should never happen. DANGER!!
errStr = "getPfnPath: Failed to get the pfn path for any of the protocols!!"
self.log.error( errStr )
return S_ERROR( errStr )
def getPfnForLfn( self, lfn ):
""" Get the full PFN constructed from the LFN.
"""
if not self.valid:
return S_ERROR( self.errorReason )
for storage in self.storages:
res = storage.getPFNBase()
if res['OK']:
fullPath = "%s%s" % ( res['Value'], lfn )
return S_OK( fullPath )
# This should never happen. DANGER!!
errStr = "getPfnForLfn: Failed to get the full pfn for any of the protocols!!"
self.log.error( errStr )
return S_ERROR( errStr )
def getPFNBase( self ):
""" Get the base to construct a PFN
"""
if not self.storages:
return S_ERROR( 'No storages defined' )
for storage in self.storages:
result = storage.getPFNBase()
if result['OK']:
return result
return result
###########################################################################################
#
# This is the generic wrapper for file operations
#
def retransferOnlineFile( self, pfn, singleFile = False ):
""" execcute 'retransferOnlineFile' operation """
return { True : self.__executeSingleFile,
False : self.__executeFunction }[bool( singleFile )]( pfn, 'retransferOnlineFile' )
def exists( self, pfn, singleFile = False ):
""" execute 'exists' operation """
return { True : self.__executeSingleFile,
False : self.__executeFunction }[bool( singleFile )]( pfn, 'exists' )
def isFile( self, pfn, singleFile = False ):
""" execute 'isFile' operation """
return { True : self.__executeSingleFile,
False : self.__executeFunction }[bool( singleFile )]( pfn, 'isFile' )
def getFile( self, pfn, localPath = False, singleFile = False ):
""" execute 'getFile' operation """
return { True : self.__executeSingleFile,
False : self.__executeFunction }[bool( singleFile )]( pfn, 'getFile', { 'localPath': localPath } )
def putFile( self, pfn, sourceSize = 0, singleFile = False ):
""" execute 'putFile' operation """
return { True : self.__executeSingleFile,
False : self.__executeFunction }[bool( singleFile )]( pfn, 'putFile', { 'sourceSize': sourceSize } )
def replicateFile( self, pfn, sourceSize = 0, singleFile = False ):
""" execute 'putFile' as replicate """
return { True : self.__executeSingleFile,
False : self.__executeFunction }[bool( singleFile )]( pfn, 'putFile', { 'sourceSize': sourceSize } )
def getFileMetadata( self, pfn, singleFile = False ):
""" execute 'getFileMetadata' operation """
return { True : self.__executeSingleFile,
False : self.__executeFunction }[bool( singleFile )]( pfn, 'getFileMetadata' )
def getFileSize( self, pfn, singleFile = False ):
""" execute 'getFileSize' operation """
return { True : self.__executeSingleFile,
False : self.__executeFunction }[bool( singleFile )]( pfn, 'getFileSize' )
def getAccessUrl( self, pfn, protocol = False, singleFile = False ):
""" execute 'getTransportURL' operation """
if not protocol:
protocols = self.turlProtocols
else:
protocols = [protocol]
return { True : self.__executeSingleFile,
False : self.__executeFunction }[bool( singleFile )]( pfn, 'getTransportURL', {'protocols': protocols} )
def removeFile( self, pfn, singleFile = False ):
""" execute 'removeFile' operation """
return { True : self.__executeSingleFile,
False : self.__executeFunction }[bool( singleFile )]( pfn, 'removeFile' )
def prestageFile( self, pfn, lifetime = 86400, singleFile = False ):
""" execute 'prestageFile' operation """
return { True : self.__executeSingleFile,
False : self.__executeFunction }[bool( singleFile )]( pfn, 'prestageFile', { 'lifetime': lifetime } )
def prestageFileStatus( self, pfn, singleFile = False ):
""" execute 'prestageFileStatus' operation """
return { True : self.__executeSingleFile,
False : self.__executeFunction }[bool( singleFile )]( pfn, 'prestageFileStatus' )
def pinFile( self, pfn, lifetime = 60 * 60 * 24, singleFile = False ):
""" execute 'pinFile' operation """
return { True : self.__executeSingleFile,
False : self.__executeFunction }[bool( singleFile )]( pfn, 'pinFile', { 'lifetime': lifetime } )
def releaseFile( self, pfn, singleFile = False ):
""" execute 'releaseFile' operation """
return { True : self.__executeSingleFile,
False : self.__executeFunction }[bool( singleFile )]( pfn, 'releaseFile' )
def isDirectory( self, pfn, singleDirectory = False ):
""" execute 'isDirectory' operation """
return { True : self.__executeSingleFile,
False : self.__executeFunction }[bool( singleDirectory )]( pfn, 'isDirectory' )
def getDirectoryMetadata( self, pfn, singleDirectory = False ):
""" execute 'getDirectoryMetadata' operation """
return { True : self.__executeSingleFile,
False : self.__executeFunction }[bool( singleDirectory )]( pfn, 'getDirectoryMetadata' )
def getDirectorySize( self, pfn, singleDirectory = False ):
""" execute 'getDirectorySize' operation """
return { True : self.__executeSingleFile,
False : self.__executeFunction }[bool( singleDirectory )]( pfn, 'getDirectorySize' )
def listDirectory( self, pfn, singleDirectory = False ):
""" execute 'listDirectory' operation """
return { True : self.__executeSingleFile,
False : self.__executeFunction }[bool( singleDirectory )]( pfn, 'listDirectory' )
def removeDirectory( self, pfn, recursive = False, singleDirectory = False ):
""" execute 'removeDirectory' operation """
return { True : self.__executeSingleFile,
False : self.__executeFunction }[bool( singleDirectory )]( pfn, 'removeDirectory', {'recursive':
recursive} )
def createDirectory( self, pfn, singleDirectory = False ):
""" execute 'createDirectory' operation """
return { True : self.__executeSingleFile,
False : self.__executeFunction }[bool( singleDirectory )]( pfn, 'createDirectory' )
def putDirectory( self, pfn, singleDirectory = False ):
""" execute 'putDirectory' operation """
return { True : self.__executeSingleFile,
False : self.__executeFunction }[bool( singleDirectory )]( pfn, 'putDirectory' )
def getDirectory( self, pfn, localPath = False, singleDirectory = False ):
""" execute 'getDirectory' operation """
return { True : self.__executeSingleFile,
False : self.__executeFunction }[bool( singleDirectory )]( pfn, 'getDirectory', { 'localPath':
localPath } )
def __executeSingleFile( self, pfn, operation, arguments = None ):
""" execute for single file """
if arguments == None:
res = self.__executeFunction( pfn, operation, {} )
else:
res = self.__executeFunction( pfn, operation, arguments )
if type( pfn ) == ListType:
pfn = pfn[0]
elif type( pfn ) == DictType:
pfn = pfn.keys()[0]
if not res['OK']:
return res
elif pfn in res['Value']['Failed']:
errorMessage = res['Value']['Failed'][pfn]
return S_ERROR( errorMessage )
else:
return S_OK( res['Value']['Successful'][pfn] )
def __executeFunction( self, pfn, method, argsDict = None ):
"""
'pfn' is the physical file name (as registered in the LFC)
'method' is the functionality to be executed
"""
## default args = no args
argsDict = argsDict if argsDict else {}
if type( pfn ) in StringTypes:
pfns = {pfn:False}
elif type( pfn ) == ListType:
pfns = {}
for url in pfn:
pfns[url] = False
elif type( pfn ) == DictType:
pfns = pfn.copy()
else:
errStr = "__executeFunction: Supplied pfns must be string or list of strings or a dictionary."
self.log.error( errStr )
return S_ERROR( errStr )
if not pfns:
self.log.verbose( "__executeFunction: No pfns supplied." )
return S_OK( {'Failed':{}, 'Successful':{}} )
self.log.verbose( "__executeFunction: Attempting to perform '%s' operation with %s pfns." % ( method,
len( pfns ) ) )
res = self.isValid( operation = method )
if not res['OK']:
return res
else:
if not self.valid:
return S_ERROR( self.errorReason )
successful = {}
failed = {}
localSE = self.isLocalSE()['Value']
# Try all of the storages one by one
for storage in self.storages:
# Determine whether to use this storage object
res = storage.getParameters()
useProtocol = True
if not res['OK']:
self.log.error( "__executeFunction: Failed to get storage parameters.", "%s %s" % ( self.name,
res['Message'] ) )
useProtocol = False
else:
protocolName = res['Value']['ProtocolName']
if not pfns:
useProtocol = False
self.log.verbose( "__executeFunction: No pfns to be attempted for %s protocol." % protocolName )
elif not ( protocolName in self.remoteProtocols ) and not localSE:
# If the SE is not local then we can't use local protocols
useProtocol = False
self.log.verbose( "__executeFunction: Protocol not appropriate for use: %s." % protocolName )
if useProtocol:
self.log.verbose( "__executeFunction: Generating %s protocol PFNs for %s." % ( len( pfns ),
protocolName ) )
res = self.__generatePfnDict( pfns, storage )
pfnDict = res['Value']
failed.update( res['Failed'] )
if not len( pfnDict ) > 0:
self.log.verbose( "__executeFunction No pfns generated for protocol %s." % protocolName )
else:
self.log.verbose( "__executeFunction: Attempting to perform '%s' for %s physical files" % ( method,
len( pfnDict ) ) )
fcn = None
if hasattr( storage, method ) and callable( getattr( storage, method ) ):
fcn = getattr( storage, method )
if not fcn:
return S_ERROR( "__executeFunction: unable to invoke %s, it isn't a member function of storage" )
pfnsToUse = {}
for pfn in pfnDict:
pfnsToUse[pfn] = pfns[pfnDict[pfn]]
res = fcn( pfnsToUse, **argsDict )
if not res['OK']:
errStr = "__executeFunction: Completely failed to perform %s." % method
self.log.error( errStr, '%s for protocol %s: %s' % ( self.name, protocolName, res['Message'] ) )
for pfn in pfnDict.values():
if pfn not in failed:
failed[pfn] = ''
failed[pfn] = "%s %s" % ( failed[pfn], res['Message'] )
else:
for protocolPfn, pfn in pfnDict.items():
if protocolPfn not in res['Value']['Successful']:
if pfn not in failed:
failed[pfn] = ''
if protocolPfn in res['Value']['Failed']:
failed[pfn] = "%s %s" % ( failed[pfn], res['Value']['Failed'][protocolPfn] )
else:
failed[pfn] = "%s %s" % ( failed[pfn], 'No error returned from plug-in' )
else:
successful[pfn] = res['Value']['Successful'][protocolPfn]
if pfn in failed:
failed.pop( pfn )
pfns.pop( pfn )
return S_OK( { 'Failed': failed, 'Successful': successful } )
def __generatePfnDict( self, pfns, storage ):
""" whatever, it creates PFN dict """
pfnDict = {}
failed = {}
for pfn in pfns:
res = pfnparse( pfn )
if not res['OK']:
errStr = "__generatePfnDict: Failed to parse supplied PFN."
self.log.error( errStr, "%s: %s" % ( pfn, res['Message'] ) )
if pfn not in failed:
failed[pfn] = ''
failed[pfn] = "%s %s" % ( failed[pfn], errStr )
else:
res = storage.getProtocolPfn( res['Value'], True )
if not res['OK']:
errStr = "__generatePfnDict %s." % res['Message']
self.log.error( errStr, 'for %s' % ( pfn ) )
if pfn not in failed:
failed[pfn] = ''
failed[pfn] = "%s %s" % ( failed[pfn], errStr )
else:
pfnDict[res['Value']] = pfn
res = S_OK( pfnDict )
res['Failed'] = failed
return res
class StageRequestAgent( AgentModule ):
def initialize( self ):
self.replicaManager = ReplicaManager()
#self.stagerClient = StorageManagerClient()
self.dataIntegrityClient = DataIntegrityClient()
self.storageDB = StorageManagementDB()
# pin lifetime = 1 day
self.pinLifetime = self.am_getOption( 'PinLifetime', THROTTLING_TIME )
# Resources helper
self.resources = Resources()
# This sets the Default Proxy to used as that defined under
# /Operations/Shifter/DataManager
# the shifterProxy option in the Configuration can be used to change this default.
self.am_setOption( 'shifterProxy', 'DataManager' )
return S_OK()
def execute( self ):
# Get the current submitted stage space and the amount of pinned space for each storage element
res = self.getStorageUsage()
if not res['OK']:
return res
return self.submitStageRequests()
def getStorageUsage( self ):
""" Fill the current Status of the SE Caches from the DB
"""
self.storageElementCache = {}
res = self.storageDB.getSubmittedStagePins()
if not res['OK']:
gLogger.fatal( "StageRequest.getStorageUsage: Failed to obtain submitted requests from StorageManagementDB.", res['Message'] )
return res
self.storageElementUsage = res['Value']
if self.storageElementUsage:
gLogger.info( "StageRequest.getStorageUsage: Active stage/pin requests found at the following sites:" )
for storageElement in sortList( self.storageElementUsage.keys() ):
seDict = self.storageElementUsage[storageElement]
# Convert to GB for printout
seDict['TotalSize'] = seDict['TotalSize'] / ( 1000 * 1000 * 1000.0 )
gLogger.info( "StageRequest.getStorageUsage: %s: %s replicas with a size of %.3f GB." %
( storageElement.ljust( 15 ), str( seDict['Replicas'] ).rjust( 6 ), seDict['TotalSize'] ) )
if not self.storageElementUsage:
gLogger.info( "StageRequest.getStorageUsage: No active stage/pin requests found." )
return S_OK()
def submitStageRequests( self ):
""" This manages the following transitions of the Replicas
* Waiting -> Offline (if the file is not found Cached)
* Waiting -> StageSubmitted (if the file is found Cached)
* Offline -> StageSubmitted (if there are not more Waiting replicas)
"""
# Retry Replicas that have not been Staged in a previous attempt
res = self._getMissingReplicas()
if not res['OK']:
gLogger.fatal( "StageRequest.submitStageRequests: Failed to get replicas from StorageManagementDB.", res['Message'] )
return res
seReplicas = res['Value']['SEReplicas']
allReplicaInfo = res['Value']['AllReplicaInfo']
if seReplicas:
gLogger.info( "StageRequest.submitStageRequests: Completing partially Staged Tasks" )
for storageElement, seReplicaIDs in seReplicas.items():
gLogger.debug( 'Staging at %s:' % storageElement, seReplicaIDs )
self._issuePrestageRequests( storageElement, seReplicaIDs, allReplicaInfo )
# Check Waiting Replicas and select those found Online and all other Replicas from the same Tasks
res = self._getOnlineReplicas()
if not res['OK']:
gLogger.fatal( "StageRequest.submitStageRequests: Failed to get replicas from StorageManagementDB.", res['Message'] )
return res
seReplicas = res['Value']['SEReplicas']
allReplicaInfo = res['Value']['AllReplicaInfo']
# Check Offline Replicas that fit in the Cache and all other Replicas from the same Tasks
res = self._getOfflineReplicas()
if not res['OK']:
gLogger.fatal( "StageRequest.submitStageRequests: Failed to get replicas from StorageManagementDB.", res['Message'] )
return res
# Merge info from both results
for storageElement, seReplicaIDs in res['Value']['SEReplicas'].items():
if storageElement not in seReplicas:
seReplicas[storageElement] = seReplicaIDs
else:
for replicaID in seReplicaIDs:
if replicaID not in seReplicas[storageElement]:
seReplicas[storageElement].append( replicaID )
allReplicaInfo.update( res['Value']['AllReplicaInfo'] )
gLogger.info( "StageRequest.submitStageRequests: Obtained %s replicas for staging." % len( allReplicaInfo ) )
for storageElement, seReplicaIDs in seReplicas.items():
gLogger.debug( 'Staging at %s:' % storageElement, seReplicaIDs )
self._issuePrestageRequests( storageElement, seReplicaIDs, allReplicaInfo )
return S_OK()
def _getMissingReplicas( self ):
""" This recovers Replicas that were not Staged on a previous attempt (the stage request failed or timed out),
while other Replicas of the same task are already Staged. If left behind they can produce a deadlock.
All SEs are considered, even if their Cache is full
"""
# Get Replicas that are in Staged/StageSubmitted
gLogger.info( 'StageRequest._getMissingReplicas: Checking Staged Replicas' )
res = self.__getStagedReplicas()
if not res['OK']:
gLogger.fatal( "StageRequest._getMissingReplicas: Failed to get replicas from StorageManagementDB.", res['Message'] )
return res
seReplicas = {}
allReplicaInfo = res['Value']['AllReplicaInfo']
replicasToStage = []
for storageElement, seReplicaIDs in res['Value']['SEReplicas'].items():
# Consider all SEs
replicasToStage.extend( seReplicaIDs )
# Get Replicas from the same Tasks as those selected
res = self.__addAssociatedReplicas( replicasToStage, seReplicas, allReplicaInfo )
if not res['OK']:
gLogger.fatal( "StageRequest._getMissingReplicas: Failed to get associated Replicas.", res['Message'] )
return res
def _getOnlineReplicas( self ):
""" This manages the transition
* Waiting -> Offline (if the file is not found Cached)
and returns the list of Cached Replicas for which the pin time has to be extended
SEs for which the cache is currently full are not considered
"""
# Get all Replicas in Waiting Status associated to Staging Tasks
gLogger.verbose( 'StageRequest._getOnlineReplicas: Checking Online Replicas to be handled' )
res = self.__getWaitingReplicas()
if not res['OK']:
gLogger.fatal( "StageRequest._getOnlineReplicas: Failed to get replicas from StorageManagementDB.", res['Message'] )
return res
seReplicas = {}
allReplicaInfo = res['Value']['AllReplicaInfo']
if not len( allReplicaInfo ):
gLogger.info( "StageRequest._getOnlineReplicas: There were no Waiting replicas found" )
return res
gLogger.info( "StageRequest._getOnlineReplicas: Obtained %s replicas Waiting for staging." % len( allReplicaInfo ) )
replicasToStage = []
for storageElement, seReplicaIDs in res['Value']['SEReplicas'].items():
if not self.__usage( storageElement ) < self.__cache( storageElement ):
gLogger.info( 'StageRequest._getOnlineReplicas: Skipping %s, current usage above limit ( %s GB )' % ( storageElement, self.__cache( storageElement ) ) )
# Do not consider those SE that have the Cache full
continue
# Check if the Replica Metadata is OK and find out if they are Online or Offline
res = self.__checkIntegrity( storageElement, seReplicaIDs, allReplicaInfo )
if not res['OK']:
gLogger.error( 'StageRequest._getOnlineReplicas: Failed to check Replica Metadata', '(%s): %s' % ( storageElement, res['Message'] ) )
else:
# keep only Online Replicas
seReplicas[storageElement] = res['Value']['Online']
replicasToStage.extend( res['Value']['Online'] )
# Get Replicas from the same Tasks as those selected
res = self.__addAssociatedReplicas( replicasToStage, seReplicas, allReplicaInfo )
if not res['OK']:
gLogger.fatal( "StageRequest._getOnlineReplicas: Failed to get associated Replicas.", res['Message'] )
return res
def _getOfflineReplicas( self ):
""" This checks Replicas in Offline status
and returns the list of Replicas to be Staged
SEs for which the cache is currently full are not considered
"""
# Get all Replicas in Waiting Status associated to Staging Tasks
gLogger.verbose( 'StageRequest._getOfflineReplicas: Checking Offline Replicas to be handled' )
res = self.__getOfflineReplicas()
if not res['OK']:
gLogger.fatal( "StageRequest._getOfflineReplicas: Failed to get replicas from StorageManagementDB.", res['Message'] )
return res
seReplicas = {}
allReplicaInfo = res['Value']['AllReplicaInfo']
if not len( allReplicaInfo ):
gLogger.info( "StageRequest._getOfflineReplicas: There were no Offline replicas found" )
return res
gLogger.info( "StageRequest._getOfflineReplicas: Obtained %s replicas Offline for staging." % len( allReplicaInfo ) )
replicasToStage = []
for storageElement, seReplicaIDs in res['Value']['SEReplicas'].items():
if not self.__usage( storageElement ) < self.__cache( storageElement ):
gLogger.info( 'StageRequest._getOfflineReplicas: Skipping %s, current usage above limit ( %s GB )' % ( storageElement, self.__cache( storageElement ) ) )
# Do not consider those SE that have the Cache full
continue
seReplicas[storageElement] = []
for replicaID in sorted( seReplicaIDs ):
seReplicas[storageElement].append( replicaID )
replicasToStage.append( replicaID )
self.__add( storageElement, allReplicaInfo[replicaID]['Size'] )
if not self.__usage( storageElement ) < self.__cache( storageElement ):
# Stop adding Replicas when the cache is full
break
# Get Replicas from the same Tasks as those selected
res = self.__addAssociatedReplicas( replicasToStage, seReplicas, allReplicaInfo )
if not res['OK']:
gLogger.fatal( "StageRequest._getOfflineReplicas: Failed to get associated Replicas.", res['Message'] )
return res
def __usage( self, storageElement ):
""" Retrieve current usage of SE
"""
if not storageElement in self.storageElementUsage:
self.storageElementUsage[storageElement] = {'TotalSize': 0.}
return self.storageElementUsage[storageElement]['TotalSize']
def __cache( self, storageElement ):
""" Retrieve cache size for SE
"""
if not storageElement in self.storageElementCache:
diskCache = self.resources.getStorageElementValue( storageElement, 'DiskCacheTB', 1. )
self.storageElementCache[storageElement] = diskCache * 1000. / THROTTLING_STEPS
return self.storageElementCache[storageElement]
def __add( self, storageElement, size ):
""" Add size (in bytes) to current usage of storageElement (in GB)
"""
if not storageElement in self.storageElementUsage:
self.storageElementUsage[storageElement] = {'TotalSize': 0.}
size = size / ( 1000 * 1000 * 1000.0 )
self.storageElementUsage[storageElement]['TotalSize'] += size
return size
def _issuePrestageRequests( self, storageElement, seReplicaIDs, allReplicaInfo ):
""" Make the request to the SE and update the DB
"""
pfnRepIDs = {}
for replicaID in seReplicaIDs:
pfn = allReplicaInfo[replicaID]['PFN']
pfnRepIDs[pfn] = replicaID
# Now issue the prestage requests for the remaining replicas
stageRequestMetadata = {}
updatedPfnIDs = []
if pfnRepIDs:
gLogger.info( "StageRequest._issuePrestageRequests: Submitting %s stage requests for %s." % ( len( pfnRepIDs ), storageElement ) )
res = self.replicaManager.prestageStorageFile( pfnRepIDs.keys(), storageElement, lifetime = self.pinLifetime )
gLogger.debug( "StageRequest._issuePrestageRequests: replicaManager.prestageStorageFile: res=", res )
#Daniela: fishy result from ReplicaManager!!! Should NOT return OK
#res= {'OK': True, 'Value': {'Successful': {}, 'Failed': {'srm://srm-lhcb.cern.ch/castor/cern.ch/grid/lhcb/data/2010/RAW/EXPRESS/LHCb/COLLISION10/71476/071476_0000000241.raw': ' SRM2Storage.__gfal_exec: Failed to perform gfal_prestage.[SE][BringOnline][SRM_INVALID_REQUEST] httpg://srm-lhcb.cern.ch:8443/srm/managerv2: User not able to access specified space token\n'}}}
#res= {'OK': True, 'Value': {'Successful': {'srm://gridka-dCache.fzk.de/pnfs/gridka.de/lhcb/data/2009/RAW/FULL/LHCb/COLLISION09/63495/063495_0000000001.raw': '-2083846379'}, 'Failed': {}}}
if not res['OK']:
gLogger.error( "StageRequest._issuePrestageRequests: Completely failed to submit stage requests for replicas.", res['Message'] )
else:
for pfn, requestID in res['Value']['Successful'].items():
if not stageRequestMetadata.has_key( requestID ):
stageRequestMetadata[requestID] = []
stageRequestMetadata[requestID].append( pfnRepIDs[pfn] )
updatedPfnIDs.append( pfnRepIDs[pfn] )
if stageRequestMetadata:
gLogger.info( "StageRequest._issuePrestageRequests: %s stage request metadata to be updated." % len( stageRequestMetadata ) )
res = self.storageDB.insertStageRequest( stageRequestMetadata, self.pinLifetime )
if not res['OK']:
gLogger.error( "StageRequest._issuePrestageRequests: Failed to insert stage request metadata.", res['Message'] )
return res
res = self.storageDB.updateReplicaStatus( updatedPfnIDs, 'StageSubmitted' )
if not res['OK']:
gLogger.error( "StageRequest._issuePrestageRequests: Failed to insert replica status.", res['Message'] )
return
def __sortBySE( self, replicaDict ):
seReplicas = {}
replicaIDs = {}
for replicaID, info in replicaDict.items():
lfn = info['LFN']
storageElement = info['SE']
size = info['Size']
pfn = info['PFN']
replicaIDs[replicaID] = {'LFN':lfn, 'PFN':pfn, 'Size':size, 'StorageElement':storageElement}
if not seReplicas.has_key( storageElement ):
seReplicas[storageElement] = []
seReplicas[storageElement].append( replicaID )
return S_OK( {'SEReplicas':seReplicas, 'AllReplicaInfo':replicaIDs} )
def __getStagedReplicas( self ):
""" This obtains the Staged replicas from the Replicas table and for each LFN the requested storage element """
# First obtain the Waiting replicas from the Replicas table
res = self.storageDB.getStagedReplicas()
if not res['OK']:
gLogger.error( "StageRequest.__getStagedReplicas: Failed to get replicas with Waiting status.", res['Message'] )
return res
if not res['Value']:
gLogger.debug( "StageRequest.__getStagedReplicas: No Waiting replicas found to process." )
else:
gLogger.debug( "StageRequest.__getStagedReplicas: Obtained %s Waiting replicas(s) to process." % len( res['Value'] ) )
return self.__sortBySE( res['Value'] )
def __getWaitingReplicas( self ):
""" This obtains the Waiting replicas from the Replicas table and for each LFN the requested storage element """
# First obtain the Waiting replicas from the Replicas table
res = self.storageDB.getWaitingReplicas()
if not res['OK']:
gLogger.error( "StageRequest.__getWaitingReplicas: Failed to get replicas with Waiting status.", res['Message'] )
return res
if not res['Value']:
gLogger.debug( "StageRequest.__getWaitingReplicas: No Waiting replicas found to process." )
else:
gLogger.debug( "StageRequest.__getWaitingReplicas: Obtained %s Waiting replicas(s) to process." % len( res['Value'] ) )
return self.__sortBySE( res['Value'] )
def __getOfflineReplicas( self ):
""" This obtains the Offline replicas from the Replicas table and for each LFN the requested storage element """
# First obtain the Waiting replicas from the Replicas table
res = self.storageDB.getOfflineReplicas()
if not res['OK']:
gLogger.error( "StageRequest.__getOfflineReplicas: Failed to get replicas with Waiting status.", res['Message'] )
return res
if not res['Value']:
gLogger.debug( "StageRequest.__getOfflineReplicas: No Waiting replicas found to process." )
else:
gLogger.debug( "StageRequest.__getOfflineReplicas: Obtained %s Waiting replicas(s) to process." % len( res['Value'] ) )
return self.__sortBySE( res['Value'] )
def __addAssociatedReplicas( self, replicasToStage, seReplicas, allReplicaInfo ):
""" Retrieve the list of Replicas that belong to the same Tasks as the provided list
"""
res = self.storageDB.getAssociatedReplicas( replicasToStage )
if not res['OK']:
gLogger.fatal( "StageRequest.__addAssociatedReplicas: Failed to get associated Replicas.", res['Message'] )
return res
addReplicas = {'Offline': {}, 'Waiting': {}}
replicaIDs = {}
for replicaID, info in res['Value'].items():
lfn = info['LFN']
storageElement = info['SE']
size = info['Size']
pfn = info['PFN']
status = info['Status']
if status not in ['Waiting', 'Offline']:
continue
if not addReplicas[status].has_key( storageElement ):
addReplicas[status][storageElement] = []
replicaIDs[replicaID] = {'LFN':lfn, 'PFN':pfn, 'Size':size, 'StorageElement':storageElement }
addReplicas[status][storageElement].append( replicaID )
waitingReplicas = addReplicas['Waiting']
offlineReplicas = addReplicas['Offline']
newReplicaInfo = replicaIDs
allReplicaInfo.update( newReplicaInfo )
# First handle Waiting Replicas for which metadata is to be checked
for storageElement, seReplicaIDs in waitingReplicas.items():
for replicaID in list( seReplicaIDs ):
if replicaID in replicasToStage:
seReplicaIDs.remove( replicaID )
res = self.__checkIntegrity( storageElement, seReplicaIDs, allReplicaInfo )
if not res['OK']:
gLogger.error( 'StageRequest.__addAssociatedReplicas: Failed to check Replica Metadata', '(%s): %s' % ( storageElement, res['Message'] ) )
else:
# keep all Replicas (Online and Offline)
if not storageElement in seReplicas:
seReplicas[storageElement] = []
seReplicas[storageElement].extend( res['Value']['Online'] )
replicasToStage.extend( res['Value']['Online'] )
seReplicas[storageElement].extend( res['Value']['Offline'] )
replicasToStage.extend( res['Value']['Offline'] )
# Then handle Offline Replicas for which metadata is already checked
for storageElement, seReplicaIDs in offlineReplicas.items():
if not storageElement in seReplicas:
seReplicas[storageElement] = []
for replicaID in sorted( seReplicaIDs ):
if replicaID in replicasToStage:
seReplicaIDs.remove( replicaID )
seReplicas[storageElement].extend( seReplicaIDs )
replicasToStage.extend( seReplicaIDs )
for replicaID in allReplicaInfo.keys():
if replicaID not in replicasToStage:
del allReplicaInfo[replicaID]
totalSize = 0
for storageElement in sorted( seReplicas.keys() ):
replicaIDs = seReplicas[storageElement]
size = 0
for replicaID in replicaIDs:
size += self.__add( storageElement, allReplicaInfo[replicaID]['Size'] )
gLogger.info( 'StageRequest.__addAssociatedReplicas: Considering %s GB to be staged at %s' % ( size, storageElement ) )
totalSize += size
gLogger.info( "StageRequest.__addAssociatedReplicas: Obtained %s GB for staging." % totalSize )
return S_OK( {'SEReplicas':seReplicas, 'AllReplicaInfo':allReplicaInfo} )
def __checkIntegrity( self, storageElement, seReplicaIDs, allReplicaInfo ):
""" Check the integrity of the files to ensure they are available
Updates status of Offline Replicas for a later pass
Return list of Online replicas to be Stage
"""
if not seReplicaIDs:
return S_OK( {'Online': [], 'Offline': []} )
pfnRepIDs = {}
for replicaID in seReplicaIDs:
pfn = allReplicaInfo[replicaID]['PFN']
pfnRepIDs[pfn] = replicaID
gLogger.info( "StageRequest.__checkIntegrity: Checking the integrity of %s replicas at %s." % ( len( pfnRepIDs ), storageElement ) )
res = self.replicaManager.getStorageFileMetadata( pfnRepIDs.keys(), storageElement )
if not res['OK']:
gLogger.error( "StageRequest.__checkIntegrity: Completely failed to obtain metadata for replicas.", res['Message'] )
return res
terminalReplicaIDs = {}
onlineReplicaIDs = []
offlineReplicaIDs = []
for pfn, metadata in res['Value']['Successful'].items():
if metadata['Size'] != allReplicaInfo[pfnRepIDs[pfn]]['Size']:
gLogger.error( "StageRequest.__checkIntegrity: PFN StorageElement size does not match FileCatalog", pfn )
terminalReplicaIDs[pfnRepIDs[pfn]] = 'PFN StorageElement size does not match FileCatalog'
pfnRepIDs.pop( pfn )
elif metadata['Lost']:
gLogger.error( "StageRequest.__checkIntegrity: PFN has been Lost by the StorageElement", pfn )
terminalReplicaIDs[pfnRepIDs[pfn]] = 'PFN has been Lost by the StorageElement'
pfnRepIDs.pop( pfn )
elif metadata['Unavailable']:
gLogger.error( "StageRequest.__checkIntegrity: PFN is declared Unavailable by the StorageElement", pfn )
terminalReplicaIDs[pfnRepIDs[pfn]] = 'PFN is declared Unavailable by the StorageElement'
pfnRepIDs.pop( pfn )
else:
if metadata['Cached']:
gLogger.verbose( "StageRequest.__checkIntegrity: Cache hit for file." )
onlineReplicaIDs.append( pfnRepIDs[pfn] )
else:
offlineReplicaIDs.append( pfnRepIDs[pfn] )
for pfn, reason in res['Value']['Failed'].items():
if re.search( 'File does not exist', reason ):
gLogger.error( "StageRequest.__checkIntegrity: PFN does not exist in the StorageElement", pfn )
terminalReplicaIDs[pfnRepIDs[pfn]] = 'PFN does not exist in the StorageElement'
pfnRepIDs.pop( pfn )
# Update the states of the replicas in the database #TODO Sent status to integrity DB
if terminalReplicaIDs:
gLogger.info( "StageRequest.__checkIntegrity: %s replicas are terminally failed." % len( terminalReplicaIDs ) )
res = self.storageDB.updateReplicaFailure( terminalReplicaIDs )
if not res['OK']:
gLogger.error( "StageRequest.__checkIntegrity: Failed to update replica failures.", res['Message'] )
if onlineReplicaIDs:
gLogger.info( "StageRequest.__checkIntegrity: %s replicas found Online." % len( onlineReplicaIDs ) )
if offlineReplicaIDs:
gLogger.info( "StageRequest.__checkIntegrity: %s replicas found Offline." % len( offlineReplicaIDs ) )
res = self.storageDB.updateReplicaStatus( offlineReplicaIDs, 'Offline' )
return S_OK( {'Online': onlineReplicaIDs, 'Offline': offlineReplicaIDs} )
def __reportProblematicFiles( self, lfns, reason ):
return S_OK()
res = self.dataIntegrityClient.setFileProblematic( lfns, reason, self.name )
if not res['OK']:
gLogger.error( "RequestPreparation.__reportProblematicFiles: Failed to report missing files.", res['Message'] )
return res
if res['Value']['Successful']:
gLogger.info( "RequestPreparation.__reportProblematicFiles: Successfully reported %s missing files." % len( res['Value']['Successful'] ) )
if res['Value']['Failed']:
gLogger.info( "RequestPreparation.__reportProblematicFiles: Failed to report %s problematic files." % len( res['Value']['Failed'] ) )
return res
def __getGridJobOutput(self, pilotReference):
""" Get the pilot job standard output and standard error files for the Grid
job reference
"""
result = pilotDB.getPilotInfo(pilotReference)
if not result['OK'] or not result['Value']:
return S_ERROR('Failed to get info for pilot ' + pilotReference)
pilotDict = result['Value'][pilotReference]
owner = pilotDict['OwnerDN']
group = pilotDict['OwnerGroup']
# FIXME: What if the OutputSandBox is not StdOut and StdErr, what do we do with other files?
result = pilotDB.getPilotOutput(pilotReference)
if result['OK']:
stdout = result['Value']['StdOut']
error = result['Value']['StdErr']
if stdout or error:
resultDict = {}
resultDict['StdOut'] = stdout
resultDict['StdErr'] = error
resultDict['OwnerDN'] = owner
resultDict['OwnerGroup'] = group
resultDict['FileList'] = []
return S_OK(resultDict)
else:
gLogger.warn('Empty pilot output found for %s' %
pilotReference)
gridType = pilotDict['GridType']
if gridType in ["LCG", "gLite", "CREAM"]:
group = getGroupOption(group, 'VOMSRole', group)
ret = gProxyManager.getPilotProxyFromVOMSGroup(owner, group)
if not ret['OK']:
gLogger.error(ret['Message'])
gLogger.error('Could not get proxy:',
'User "%s", Group "%s"' % (owner, group))
return S_ERROR("Failed to get the pilot's owner proxy")
proxy = ret['Value']
pilotStamp = pilotDict['PilotStamp']
result = getPilotOutput(proxy, gridType, pilotReference,
pilotStamp)
if not result['OK']:
return S_ERROR('Failed to get pilot output: ' +
result['Message'])
# FIXME: What if the OutputSandBox is not StdOut and StdErr, what do we do with other files?
stdout = result['StdOut']
error = result['StdErr']
fileList = result['FileList']
if stdout:
result = pilotDB.storePilotOutput(pilotReference, stdout,
error)
if not result['OK']:
gLogger.error('Failed to store pilot output:',
result['Message'])
resultDict = {}
resultDict['StdOut'] = stdout
resultDict['StdErr'] = error
resultDict['OwnerDN'] = owner
resultDict['OwnerGroup'] = group
resultDict['FileList'] = fileList
return S_OK(resultDict)
else:
# Instantiate the appropriate CE
ceFactory = ComputingElementFactory()
result = Resources(group=group).getQueueDescription(
pilotDict['Queue'])
if not result['OK']:
return result
queueDict = result['Value']
result = ceFactory.getCE(gridType, pilotDict['DestinationSite'],
queueDict)
if not result['OK']:
return result
ce = result['Value']
pilotStamp = pilotDict['PilotStamp']
pRef = pilotReference
if pilotStamp:
pRef = pRef + ':::' + pilotStamp
result = ce.getJobOutput(pRef)
if not result['OK']:
return result
stdout, error = result['Value']
if stdout:
result = pilotDB.storePilotOutput(pilotReference, stdout,
error)
if not result['OK']:
gLogger.error('Failed to store pilot output:',
result['Message'])
resultDict = {}
resultDict['StdOut'] = stdout
resultDict['StdErr'] = error
resultDict['OwnerDN'] = owner
resultDict['OwnerGroup'] = group
resultDict['FileList'] = []
return S_OK(resultDict)
import sys, os
import DIRAC
from DIRAC import gLogger, gConfig
from DIRAC.Resources.Catalog.FileCatalogFactory import FileCatalogFactory
from DIRAC.Core.Security.ProxyInfo import getVOfromProxyGroup
from DIRAC.ConfigurationSystem.Client.Helpers.Resources import Resources
if __name__ == "__main__":
result = getVOfromProxyGroup()
if not result['OK']:
gLogger.notice('Error:', result['Message'])
DIRAC.exit(1)
vo = result['Value']
resources = Resources(vo=vo)
result = gConfig.getSections("/LocalSite/Catalogs")
if result['OK']:
catalogs = result['Value']
userCatalogs = []
for switch in Script.getUnprocessedSwitches():
if switch[0].lower() == "f" or switch[0].lower() == "file-catalog":
userCatalogs.append(switch[1])
if userCatalogs:
catalogs = userCatalogs
from DIRAC.Resources.Catalog.FileCatalog import FileCatalog
from DIRAC.DataManagementSystem.Client.FileCatalogClientCLI import FileCatalogClientCLI
if catalogs:
class Synchronizer(object):
'''
Every time there is a successful write on the CS, Synchronizer().sync() is
executed. It updates the database with the values on the CS.
'''
def __init__(self):
"""
Constructor.
examples:
>>> s = Synchronizer()
"""
self.log = gLogger.getSubLogger(self.__class__.__name__)
self.operations = Operations()
self.resources = Resources()
self.rStatus = ResourceStatusClient.ResourceStatusClient()
self.rssConfig = RssConfiguration()
self.diracAdmin = DiracAdmin()
def sync(self, _eventName, _params):
'''
Main synchronizer method. It synchronizes the three types of elements: Sites,
Resources and Nodes. Each _syncX method returns a dictionary with the additions
and deletions.
examples:
>>> s.sync( None, None )
S_OK()
:Parameters:
**_eventName** - any
this parameter is ignored, but needed by caller function.
**_params** - any
this parameter is ignored, but needed by caller function.
:return: S_OK
'''
defSyncResult = {'added': [], 'deleted': []}
# Sites
syncSites = self._syncSites()
if not syncSites['OK']:
self.log.error(syncSites['Message'])
syncSites = (syncSites['OK'] and syncSites['Value']) or defSyncResult
# Resources
syncResources = self._syncResources()
if not syncResources['OK']:
self.log.error(syncResources['Message'])
syncResources = (syncResources['OK']
and syncResources['Value']) or defSyncResult
# Nodes
syncNodes = self._syncNodes()
if not syncNodes['OK']:
self.log.error(syncNodes['Message'])
syncNodes = (syncNodes['OK'] and syncNodes['Value']) or defSyncResult
# Notify via email to :
self.notify(syncSites, syncResources, syncNodes)
return S_OK()
def notify(self, syncSites, syncResources, syncNodes):
"""
Method sending email notification with the result of the synchronization. Email
is sent to Operations( EMail/Production ) email address.
examples:
>>> s.notify( {}, {}, {} )
>>> s.notify( { 'Site' : { 'added' : [], 'deleted' : [ 'RubbishSite' ] }, {}, {} )
>>> s.notify( { 'Site' : { 'added' : [], 'deleted' : [ 'RubbishSite' ] },
{ 'Computing : { 'added' : [ 'newCE01', 'newCE02' ], 'deleted' : [] }}, {} )
:Parameters:
**syncSites** - dict() ( keys: added, deleted )
dictionary with the sites added and deleted from the DB
**syncResources** - dict() ( keys: added, deleted )
dictionary with the resources added and deleted from the DB
**syncNodes** - dict() ( keys: added, deleted )
dictionary with the nodes added and deleted from the DB
:return: S_OK
"""
# Human readable summary
msgBody = self.getBody(syncSites, syncResources, syncNodes)
self.log.info(msgBody)
# Email addresses
toAddress = self.operations.getValue('EMail/Production', '')
fromAddress = self.rssConfig.getConfigFromAddress('')
if toAddress and fromAddress and msgBody:
# Subject of the email
setup = gConfig.getValue('DIRAC/Setup')
subject = '[RSS](%s) CS Synchronization' % setup
self.diracAdmin.sendMail(toAddress,
subject,
msgBody,
fromAddress=fromAddress)
def getBody(self, syncSites, syncResources, syncNodes):
"""
Method that given the outputs of the three synchronization methods builds a
human readable string.
examples:
>>> s.getBody( {}, {}, {} )
''
>>> s.getBody( { 'Site' : { 'added' : [], 'deleted' : [ 'RubbishSite' ] }, {}, {} )
'''
SITES:
Site:
deleted:1
RubbishSite
'''
>>> s.getBody( { 'Site' : { 'added' : [], 'deleted' : [ 'RubbishSite' ] },
{ 'Computing : { 'added' : [ 'newCE01', 'newCE02' ], 'deleted' : [] }}, {} )
'''
SITES:
Site:
deleted:1
RubbishSite
RESOURCES:
Computing:
added:2
newCE01
newCE02
'''
:Parameters:
**syncSites** - dict() ( keys: added, deleted )
dictionary with the sites added and deleted from the DB
**syncResources** - dict() ( keys: added, deleted )
dictionary with the resources added and deleted from the DB
**syncNodes** - dict() ( keys: added, deleted )
dictionary with the nodes added and deleted from the DB
:return: str
"""
syncMsg = ''
for element, syncResult in [('SITES', syncSites),
('RESOURCES', syncResources),
('NODES', syncNodes)]:
elementsMsg = ''
for elementType, elements in syncResult.items():
elementMsg = ''
if elements['added']:
elementMsg += '\n %s added: %d \n' % (
elementType, len(elements['added']))
elementMsg += ' ' + '\n '.join(elements['added'])
if elements['deleted']:
elementMsg += '\n %s deleted: %d \n' % (
elementType, len(elements['deleted']))
elementMsg += ' ' + '\n '.join(elements['deleted'])
if elementMsg:
elementsMsg += '\n\n%s:\n' % elementType
elementsMsg += elementMsg
if elementsMsg:
syncMsg += '\n\n%s:' % element + elementsMsg
return syncMsg
#.............................................................................
# Sync methods: Site, Resource & Node
def _syncSites(self):
"""
Method that synchronizes sites ( using their canonical name: CERN.ch ) with
elementType = 'Site'. It gets from the CS the eligible site names and then
synchronizes them with the DB. If not on the DB, they are added. If in the DB
but not on the CS, they are deleted.
examples:
>> s._syncSites()
S_OK( { 'Site' : { 'added' : [], 'deleted' : [ 'RubbishSite' ] } } )
:return: S_OK( { 'Site' : { 'added' : [], 'deleted' : [] }} ) | S_ERROR
"""
# Get site names from the CS
foundSites = self.resources.getEligibleSites()
if not foundSites['OK']:
return foundSites
sites = {}
# Synchronize with the DB
resSync = self.__dbSync('Site', 'Site', foundSites['Value'])
if not resSync['OK']:
self.log.error('Error synchronizing Sites')
self.log.error(resSync['Message'])
else:
sites = resSync['Value']
return S_OK({'Site': sites})
def _syncResources(self):
"""
Method that synchronizes resources as defined on RESOURCE_NODE_MAPPING dictionary
keys. It makes one sync round per key ( elementType ). Gets from the CS the
eligible Resource/<elementType> names and then synchronizes them with the DB.
If not on the DB, they are added. If in the DB but not on the CS, they are deleted.
examples:
>>> s._syncResources()
S_OK( { 'Computing' : { 'added' : [ 'newCE01', 'newCE02' ], 'deleted' : [] },
'Storage' : { 'added' : [], 'deleted' : [] },
... } )
:return: S_OK( { 'RESOURCE_NODE_MAPPINGKey1' : { 'added' : [], 'deleted' : [] }, ...} )
"""
resources = {}
# Iterate over the different elementTypes for Resource ( Computing, Storage... )
for elementType in RESOURCE_NODE_MAPPING.keys():
# Get Resource / <elementType> names from CS
foundResources = self.resources.getEligibleResources(elementType)
if not foundResources['OK']:
self.log.error(foundResources['Message'])
continue
# Translate CS result into a list
foundResources = foundResources['Value']
# Synchronize with the DB
resSync = self.__dbSync('Resource', elementType, foundResources)
if not resSync['OK']:
self.log.error('Error synchronizing %s %s' %
('Resource', elementType))
self.log.error(resSync['Message'])
else:
resources[elementType] = resSync['Value']
return S_OK(resources)
def _syncNodes(self):
"""
Method that synchronizes resources as defined on RESOURCE_NODE_MAPPING dictionary
values. It makes one sync round per key ( elementType ). Gets from the CS the
eligible Node/<elementType> names and then synchronizes them with the DB.
If not on the DB, they are added. If in the DB but not on the CS, they are deleted.
examples:
>>> s._syncNodes()
S_OK( { 'Queue' : { 'added' : [], 'deleted' : [] },
... } )
:return: S_OK( { 'RESOURCE_NODE_MAPPINGValue1' : { 'added' : [], 'deleted' : [] }, ...} )
"""
nodes = {}
# Iterate over the different elementTypes for Node ( Queue, AccessProtocol... )
for elementType in RESOURCE_NODE_MAPPING.values():
# Get Node / <elementType> names from CS
foundNodes = self.resources.getEligibleNodes(elementType)
if not foundNodes['OK']:
self.log.error(foundNodes['Value'])
continue
# Translate CS result into a list : maps NodeName to SiteName<>NodeName to
# avoid duplicates
# Looong list comprehension, sorry !
foundNodes = [
'%s<>%s' % (key, item)
for key, subDict in foundNodes['Value'].items()
for subList in subDict.values() for item in subList
]
# Synchronize with the DB
resSync = self.__dbSync('Node', elementType, foundNodes)
if not resSync['OK']:
self.log.error('Error synchronizing %s %s' %
('Node', elementType))
self.log.error(resSync['Message'])
else:
nodes[elementType] = resSync['Value']
return S_OK(nodes)
#.............................................................................
# DB sync actions
def __dbSync(self, elementFamily, elementType, elementsCS):
"""
Method synchronizing CS and DB. Compares <elementsCS> with <elementsDB>
given the elementFamily and elementType ( e.g. Resource / Computing ).
If there are missing elements in the DB, are inserted. If are missing elements
in the CS, are deleted from the DB. Note that the logs from the RSS DB
are kept ! ( just in case ).
:Parameters:
**elementFamily** - str
any of the valid element families : Site, Resource, Node
**elementType** - str
any of the valid element types for <elementFamily>
**elementsCS** - list
list with the elements for <elementFamily>/<elementType> found in the CS
:return: S_OK( { 'added' : [], 'deleted' : [] } ) | S_ERROR
"""
# deleted, added default response
syncRes = {
'deleted': [],
'added': [],
}
# Gets <elementFamily>/<elementType> elements from DB
elementsDB = self.rStatus.selectStatusElement(
elementFamily,
'Status',
elementType=elementType,
meta={'columns': ['name']})
if not elementsDB['OK']:
return elementsDB
elementsDB = [elementDB[0] for elementDB in elementsDB['Value']]
# Elements in DB but not in CS -> to be deleted
toBeDeleted = list(set(elementsDB).difference(set(elementsCS)))
if toBeDeleted:
resDelete = self.__dbDelete(elementFamily, elementType,
toBeDeleted)
if not resDelete['OK']:
return resDelete
else:
syncRes['deleted'] = toBeDeleted
# Elements in CS but not in DB -> to be added
toBeAdded = list(set(elementsCS).difference(set(elementsDB)))
if toBeAdded:
resInsert = self.__dbInsert(elementFamily, elementType, toBeAdded)
if not resInsert['OK']:
return resInsert
else:
syncRes['added'] = toBeAdded
return S_OK(syncRes)
def __dbDelete(self, elementFamily, elementType, toBeDeleted):
"""
Method that given the elementFamily and elementType, deletes all entries
in the History and Status tables for the given elements in toBeDeleted ( all
their status Types ).
:Parameters:
**elementFamily** - str
any of the valid element families : Site, Resource, Node
**elementType** - str
any of the valid element types for <elementFamily>, just used for logging
purposes.
**toBeDeleted** - list
list with the elements to be deleted
:return: S_OK | S_ERROR
"""
self.log.info('Deleting %s %s:' % (elementFamily, elementType))
self.log.info(toBeDeleted)
return self.rStatus._extermineStatusElement(elementFamily, toBeDeleted)
def __dbInsert(self, elementFamily, elementType, toBeAdded):
"""
Method that given the elementFamily and elementType, adds all elements in
toBeAdded with their respective statusTypes, obtained from the CS. They
are synchronized with status 'Unknown' and reason 'Synchronized'.
:Parameters:
**elementFamily** - str
any of the valid element families : Site, Resource, Node
**elementType** - str
any of the valid element types for <elementFamily>
**toBeDeleted** - list
list with the elements to be added
:return: S_OK | S_ERROR
"""
self.log.info('Adding %s %s:' % (elementFamily, elementType))
self.log.info(toBeAdded)
statusTypes = self.rssConfig.getConfigStatusType(elementType)
for element in toBeAdded:
for statusType in statusTypes:
resInsert = self.rStatus.addIfNotThereStatusElement(
elementFamily,
'Status',
name=element,
statusType=statusType,
status='Unknown',
elementType=elementType,
reason='Synchronized')
if not resInsert['OK']:
return resInsert
return S_OK()
#...............................................................................
#
# def _syncUsers( self ):
# '''
# Sync Users: compares CS with DB and does the necessary modifications.
# '''
#
# gLogger.verbose( '-- Synchronizing users --')
#
# usersCS = CSHelpers.getRegistryUsers()
# if not usersCS[ 'OK' ]:
# return usersCS
# usersCS = usersCS[ 'Value' ]
#
# gLogger.verbose( '%s users found in CS' % len( usersCS ) )
#
# usersDB = self.rManagement.selectUserRegistryCache( meta = { 'columns' : [ 'login' ] } )
# if not usersDB[ 'OK' ]:
# return usersDB
# usersDB = [ userDB[0] for userDB in usersDB[ 'Value' ] ]
#
# # Users that are in DB but not in CS
# toBeDeleted = list( set( usersDB ).difference( set( usersCS.keys() ) ) )
# gLogger.verbose( '%s users to be deleted' % len( toBeDeleted ) )
#
# # Delete users
# # FIXME: probably it is not needed since there is a DatabaseCleanerAgent
# for userLogin in toBeDeleted:
#
# deleteQuery = self.rManagement.deleteUserRegistryCache( login = userLogin )
#
# gLogger.verbose( '... %s' % userLogin )
# if not deleteQuery[ 'OK' ]:
# return deleteQuery
#
# # AddOrModify Users
# for userLogin, userDict in usersCS.items():
#
# _name = userDict[ 'DN' ].split( '=' )[ -1 ]
# _email = userDict[ 'Email' ]
#
# query = self.rManagement.addOrModifyUserRegistryCache( userLogin, _name, _email )
# gLogger.verbose( '-> %s' % userLogin )
# if not query[ 'OK' ]:
# return query
#
# return S_OK()
################################################################################
#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF
ceFlag = True
return DIRAC.S_OK()
Script.registerSwitch("V:", "vo=",
"choose resources eligible for the given VO", setVO)
Script.registerSwitch("S", "se", "display storage element information",
setSEFlag)
Script.registerSwitch("C", "ce", "display computing element information",
setSEFlag)
Script.setUsageMessage('\n'.join([
__doc__.split('\n')[1],
'Usage:',
' %s [option|cfgfile] ...' % Script.scriptName,
]))
Script.parseCommandLine()
from DIRAC.ConfigurationSystem.Client.Helpers.Resources import Resources, getSites
resources = Resources(vo='biomed')
result = resources.getEligibleSites()
if not result['OK']:
print "ERROR:", result['Message']
siteList = [
resources.getSiteFullName(site)['Value'] for site in result['Value']
]
print siteList
class StorageFactory:
def __init__(self, useProxy=False, vo=None):
self.valid = True
self.proxy = False
self.proxy = useProxy
self.resourceStatus = ResourceStatus()
self.resourcesHelper = Resources(vo=vo)
###########################################################################################
#
# Below are public methods for obtaining storage objects
#
def getStorageName(self, initialName):
return self._getConfigStorageName(initialName)
def getStorage(self, parameterDict):
""" This instantiates a single storage for the details provided and doesn't check the CS.
"""
# The storage name must be supplied.
if parameterDict.has_key('StorageName'):
storageName = parameterDict['StorageName']
else:
errStr = "StorageFactory.getStorage: StorageName must be supplied"
gLogger.error(errStr)
return S_ERROR(errStr)
# ProtocolName must be supplied otherwise nothing with work.
if parameterDict.has_key('ProtocolName'):
protocolName = parameterDict['ProtocolName']
else:
errStr = "StorageFactory.getStorage: ProtocolName must be supplied"
gLogger.error(errStr)
return S_ERROR(errStr)
# The other options need not always be specified
if parameterDict.has_key('Protocol'):
protocol = parameterDict['Protocol']
else:
protocol = ''
if parameterDict.has_key('Port'):
port = parameterDict['Port']
else:
port = ''
if parameterDict.has_key('Host'):
host = parameterDict['Host']
else:
host = ''
if parameterDict.has_key('Path'):
path = parameterDict['Path']
else:
path = ''
if parameterDict.has_key('SpaceToken'):
spaceToken = parameterDict['SpaceToken']
else:
spaceToken = ''
if parameterDict.has_key('WSUrl'):
wsPath = parameterDict['WSUrl']
else:
wsPath = ''
return self.__generateStorageObject(storageName, protocolName,
protocol, path, host, port,
spaceToken, wsPath)
def getStorages(self, storageName, protocolList=[]):
""" Get an instance of a Storage based on the DIRAC SE name based on the CS entries CS
'storageName' is the DIRAC SE name i.e. 'CERN-RAW'
'protocolList' is an optional list of protocols if a sub-set is desired i.e ['SRM2','SRM1']
"""
self.remoteProtocols = []
self.localProtocols = []
self.name = ''
self.options = {}
self.protocolDetails = []
self.storages = []
# Get the name of the storage provided
res = self._getConfigStorageName(storageName)
if not res['OK']:
self.valid = False
return res
storageName = res['Value']
self.name = storageName
# Get the options defined in the CS for this storage
res = self._getConfigStorageOptions(storageName)
if not res['OK']:
self.valid = False
return res
self.options = res['Value']
# Get the protocol specific details
res = self._getConfigStorageProtocols(storageName)
if not res['OK']:
self.valid = False
return res
self.protocolDetails = res['Value']
requestedLocalProtocols = []
requestedRemoteProtocols = []
requestedProtocolDetails = []
turlProtocols = []
# Generate the protocol specific plug-ins
self.storages = []
for protocolDict in self.protocolDetails:
protocolName = protocolDict['ProtocolName']
protocolRequested = True
if protocolList:
if protocolName not in protocolList:
protocolRequested = False
if protocolRequested:
protocol = protocolDict['Protocol']
host = protocolDict['Host']
path = protocolDict['Path']
port = protocolDict['Port']
spaceToken = protocolDict['SpaceToken']
wsUrl = protocolDict['WSUrl']
res = self.__generateStorageObject(storageName,
protocolName,
protocol,
path=path,
host=host,
port=port,
spaceToken=spaceToken,
wsUrl=wsUrl)
if res['OK']:
self.storages.append(res['Value'])
if protocolName in self.localProtocols:
turlProtocols.append(protocol)
requestedLocalProtocols.append(protocolName)
if protocolName in self.remoteProtocols:
requestedRemoteProtocols.append(protocolName)
requestedProtocolDetails.append(protocolDict)
else:
gLogger.info(res['Message'])
if len(self.storages) > 0:
resDict = {}
resDict['StorageName'] = self.name
resDict['StorageOptions'] = self.options
resDict['StorageObjects'] = self.storages
resDict['LocalProtocols'] = requestedLocalProtocols
resDict['RemoteProtocols'] = requestedRemoteProtocols
resDict['ProtocolOptions'] = requestedProtocolDetails
resDict['TurlProtocols'] = turlProtocols
return S_OK(resDict)
else:
errStr = "StorageFactory.getStorages: Failed to instantiate any storage protocols."
gLogger.error(errStr, self.name)
return S_ERROR(errStr)
###########################################################################################
#
# Below are internal methods for obtaining section/option/value configuration
#
def _getConfigStorageName(self, storageName):
"""
This gets the name of the storage the configuration service.
If the storage is an alias for another the resolution is performed.
'storageName' is the storage section to check in the CS
"""
result = self.resourcesHelper.getStorageElementOptionsDict(storageName)
if not result['OK']:
errStr = "StorageFactory._getConfigStorageName: Failed to get storage options"
gLogger.error(errStr, result['Message'])
return S_ERROR(errStr)
if not result['Value']:
errStr = "StorageFactory._getConfigStorageName: Supplied storage doesn't exist."
gLogger.error(errStr, storageName)
return S_ERROR(errStr)
seConfig = result['Value']
resolvedName = seConfig.get('Alias', storageName)
return S_OK(resolvedName)
def _getConfigStorageOptions(self, storageName):
""" Get the options associated to the StorageElement as defined in the CS
"""
result = self.resourcesHelper.getStorageElementOptionsDict(storageName)
if not result['OK']:
errStr = "StorageFactory._getStorageOptions: Failed to get storage options."
gLogger.error(errStr, "%s: %s" % (storageName, result['Message']))
return S_ERROR(errStr)
optionsDict = result['Value']
result = self.resourceStatus.getStorageElementStatus(storageName)
if not result['OK']:
errStr = "StorageFactory._getStorageOptions: Failed to get storage status"
gLogger.error(errStr, "%s: %s" % (storageName, result['Message']))
return S_ERROR(errStr)
optionsDict.update(result['Value'][storageName])
return S_OK(optionsDict)
def _getConfigStorageProtocols(self, storageName):
""" Protocol specific information is present as sections in the Storage configuration
"""
result = getSiteForResource('Storage', storageName)
if not result['OK']:
return result
site = result['Value']
result = self.resourcesHelper.getEligibleNodes('AccessProtocol', {
'Site': site,
'Resource': storageName
})
if not result['OK']:
return result
nodesDict = result['Value']
protocols = []
for site in nodesDict:
for se in nodesDict[site]:
protocols.extend(nodesDict[site][se])
sortedProtocols = sortList(protocols)
protocolDetails = []
for protocol in sortedProtocols:
result = self._getConfigStorageProtocolDetails(
storageName, protocol)
if not result['OK']:
return result
protocolDetails.append(result['Value'])
self.protocols = self.localProtocols + self.remoteProtocols
return S_OK(protocolDetails)
def _getConfigStorageProtocolDetails(self, storageName, protocol):
"""
Parse the contents of the protocol block
"""
# First obtain the options that are available
result = getSiteForResource('Storage', storageName)
if not result['OK']:
return result
site = result['Value']
result = self.resourcesHelper.getNodeOptionsDict(
site, 'Storage', storageName, protocol)
if not result['OK']:
return result
optionsDict = result['Value']
# We must have certain values internally even if not supplied in CS
protocolDict = {
'Access': '',
'Host': '',
'Path': '',
'Port': '',
'Protocol': '',
'ProtocolName': '',
'SpaceToken': '',
'WSUrl': ''
}
for option in optionsDict:
protocolDict[option] = optionsDict[option]
# Now update the local and remote protocol lists.
# A warning will be given if the Access option is not set.
if protocolDict['Access'] == 'remote':
self.remoteProtocols.append(protocolDict['ProtocolName'])
elif protocolDict['Access'] == 'local':
self.localProtocols.append(protocolDict['ProtocolName'])
else:
errStr = "StorageFactory.__getProtocolDetails: The 'Access' option for %s:%s is neither 'local' or 'remote'." % (
storageName, protocol)
gLogger.warn(errStr)
# The ProtocolName option must be defined
if not protocolDict['ProtocolName']:
errStr = "StorageFactory.__getProtocolDetails: 'ProtocolName' option is not defined."
gLogger.error(errStr, "%s: %s" % (storageName, protocol))
return S_ERROR(errStr)
return S_OK(protocolDict)
###########################################################################################
#
# Below is the method for obtaining the object instantiated for a provided storage configuration
#
def __generateStorageObject(self,
storageName,
protocolName,
protocol,
path=None,
host=None,
port=None,
spaceToken=None,
wsUrl=None):
storageType = protocolName
if self.proxy:
storageType = 'Proxy'
moduleRootPaths = getInstalledExtensions()
moduleLoaded = False
path = path.rstrip('/')
if not path:
path = '/'
for moduleRootPath in moduleRootPaths:
if moduleLoaded:
break
gLogger.verbose("Trying to load from root path %s" %
moduleRootPath)
moduleFile = os.path.join(rootPath, moduleRootPath, "Resources",
"Storage", "%sStorage.py" % storageType)
gLogger.verbose("Looking for file %s" % moduleFile)
if not os.path.isfile(moduleFile):
continue
try:
# This inforces the convention that the plug in must be named after the protocol
moduleName = "%sStorage" % (storageType)
storageModule = __import__(
'%s.Resources.Storage.%s' % (moduleRootPath, moduleName),
globals(), locals(), [moduleName])
except Exception, x:
errStr = "StorageFactory._generateStorageObject: Failed to import %s: %s" % (
storageName, x)
gLogger.exception(errStr)
return S_ERROR(errStr)
try:
evalString = "storageModule.%s(storageName,protocol,path,host,port,spaceToken,wsUrl)" % moduleName
storage = eval(evalString)
if not storage.isOK():
errStr = "StorageFactory._generateStorageObject: Failed to instantiate storage plug in."
gLogger.error(errStr, "%s" % (moduleName))
return S_ERROR(errStr)
except Exception, x:
errStr = "StorageFactory._generateStorageObject: Failed to instantiate %s(): %s" % (
moduleName, x)
gLogger.exception(errStr)
return S_ERROR(errStr)
# If use proxy, keep the original protocol name
if self.proxy:
storage.protocolName = protocolName
return S_OK(storage)
def __lookForCE(self):
knownces = self.am_getOption('BannedCEs', [])
resources = Resources(self.voName)
result = resources.getEligibleResources('Computing',
{'CEType': ['LCG', 'CREAM']})
if not result['OK']:
return result
siteDict = result['Value']
for site in siteDict:
knownces += siteDict[site]
# result = gConfig.getSections( '/Resources/Sites' )
# if not result['OK']:
# return
# grids = result['Value']
#
# for grid in grids:
#
# result = gConfig.getSections( '/Resources/Sites/%s' % grid )
# if not result['OK']:
# return
# sites = result['Value']
#
# for site in sites:
# opt = gConfig.getOptionsDict( '/Resources/Sites/%s/%s' % ( grid, site ) )['Value']
# ces = List.fromChar( opt.get( 'CE', '' ) )
# knownces += ces
response = ldapCEState('', vo=self.voName)
if not response['OK']:
self.log.error("Error during BDII request", response['Message'])
response = self.__checkAlternativeBDIISite(ldapCEState, '',
self.voName)
return response
newces = {}
for queue in response['Value']:
try:
queuename = queue['GlueCEUniqueID']
except:
continue
cename = queuename.split(":")[0]
if not cename in knownces:
newces[cename] = None
self.log.debug("newce", cename)
body = ""
possibleNewSites = []
for ce in newces.iterkeys():
response = ldapCluster(ce)
if not response['OK']:
self.log.warn("Error during BDII request", response['Message'])
response = self.__checkAlternativeBDIISite(ldapCluster, ce)
continue
clusters = response['Value']
if len(clusters) != 1:
self.log.warn("Error in cluster length",
" CE %s Length %d" % (ce, len(clusters)))
if len(clusters) == 0:
continue
cluster = clusters[0]
fkey = cluster.get('GlueForeignKey', [])
if type(fkey) == type(''):
fkey = [fkey]
nameBDII = None
for entry in fkey:
if entry.count('GlueSiteUniqueID'):
nameBDII = entry.split('=')[1]
break
if not nameBDII:
continue
cestring = "CE: %s, GOCDB Name: %s" % (ce, nameBDII)
self.log.info(cestring)
response = ldapCE(ce)
if not response['OK']:
self.log.warn("Error during BDII request", response['Message'])
response = self.__checkAlternativeBDIISite(ldapCE, ce)
continue
ceinfos = response['Value']
if len(ceinfos):
ceinfo = ceinfos[0]
systemName = ceinfo.get('GlueHostOperatingSystemName',
'Unknown')
systemVersion = ceinfo.get('GlueHostOperatingSystemVersion',
'Unknown')
systemRelease = ceinfo.get('GlueHostOperatingSystemRelease',
'Unknown')
else:
systemName = "Unknown"
systemVersion = "Unknown"
systemRelease = "Unknown"
osstring = "SystemName: %s, SystemVersion: %s, SystemRelease: %s" % (
systemName, systemVersion, systemRelease)
self.log.info(osstring)
response = ldapCEState(ce, vo=self.voName)
if not response['OK']:
self.log.warn("Error during BDII request", response['Message'])
response = self.__checkAlternativeBDIISite(
ldapCEState, ce, self.voName)
continue
newcestring = "\n\n%s\n%s" % (cestring, osstring)
usefull = False
cestates = response['Value']
for cestate in cestates:
queuename = cestate.get('GlueCEUniqueID', 'UnknownName')
queuestatus = cestate.get('GlueCEStateStatus', 'UnknownStatus')
queuestring = "%s %s" % (queuename, queuestatus)
self.log.info(queuestring)
newcestring += "\n%s" % queuestring
if queuestatus.count('Production'):
usefull = True
if usefull:
body += newcestring
possibleNewSites.append(
'dirac-admin-add-site DIRACSiteName %s %s' %
(nameBDII, ce))
if body:
body = "We are glade to inform You about new CE(s) possibly suitable for %s:\n" % self.voName + body
body += "\n\nTo suppress information about CE add its name to BannedCEs list."
for possibleNewSite in possibleNewSites:
body = "%s\n%s" % (body, possibleNewSite)
self.log.info(body)
if self.addressTo and self.addressFrom:
notification = NotificationClient()
result = notification.sendMail(self.addressTo,
self.subject,
body,
self.addressFrom,
localAttempt=False)
return S_OK()
import DIRAC
from DIRAC import gConfig,gLogger
from DIRAC.ResourceStatusSystem.Client.ResourceStatus import ResourceStatus
from DIRAC.Core.Utilities.List import sortList
from DIRAC.ConfigurationSystem.Client.Helpers.Resources import Resources
from DIRAC.Core.Security.ProxyInfo import getVOfromProxyGroup
if __name__ == "__main__":
result = getVOfromProxyGroup()
if not result['OK']:
gLogger.notice( 'Error:', result['Message'] )
DIRAC.exit( 1 )
vo = result['Value']
resources = Resources( vo = vo )
result = resources.getEligibleStorageElements()
if not result['OK']:
gLogger.notice( 'Error:', result['Message'] )
DIRAC.exit( 2 )
seList = sortList( result[ 'Value' ] )
resourceStatus = ResourceStatus()
result = resourceStatus.getStorageStatus( seList )
if not result['OK']:
gLogger.notice( 'Error:', result['Message'] )
DIRAC.exit( 3 )
for k,v in result[ 'Value' ].items():
print 'ERROR: Could not contact Configuration Service'
exitCode = 2
DIRAC.exit( exitCode )
res = getProxyInfo()
if not res[ 'OK' ]:
gLogger.error( 'Failed to get proxy information', res[ 'Message' ] )
DIRAC.exit( 2 )
userName = res['Value'].get( 'username' )
if not userName:
gLogger.error( 'Failed to get username for proxy' )
DIRAC.exit( 2 )
if site:
res = Resources().getStorageElements( site )
if not res[ 'OK' ]:
gLogger.error( 'The provided site (%s) is not known.' % site )
DIRAC.exit( -1 )
ses.extend( res[ 'Value' ][ 'SE' ].replace( ' ', '' ).split( ',' ) )
if not ses:
gLogger.error( 'There were no SEs provided' )
DIRAC.exit( -1 )
readBanned = []
writeBanned = []
checkBanned = []
resourceStatus = ResourceStatus()
class ResourcesTestCase( unittest.TestCase ):
def setUp( self ):
Script.disableCS( )
Script.parseCommandLine()
self.resources = Resources()
def test_getSites( self ):
print
result = self.resources.getSites( {'Name':['CERN','CPPM','PNPI']} )
self.assertTrue( result['OK'], 'getSites' )
sites = result['Value']
print sites
result = self.resources.getEligibleSites( {'Name':['CERN','CPPM','PNPI']} )
self.assertTrue( result['OK'], 'getEligibleSites' )
eligibleSites = result['Value']
self.assertEqual(sites, eligibleSites, 'sites and eligible sites are the same')
def test_getResources( self ):
print
result = self.resources.getResources( 'CERN', 'Storage' )
self.assertTrue( result['OK'], 'getResources' )
ses = result['Value']
print ses
def test_getNodes( self ):
print
result = self.resources.getNodes( 'CERN::ce130', 'Queue' )
self.assertTrue( result['OK'], 'getNodes' )
nodes = result['Value']
print nodes
def test_getEligibleResources( self ):
print
result = self.resources.getEligibleResources( 'Computing', { 'Site':['CERN','CPPM','Zurich'],'SubmissionMode':'Direct' } )
self.assertTrue( result['OK'], 'getEligibleResources' )
ces = result['Value']
print ces
def test_getEligibleNodes( self ):
print
result = self.resources.getEligibleNodes( 'AccessProtocol',
{ 'Site':['CERN','CPPM','Zurich'] },
{ 'Protocol':'srm' } )
self.assertTrue( result['OK'], 'getEligibleNodes' )
aps = result['Value']
print aps
def test_getEligibleComputingElements( self ):
siteMask = ['LCG.CERN.ch','LCG.CPPM.fr']
result = self.resources.getEligibleResources( 'Computing', {'Site':siteMask,
'SubmissionMode':'gLite',
'CEType':['LCG','CREAM']} )
self.assertTrue( result['OK'], 'getEligibleResources' )
print
for ce in result['Value']:
ceHost = self.resources.getComputingElementValue( ce, 'Host', 'unknown' )
print ce, ceHost
def initSEs():
'''
Initializes SEs statuses taking their values from the CS.
'''
subLogger.info( 'Initializing SEs' )
resources = Resources()
ses = resources.getEligibleStorageElements()
if not ses[ 'OK' ]:
return ses
ses = ses[ 'Value' ]
statuses = StateMachine.RSSMachine( None ).getStates()
statusTypes = RssConfiguration.RssConfiguration().getConfigStatusType( 'StorageElement' )
reason = 'dirac-rss-sync'
subLogger.debug( statuses )
subLogger.debug( statusTypes )
rssClient = ResourceStatusClient.ResourceStatusClient()
for se in ses:
subLogger.debug( se )
#opts = gConfig.getOptionsDict( '/Resources/StorageElements/%s' % se )
opts = resources.getStorageElementOptionsDict( se )
if not opts[ 'OK' ]:
subLogger.warn( opts[ 'Message' ] )
continue
opts = opts[ 'Value' ]
subLogger.debug( opts )
# We copy the list into a new object to remove items INSIDE the loop !
statusTypesList = statusTypes[:]
for statusType, status in opts.iteritems():
#Sanity check...
if not statusType in statusTypesList:
continue
#Transforms statuses to RSS terms
if status in ( 'NotAllowed', 'InActive' ):
status = 'Banned'
if not status in statuses:
subLogger.error( '%s not a valid status for %s - %s' % ( status, se, statusType ) )
continue
# We remove from the backtracking
statusTypesList.remove( statusType )
subLogger.debug( [ se,statusType,status,reason ] )
result = rssClient.modifyStatusElement( 'Resource', 'Status', name = se,
statusType = statusType, status = status,
reason = reason )
if not result[ 'OK' ]:
subLogger.error( 'Failed to modify' )
subLogger.error( result[ 'Message' ] )
continue
#Backtracking: statusTypes not present on CS
for statusType in statusTypesList:
result = rssClient.modifyStatusElement( 'Resource', 'Status', name = se,
statusType = statusType, status = DEFAULT_STATUS,
reason = reason )
if not result[ 'OK' ]:
subLogger.error( 'Error in backtracking for %s,%s,%s' % ( se, statusType, status ) )
subLogger.error( result[ 'Message' ] )
return S_OK()
def setUp( self ): Script.disableCS( ) Script.parseCommandLine() self.resources = Resources()
import DIRAC
from DIRAC import gConfig, gLogger
from DIRAC.ResourceStatusSystem.Client.ResourceStatus import ResourceStatus
from DIRAC.Core.Utilities.List import sortList
from DIRAC.ConfigurationSystem.Client.Helpers.Resources import Resources
from DIRAC.Core.Security.ProxyInfo import getVOfromProxyGroup
if __name__ == "__main__":
result = getVOfromProxyGroup()
if not result['OK']:
gLogger.notice('Error:', result['Message'])
DIRAC.exit(1)
vo = result['Value']
resources = Resources(vo=vo)
result = resources.getEligibleStorageElements()
if not result['OK']:
gLogger.notice('Error:', result['Message'])
DIRAC.exit(2)
seList = sortList(result['Value'])
resourceStatus = ResourceStatus()
result = resourceStatus.getStorageStatus(seList)
if not result['OK']:
gLogger.notice('Error:', result['Message'])
DIRAC.exit(3)
for k, v in result['Value'].items():
def _resolveCECandidates( self, taskQueueDict ):
"""
Return a list of CEs for this TaskQueue
"""
# assume user knows what they're doing and avoid site mask e.g. sam jobs
if 'GridCEs' in taskQueueDict and taskQueueDict['GridCEs']:
self.log.info( 'CEs requested by TaskQueue %s:' % taskQueueDict['TaskQueueID'],
', '.join( taskQueueDict['GridCEs'] ) )
return taskQueueDict['GridCEs']
# Get the mask
siteStatus = SiteStatus()
ret = siteStatus.getUsableSites( 'ComputingAccess' )
if not ret['OK']:
self.log.error( 'Can not retrieve site Mask from DB:', ret['Message'] )
return []
usableSites = ret['Value']
if not usableSites:
self.log.error( 'Site mask is empty' )
return []
self.log.verbose( 'Site Mask: %s' % ', '.join( usableSites ) )
# remove banned sites from siteMask
if 'BannedSites' in taskQueueDict:
for site in taskQueueDict['BannedSites']:
if site in usableSites:
usableSites.remove( site )
self.log.verbose( 'Removing banned site %s from site Mask' % site )
# remove from the mask if a Site is given
siteMask = [ site for site in usableSites if 'Sites' not in taskQueueDict or site in taskQueueDict['Sites'] ]
if not siteMask:
# pilot can not be submitted
self.log.info( 'No Valid Site Candidate in Mask for TaskQueue %s' % taskQueueDict['TaskQueueID'] )
return []
self.log.info( 'Site Candidates for TaskQueue %s:' % taskQueueDict['TaskQueueID'], ', '.join( siteMask ) )
# Get CE's associates to the given site Names
ceMask = []
resources = Resources( vo = self.virtualOrganization )
result = resources.getEligibleResources( 'Computing', {'Site':siteMask,
'SubmissionMode':'gLite',
'CEType':['LCG','CREAM']} )
if not result['OK']:
self.log.error( "Failed to get eligible ce's:", result['Message'] )
return []
ces = result['Value']
for ce in ces:
ceHost = resources.getComputingElementValue( ce, 'Host', 'unknown' )
if ceHost != 'unknown':
ceMask.append( ceHost )
if not ceMask:
self.log.info( 'No CE Candidate found for TaskQueue %s:' % taskQueueDict['TaskQueueID'], ', '.join( siteMask ) )
self.log.verbose( 'CE Candidates for TaskQueue %s:' % taskQueueDict['TaskQueueID'], ', '.join( ceMask ) )
return ceMask
def __init__( self, name, protocols = None, vo = None ):
""" c'tor
:param str name: SE name
:param list protocols: requested protocols
"""
self.vo = vo
if not vo:
result = getVOfromProxyGroup()
if not result['OK']:
return result
self.vo = result['Value']
self.opHelper = Operations( vo = self.vo )
self.resources = Resources( vo = self.vo )
proxiedProtocols = gConfig.getValue( '/LocalSite/StorageElements/ProxyProtocols', "" ).split( ',' )
result = self.resources.getAccessProtocols( name )
if result['OK']:
ap = result['Value'][0]
useProxy = ( self.resources.getAccessProtocolValue( ap, "Protocol", "UnknownProtocol" )
in proxiedProtocols )
#print "Proxy", name, proxiedProtocols, \
#gConfig.getValue( "/Resources/StorageElements/%s/AccessProtocol.1/Protocol" % name, "xxx" )
if not useProxy:
useProxy = gConfig.getValue( '/LocalSite/StorageElements/%s/UseProxy' % name, False )
if not useProxy:
useProxy = self.opHelper.getValue( '/Services/StorageElements/%s/UseProxy' % name, False )
self.valid = True
if protocols == None:
res = StorageFactory( useProxy ).getStorages( name, protocolList = [] )
else:
res = StorageFactory( useProxy ).getStorages( name, protocolList = protocols )
if not res['OK']:
self.valid = False
self.name = name
self.errorReason = res['Message']
else:
factoryDict = res['Value']
self.name = factoryDict['StorageName']
self.options = factoryDict['StorageOptions']
self.localProtocols = factoryDict['LocalProtocols']
self.remoteProtocols = factoryDict['RemoteProtocols']
self.storages = factoryDict['StorageObjects']
self.protocolOptions = factoryDict['ProtocolOptions']
self.turlProtocols = factoryDict['TurlProtocols']
self.log = gLogger.getSubLogger( "SE[%s]" % self.name )
self.readMethods = [ 'getFile',
'getAccessUrl',
'getTransportURL',
'prestageFile',
'prestageFileStatus',
'getDirectory']
self.writeMethods = [ 'retransferOnlineFile',
'putFile',
'replicateFile',
'pinFile',
'releaseFile',
'createDirectory',
'putDirectory' ]
self.removeMethods = [ 'removeFile', 'removeDirectory' ]
self.checkMethods = [ 'exists',
'getDirectoryMetadata',
'getDirectorySize',
'getFileSize',
'getFileMetadata',
'listDirectory',
'isDirectory',
'isFile',
]
self.okMethods = [ 'getLocalProtocols',
'getPfnForProtocol',
'getPfnForLfn',
'getPfnPath',
'getProtocols',
'getRemoteProtocols',
'getStorageElementName',
'getStorageElementOption',
'getStorageParameters',
'isLocalSE' ]
self.__resourceStatus = ResourceStatus()
class StorageFactory:
def __init__(self, useProxy=False, vo=None):
self.valid = True
self.proxy = False
self.proxy = useProxy
self.resourceStatus = ResourceStatus()
self.resourcesHelper = Resources(vo=vo)
###########################################################################################
#
# Below are public methods for obtaining storage objects
#
def getStorageName(self, initialName):
return self._getConfigStorageName(initialName)
def getStorage(self, parameterDict):
""" This instantiates a single storage for the details provided and doesn't check the CS.
"""
# The storage name must be supplied.
if parameterDict.has_key("StorageName"):
storageName = parameterDict["StorageName"]
else:
errStr = "StorageFactory.getStorage: StorageName must be supplied"
gLogger.error(errStr)
return S_ERROR(errStr)
# ProtocolName must be supplied otherwise nothing with work.
if parameterDict.has_key("ProtocolName"):
protocolName = parameterDict["ProtocolName"]
else:
errStr = "StorageFactory.getStorage: ProtocolName must be supplied"
gLogger.error(errStr)
return S_ERROR(errStr)
# The other options need not always be specified
if parameterDict.has_key("Protocol"):
protocol = parameterDict["Protocol"]
else:
protocol = ""
if parameterDict.has_key("Port"):
port = parameterDict["Port"]
else:
port = ""
if parameterDict.has_key("Host"):
host = parameterDict["Host"]
else:
host = ""
if parameterDict.has_key("Path"):
path = parameterDict["Path"]
else:
path = ""
if parameterDict.has_key("SpaceToken"):
spaceToken = parameterDict["SpaceToken"]
else:
spaceToken = ""
if parameterDict.has_key("WSUrl"):
wsPath = parameterDict["WSUrl"]
else:
wsPath = ""
return self.__generateStorageObject(
storageName, protocolName, protocol, path, host, port, spaceToken, wsPath, parameterDict
)
def getStorages(self, storageName, protocolList=[]):
""" Get an instance of a Storage based on the DIRAC SE name based on the CS entries CS
'storageName' is the DIRAC SE name i.e. 'CERN-RAW'
'protocolList' is an optional list of protocols if a sub-set is desired i.e ['SRM2','SRM1']
"""
self.remoteProtocols = []
self.localProtocols = []
self.name = ""
self.options = {}
self.protocolDetails = []
self.storages = []
# Get the name of the storage provided
res = self._getConfigStorageName(storageName)
if not res["OK"]:
self.valid = False
return res
storageName = res["Value"]
self.name = storageName
# Get the options defined in the CS for this storage
res = self._getConfigStorageOptions(storageName)
if not res["OK"]:
self.valid = False
return res
self.options = res["Value"]
# Get the protocol specific details
res = self._getConfigStorageProtocols(storageName)
if not res["OK"]:
self.valid = False
return res
self.protocolDetails = res["Value"]
requestedLocalProtocols = []
requestedRemoteProtocols = []
requestedProtocolDetails = []
turlProtocols = []
# Generate the protocol specific plug-ins
self.storages = []
for protocolDict in self.protocolDetails:
protocolName = protocolDict["ProtocolName"]
protocolRequested = True
if protocolList:
if protocolName not in protocolList:
protocolRequested = False
if protocolRequested:
protocol = protocolDict["Protocol"]
host = protocolDict["Host"]
path = protocolDict["Path"]
port = protocolDict["Port"]
spaceToken = protocolDict["SpaceToken"]
wsUrl = protocolDict["WSUrl"]
res = self.__generateStorageObject(
storageName,
protocolName,
protocol,
path=path,
host=host,
port=port,
spaceToken=spaceToken,
wsUrl=wsUrl,
parameters=protocolDict,
)
if res["OK"]:
self.storages.append(res["Value"])
if protocolName in self.localProtocols:
turlProtocols.append(protocol)
requestedLocalProtocols.append(protocolName)
if protocolName in self.remoteProtocols:
requestedRemoteProtocols.append(protocolName)
requestedProtocolDetails.append(protocolDict)
else:
gLogger.info(res["Message"])
if len(self.storages) > 0:
resDict = {}
resDict["StorageName"] = self.name
resDict["StorageOptions"] = self.options
resDict["StorageObjects"] = self.storages
resDict["LocalProtocols"] = requestedLocalProtocols
resDict["RemoteProtocols"] = requestedRemoteProtocols
resDict["ProtocolOptions"] = requestedProtocolDetails
resDict["TurlProtocols"] = turlProtocols
return S_OK(resDict)
else:
errStr = "StorageFactory.getStorages: Failed to instantiate any storage protocols."
gLogger.error(errStr, self.name)
return S_ERROR(errStr)
###########################################################################################
#
# Below are internal methods for obtaining section/option/value configuration
#
def _getConfigStorageName(self, storageName):
"""
This gets the name of the storage the configuration service.
If the storage is an alias for another the resolution is performed.
'storageName' is the storage section to check in the CS
"""
result = self.resourcesHelper.getStorageElementOptionsDict(storageName)
if not result["OK"]:
errStr = "StorageFactory._getConfigStorageName: Failed to get storage options"
gLogger.error(errStr, result["Message"])
return S_ERROR(errStr)
if not result["Value"]:
errStr = "StorageFactory._getConfigStorageName: Supplied storage doesn't exist."
gLogger.error(errStr, storageName)
return S_ERROR(errStr)
if "Alias" in res["Value"]:
configPath = "%s/%s/Alias" % (self.rootConfigPath, storageName)
aliasName = gConfig.getValue(configPath)
result = self._getConfigStorageName(aliasName)
if not result["OK"]:
errStr = "StorageFactory._getConfigStorageName: Supplied storage doesn't exist."
gLogger.error(errStr, configPath)
return S_ERROR(errStr)
resolvedName = result["Value"]
else:
resolvedName = storageName
return S_OK(resolvedName)
def _getConfigStorageOptions(self, storageName):
""" Get the options associated to the StorageElement as defined in the CS
"""
result = self.resourcesHelper.getStorageElementOptionsDict(storageName)
if not result["OK"]:
errStr = "StorageFactory._getStorageOptions: Failed to get storage options."
gLogger.error(errStr, "%s: %s" % (storageName, result["Message"]))
return S_ERROR(errStr)
optionsDict = result["Value"]
result = self.resourceStatus.getStorageStatus(storageName, "ReadAccess")
if not result["OK"]:
errStr = "StorageFactory._getStorageOptions: Failed to get storage status"
gLogger.error(errStr, "%s: %s" % (storageName, result["Message"]))
return S_ERROR(errStr)
# optionsDict.update( result[ 'Value' ][ storageName ] )
return S_OK(optionsDict)
def _getConfigStorageProtocols(self, storageName):
""" Protocol specific information is present as sections in the Storage configuration
"""
result = getSiteForResource(storageName)
if not result["OK"]:
return result
site = result["Value"]
result = self.resourcesHelper.getEligibleNodes("AccessProtocol", {"Site": site, "Resource": storageName})
if not result["OK"]:
return result
nodesList = result["Value"]
protocols = []
for node in nodesList:
protocols.append(node)
protocolDetails = []
for protocol in protocols:
result = self._getConfigStorageProtocolDetails(protocol)
if not result["OK"]:
return result
protocolDetails.append(result["Value"])
self.protocols = self.localProtocols + self.remoteProtocols
return S_OK(protocolDetails)
def _getConfigStorageProtocolDetails(self, protocol):
"""
Parse the contents of the protocol block
"""
result = self.resourcesHelper.getAccessProtocolOptionsDict(protocol)
if not result["OK"]:
return result
optionsDict = result["Value"]
# We must have certain values internally even if not supplied in CS
protocolDict = {
"Access": "",
"Host": "",
"Path": "",
"Port": "",
"Protocol": "",
"ProtocolName": "",
"SpaceToken": "",
"WSUrl": "",
}
for option in optionsDict:
protocolDict[option] = optionsDict[option]
# Now update the local and remote protocol lists.
# A warning will be given if the Access option is not set.
if protocolDict["Access"] == "remote":
self.remoteProtocols.append(protocolDict["ProtocolName"])
elif protocolDict["Access"] == "local":
self.localProtocols.append(protocolDict["ProtocolName"])
else:
errStr = (
"StorageFactory.__getProtocolDetails: The 'Access' option for %s is neither 'local' or 'remote'."
% protocol
)
gLogger.warn(errStr)
# The ProtocolName option must be defined
if not protocolDict["ProtocolName"]:
errStr = "StorageFactory.__getProtocolDetails: 'ProtocolName' option is not defined."
gLogger.error(errStr, "%s" % protocol)
return S_ERROR(errStr)
return S_OK(protocolDict)
###########################################################################################
#
# Below is the method for obtaining the object instantiated for a provided storage configuration
#
def __generateStorageObject(
self,
storageName,
protocolName,
protocol,
path=None,
host=None,
port=None,
spaceToken=None,
wsUrl=None,
parameters={},
):
storageType = protocolName
if self.proxy:
storageType = "Proxy"
moduleRootPaths = getInstalledExtensions()
moduleLoaded = False
path = path.rstrip("/")
if not path:
path = "/"
for moduleRootPath in moduleRootPaths:
if moduleLoaded:
break
gLogger.verbose("Trying to load from root path %s" % moduleRootPath)
moduleFile = os.path.join(rootPath, moduleRootPath, "Resources", "Storage", "%sStorage.py" % storageType)
gLogger.verbose("Looking for file %s" % moduleFile)
if not os.path.isfile(moduleFile):
continue
try:
# This inforces the convention that the plug in must be named after the protocol
moduleName = "%sStorage" % (storageType)
storageModule = __import__(
"%s.Resources.Storage.%s" % (moduleRootPath, moduleName), globals(), locals(), [moduleName]
)
except Exception, x:
errStr = "StorageFactory._generateStorageObject: Failed to import %s: %s" % (storageName, x)
gLogger.exception(errStr)
return S_ERROR(errStr)
try:
evalString = "storageModule.%s(storageName,protocol,path,host,port,spaceToken,wsUrl)" % moduleName
storage = eval(evalString)
if not storage.isOK():
errStr = "StorageFactory._generateStorageObject: Failed to instantiate storage plug in."
gLogger.error(errStr, "%s" % (moduleName))
return S_ERROR(errStr)
except Exception, x:
errStr = "StorageFactory._generateStorageObject: Failed to instantiate %s(): %s" % (moduleName, x)
gLogger.exception(errStr)
return S_ERROR(errStr)
# Set extra parameters if any
if parameters:
result = storage.setParameters(parameters)
if not result["OK"]:
return result
# If use proxy, keep the original protocol name
if self.proxy:
storage.protocolName = protocolName
return S_OK(storage)
def __lookForCE( self ):
knownces = self.am_getOption( 'BannedCEs', [] )
resources = Resources( self.voName )
result = resources.getEligibleResources( 'Computing', {'CEType':['LCG','CREAM'] } )
if not result['OK']:
return result
knownces = [ resources.getComputingElementValue( x, 'Host' ) for x in result['Value'] ]
# result = gConfig.getSections( '/Resources/Sites' )
# if not result['OK']:
# return
# grids = result['Value']
#
# for grid in grids:
#
# result = gConfig.getSections( '/Resources/Sites/%s' % grid )
# if not result['OK']:
# return
# sites = result['Value']
#
# for site in sites:
# opt = gConfig.getOptionsDict( '/Resources/Sites/%s/%s' % ( grid, site ) )['Value']
# ces = List.fromChar( opt.get( 'CE', '' ) )
# knownces += ces
response = ldapCEState( '', vo = self.voName )
if not response['OK']:
self.log.error( "Error during BDII request", response['Message'] )
response = self.__checkAlternativeBDIISite( ldapCEState, '', self.voName )
return response
newces = {}
for queue in response['Value']:
try:
queuename = queue['GlueCEUniqueID']
except:
continue
cename = queuename.split( ":" )[0]
if not cename in knownces:
newces[cename] = None
self.log.debug( "newce", cename )
body = ""
possibleNewSites = []
for ce in newces.iterkeys():
response = ldapCluster( ce )
if not response['OK']:
self.log.warn( "Error during BDII request", response['Message'] )
response = self.__checkAlternativeBDIISite( ldapCluster, ce )
continue
clusters = response['Value']
if len( clusters ) != 1:
self.log.warn( "Error in cluster length", " CE %s Length %d" % ( ce, len( clusters ) ) )
if len( clusters ) == 0:
continue
cluster = clusters[0]
fkey = cluster.get( 'GlueForeignKey', [] )
if type( fkey ) == type( '' ):
fkey = [fkey]
nameBDII = None
for entry in fkey:
if entry.count( 'GlueSiteUniqueID' ):
nameBDII = entry.split( '=' )[1]
break
if not nameBDII:
continue
cestring = "CE: %s, GOCDB Name: %s" % ( ce, nameBDII )
self.log.info( cestring )
response = ldapCE( ce )
if not response['OK']:
self.log.warn( "Error during BDII request", response['Message'] )
response = self.__checkAlternativeBDIISite( ldapCE, ce )
continue
ceinfos = response['Value']
if len( ceinfos ):
ceinfo = ceinfos[0]
systemName = ceinfo.get( 'GlueHostOperatingSystemName', 'Unknown' )
systemVersion = ceinfo.get( 'GlueHostOperatingSystemVersion', 'Unknown' )
systemRelease = ceinfo.get( 'GlueHostOperatingSystemRelease', 'Unknown' )
else:
systemName = "Unknown"
systemVersion = "Unknown"
systemRelease = "Unknown"
osstring = "SystemName: %s, SystemVersion: %s, SystemRelease: %s" % ( systemName, systemVersion, systemRelease )
self.log.info( osstring )
response = ldapCEState( ce, vo = self.voName )
if not response['OK']:
self.log.warn( "Error during BDII request", response['Message'] )
response = self.__checkAlternativeBDIISite( ldapCEState, ce, self.voName )
continue
newcestring = "\n\n%s\n%s" % ( cestring, osstring )
usefull = False
cestates = response['Value']
for cestate in cestates:
queuename = cestate.get( 'GlueCEUniqueID', 'UnknownName' )
queuestatus = cestate.get( 'GlueCEStateStatus', 'UnknownStatus' )
queuestring = "%s %s" % ( queuename, queuestatus )
self.log.info( queuestring )
newcestring += "\n%s" % queuestring
if queuestatus.count( 'Production' ):
usefull = True
if usefull:
body += newcestring
possibleNewSites.append( 'dirac-admin-add-site DIRACSiteName %s %s' % ( nameBDII, ce ) )
if body:
body = "We are glad to inform You about new CE(s) possibly suitable for %s:\n" % self.voName + body
body += "\n\nTo suppress information about CE add its name to BannedCEs list."
for possibleNewSite in possibleNewSites:
body = "%s\n%s" % ( body, possibleNewSite )
self.log.info( body )
if self.addressTo and self.addressFrom:
notification = NotificationClient()
result = notification.sendMail( self.addressTo, self.subject, body, self.addressFrom, localAttempt = False )
return S_OK()
class Synchronizer( object ):
'''
Every time there is a successful write on the CS, Synchronizer().sync() is
executed. It updates the database with the values on the CS.
'''
def __init__( self ):
"""
Constructor.
examples:
>>> s = Synchronizer()
"""
self.log = gLogger.getSubLogger( self.__class__.__name__ )
self.operations = Operations()
self.resources = Resources()
self.rStatus = ResourceStatusClient.ResourceStatusClient()
self.rssConfig = RssConfiguration()
self.diracAdmin = DiracAdmin()
def sync( self, _eventName, _params ):
'''
Main synchronizer method. It synchronizes the three types of elements: Sites,
Resources and Nodes. Each _syncX method returns a dictionary with the additions
and deletions.
examples:
>>> s.sync( None, None )
S_OK()
:Parameters:
**_eventName** - any
this parameter is ignored, but needed by caller function.
**_params** - any
this parameter is ignored, but needed by caller function.
:return: S_OK
'''
defSyncResult = { 'added' : [], 'deleted' : [] }
# Sites
syncSites = self._syncSites()
if not syncSites[ 'OK' ]:
self.log.error( syncSites[ 'Message' ] )
syncSites = ( syncSites[ 'OK' ] and syncSites[ 'Value' ] ) or defSyncResult
# Resources
syncResources = self._syncResources()
if not syncResources[ 'OK' ]:
self.log.error( syncResources[ 'Message' ] )
syncResources = ( syncResources[ 'OK' ] and syncResources[ 'Value' ] ) or defSyncResult
# Nodes
syncNodes = self._syncNodes()
if not syncNodes[ 'OK' ]:
self.log.error( syncNodes[ 'Message' ] )
syncNodes = ( syncNodes[ 'OK' ] and syncNodes[ 'Value' ] ) or defSyncResult
# Notify via email to :
self.notify( syncSites, syncResources, syncNodes )
return S_OK()
def notify( self, syncSites, syncResources, syncNodes ):
"""
Method sending email notification with the result of the synchronization. Email
is sent to Operations( EMail/Production ) email address.
examples:
>>> s.notify( {}, {}, {} )
>>> s.notify( { 'Site' : { 'added' : [], 'deleted' : [ 'RubbishSite' ] }, {}, {} )
>>> s.notify( { 'Site' : { 'added' : [], 'deleted' : [ 'RubbishSite' ] },
{ 'Computing : { 'added' : [ 'newCE01', 'newCE02' ], 'deleted' : [] }}, {} )
:Parameters:
**syncSites** - dict() ( keys: added, deleted )
dictionary with the sites added and deleted from the DB
**syncResources** - dict() ( keys: added, deleted )
dictionary with the resources added and deleted from the DB
**syncNodes** - dict() ( keys: added, deleted )
dictionary with the nodes added and deleted from the DB
:return: S_OK
"""
# Human readable summary
msgBody = self.getBody( syncSites, syncResources, syncNodes )
self.log.info( msgBody )
# Email addresses
toAddress = self.operations.getValue( 'EMail/Production', '' )
fromAddress = self.rssConfig.getConfigFromAddress( '' )
if toAddress and fromAddress and msgBody:
# Subject of the email
setup = gConfig.getValue( 'DIRAC/Setup' )
subject = '[RSS](%s) CS Synchronization' % setup
self.diracAdmin.sendMail( toAddress, subject, msgBody, fromAddress = fromAddress )
def getBody( self, syncSites, syncResources, syncNodes ):
"""
Method that given the outputs of the three synchronization methods builds a
human readable string.
examples:
>>> s.getBody( {}, {}, {} )
''
>>> s.getBody( { 'Site' : { 'added' : [], 'deleted' : [ 'RubbishSite' ] }, {}, {} )
'''
SITES:
Site:
deleted:1
RubbishSite
'''
>>> s.getBody( { 'Site' : { 'added' : [], 'deleted' : [ 'RubbishSite' ] },
{ 'Computing : { 'added' : [ 'newCE01', 'newCE02' ], 'deleted' : [] }}, {} )
'''
SITES:
Site:
deleted:1
RubbishSite
RESOURCES:
Computing:
added:2
newCE01
newCE02
'''
:Parameters:
**syncSites** - dict() ( keys: added, deleted )
dictionary with the sites added and deleted from the DB
**syncResources** - dict() ( keys: added, deleted )
dictionary with the resources added and deleted from the DB
**syncNodes** - dict() ( keys: added, deleted )
dictionary with the nodes added and deleted from the DB
:return: str
"""
syncMsg = ''
for element, syncResult in [ ( 'SITES', syncSites ), ( 'RESOURCES', syncResources ),
( 'NODES', syncNodes ) ]:
elementsMsg = ''
for elementType, elements in syncResult.items():
elementMsg = ''
if elements[ 'added' ]:
elementMsg += '\n %s added: %d \n' % ( elementType, len( elements[ 'added' ] ) )
elementMsg += ' ' + '\n '.join( elements[ 'added' ] )
if elements[ 'deleted' ]:
elementMsg += '\n %s deleted: %d \n' % ( elementType, len( elements[ 'deleted' ] ) )
elementMsg += ' ' + '\n '.join( elements[ 'deleted' ] )
if elementMsg:
elementsMsg += '\n\n%s:\n' % elementType
elementsMsg += elementMsg
if elementsMsg:
syncMsg += '\n\n%s:' % element + elementsMsg
return syncMsg
#.............................................................................
# Sync methods: Site, Resource & Node
def _syncSites( self ):
"""
Method that synchronizes sites ( using their canonical name: CERN.ch ) with
elementType = 'Site'. It gets from the CS the eligible site names and then
synchronizes them with the DB. If not on the DB, they are added. If in the DB
but not on the CS, they are deleted.
examples:
>> s._syncSites()
S_OK( { 'Site' : { 'added' : [], 'deleted' : [ 'RubbishSite' ] } } )
:return: S_OK( { 'Site' : { 'added' : [], 'deleted' : [] }} ) | S_ERROR
"""
# Get site names from the CS
foundSites = self.resources.getEligibleSites()
if not foundSites[ 'OK' ]:
return foundSites
sites = {}
# Synchronize with the DB
resSync = self.__dbSync( 'Site', 'Site', foundSites[ 'Value' ] )
if not resSync[ 'OK' ]:
self.log.error( 'Error synchronizing Sites' )
self.log.error( resSync[ 'Message' ] )
else:
sites = resSync[ 'Value' ]
return S_OK( { 'Site' : sites } )
def _syncResources( self ):
"""
Method that synchronizes resources as defined on RESOURCE_NODE_MAPPING dictionary
keys. It makes one sync round per key ( elementType ). Gets from the CS the
eligible Resource/<elementType> names and then synchronizes them with the DB.
If not on the DB, they are added. If in the DB but not on the CS, they are deleted.
examples:
>>> s._syncResources()
S_OK( { 'Computing' : { 'added' : [ 'newCE01', 'newCE02' ], 'deleted' : [] },
'Storage' : { 'added' : [], 'deleted' : [] },
... } )
:return: S_OK( { 'RESOURCE_NODE_MAPPINGKey1' : { 'added' : [], 'deleted' : [] }, ...} )
"""
resources = {}
# Iterate over the different elementTypes for Resource ( Computing, Storage... )
for elementType in RESOURCE_NODE_MAPPING.keys():
# Get Resource / <elementType> names from CS
foundResources = self.resources.getEligibleResources( elementType )
if not foundResources[ 'OK' ]:
self.log.error( foundResources[ 'Message' ] )
continue
# Translate CS result into a list
foundResources = foundResources[ 'Value' ]
# Synchronize with the DB
resSync = self.__dbSync( 'Resource', elementType, foundResources )
if not resSync[ 'OK' ]:
self.log.error( 'Error synchronizing %s %s' % ( 'Resource', elementType ) )
self.log.error( resSync[ 'Message' ] )
else:
resources[ elementType ] = resSync[ 'Value' ]
return S_OK( resources )
def _syncNodes( self ):
"""
Method that synchronizes resources as defined on RESOURCE_NODE_MAPPING dictionary
values. It makes one sync round per key ( elementType ). Gets from the CS the
eligible Node/<elementType> names and then synchronizes them with the DB.
If not on the DB, they are added. If in the DB but not on the CS, they are deleted.
examples:
>>> s._syncNodes()
S_OK( { 'Queue' : { 'added' : [], 'deleted' : [] },
... } )
:return: S_OK( { 'RESOURCE_NODE_MAPPINGValue1' : { 'added' : [], 'deleted' : [] }, ...} )
"""
nodes = {}
# Iterate over the different elementTypes for Node ( Queue, AccessProtocol... )
for elementType in RESOURCE_NODE_MAPPING.values():
# Get Node / <elementType> names from CS
foundNodes = self.resources.getEligibleNodes( elementType )
if not foundNodes[ 'OK' ]:
self.log.error( foundNodes[ 'Value' ] )
continue
# Translate CS result into a list : maps NodeName to SiteName<>NodeName to
# avoid duplicates
# Looong list comprehension, sorry !
foundNodes = [ '%s<>%s' % ( key, item ) for key, subDict in foundNodes[ 'Value' ].items()
for subList in subDict.values() for item in subList ]
# Synchronize with the DB
resSync = self.__dbSync( 'Node', elementType, foundNodes )
if not resSync[ 'OK' ]:
self.log.error( 'Error synchronizing %s %s' % ( 'Node', elementType ) )
self.log.error( resSync[ 'Message' ] )
else:
nodes[ elementType ] = resSync[ 'Value' ]
return S_OK( nodes )
#.............................................................................
# DB sync actions
def __dbSync( self, elementFamily, elementType, elementsCS ):
"""
Method synchronizing CS and DB. Compares <elementsCS> with <elementsDB>
given the elementFamily and elementType ( e.g. Resource / Computing ).
If there are missing elements in the DB, are inserted. If are missing elements
in the CS, are deleted from the DB. Note that the logs from the RSS DB
are kept ! ( just in case ).
:Parameters:
**elementFamily** - str
any of the valid element families : Site, Resource, Node
**elementType** - str
any of the valid element types for <elementFamily>
**elementsCS** - list
list with the elements for <elementFamily>/<elementType> found in the CS
:return: S_OK( { 'added' : [], 'deleted' : [] } ) | S_ERROR
"""
# deleted, added default response
syncRes = {
'deleted' : [],
'added' : [],
}
# Gets <elementFamily>/<elementType> elements from DB
elementsDB = self.rStatus.selectStatusElement( elementFamily, 'Status',
elementType = elementType,
meta = { 'columns' : [ 'name' ] } )
if not elementsDB[ 'OK' ]:
return elementsDB
elementsDB = [ elementDB[ 0 ] for elementDB in elementsDB[ 'Value' ] ]
# Elements in DB but not in CS -> to be deleted
toBeDeleted = list( set( elementsDB ).difference( set( elementsCS ) ) )
if toBeDeleted:
resDelete = self.__dbDelete( elementFamily, elementType, toBeDeleted )
if not resDelete[ 'OK' ]:
return resDelete
else:
syncRes[ 'deleted' ] = toBeDeleted
# Elements in CS but not in DB -> to be added
toBeAdded = list( set( elementsCS ).difference( set( elementsDB ) ) )
if toBeAdded:
resInsert = self.__dbInsert( elementFamily, elementType, toBeAdded )
if not resInsert[ 'OK' ]:
return resInsert
else:
syncRes[ 'added' ] = toBeAdded
return S_OK( syncRes )
def __dbDelete( self, elementFamily, elementType, toBeDeleted ):
"""
Method that given the elementFamily and elementType, deletes all entries
in the History and Status tables for the given elements in toBeDeleted ( all
their status Types ).
:Parameters:
**elementFamily** - str
any of the valid element families : Site, Resource, Node
**elementType** - str
any of the valid element types for <elementFamily>, just used for logging
purposes.
**toBeDeleted** - list
list with the elements to be deleted
:return: S_OK | S_ERROR
"""
self.log.info( 'Deleting %s %s:' % ( elementFamily, elementType ) )
self.log.info( toBeDeleted )
return self.rStatus._extermineStatusElement( elementFamily, toBeDeleted )
def __dbInsert( self, elementFamily, elementType, toBeAdded ):
"""
Method that given the elementFamily and elementType, adds all elements in
toBeAdded with their respective statusTypes, obtained from the CS. They
are synchronized with status 'Unknown' and reason 'Synchronized'.
:Parameters:
**elementFamily** - str
any of the valid element families : Site, Resource, Node
**elementType** - str
any of the valid element types for <elementFamily>
**toBeDeleted** - list
list with the elements to be added
:return: S_OK | S_ERROR
"""
self.log.info( 'Adding %s %s:' % ( elementFamily, elementType ) )
self.log.info( toBeAdded )
statusTypes = self.rssConfig.getConfigStatusType( elementType )
for element in toBeAdded:
for statusType in statusTypes:
resInsert = self.rStatus.addIfNotThereStatusElement( elementFamily, 'Status',
name = element,
statusType = statusType,
status = 'Unknown',
elementType = elementType,
reason = 'Synchronized')
if not resInsert[ 'OK' ]:
return resInsert
return S_OK()
#...............................................................................
#
# def _syncUsers( self ):
# '''
# Sync Users: compares CS with DB and does the necessary modifications.
# '''
#
# gLogger.verbose( '-- Synchronizing users --')
#
# usersCS = CSHelpers.getRegistryUsers()
# if not usersCS[ 'OK' ]:
# return usersCS
# usersCS = usersCS[ 'Value' ]
#
# gLogger.verbose( '%s users found in CS' % len( usersCS ) )
#
# usersDB = self.rManagement.selectUserRegistryCache( meta = { 'columns' : [ 'login' ] } )
# if not usersDB[ 'OK' ]:
# return usersDB
# usersDB = [ userDB[0] for userDB in usersDB[ 'Value' ] ]
#
# # Users that are in DB but not in CS
# toBeDeleted = list( set( usersDB ).difference( set( usersCS.keys() ) ) )
# gLogger.verbose( '%s users to be deleted' % len( toBeDeleted ) )
#
# # Delete users
# # FIXME: probably it is not needed since there is a DatabaseCleanerAgent
# for userLogin in toBeDeleted:
#
# deleteQuery = self.rManagement.deleteUserRegistryCache( login = userLogin )
#
# gLogger.verbose( '... %s' % userLogin )
# if not deleteQuery[ 'OK' ]:
# return deleteQuery
#
# # AddOrModify Users
# for userLogin, userDict in usersCS.items():
#
# _name = userDict[ 'DN' ].split( '=' )[ -1 ]
# _email = userDict[ 'Email' ]
#
# query = self.rManagement.addOrModifyUserRegistryCache( userLogin, _name, _email )
# gLogger.verbose( '-> %s' % userLogin )
# if not query[ 'OK' ]:
# return query
#
# return S_OK()
################################################################################
#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF
def export_killPilot(self, pilotRefList):
""" Kill the specified pilots
"""
# Make a list if it is not yet
pilotRefs = list(pilotRefList)
if type(pilotRefList) in StringTypes:
pilotRefs = [pilotRefList]
# Regroup pilots per site and per owner
pilotRefDict = {}
for pilotReference in pilotRefs:
result = pilotDB.getPilotInfo(pilotReference)
if not result['OK'] or not result['Value']:
return S_ERROR('Failed to get info for pilot ' +
pilotReference)
pilotDict = result['Value'][pilotReference]
owner = pilotDict['OwnerDN']
group = pilotDict['OwnerGroup']
queue = '@@@'.join([
owner, group, pilotDict['GridSite'],
pilotDict['DestinationSite'], pilotDict['Queue']
])
gridType = pilotDict['GridType']
pilotRefDict.setdefault(queue, {})
pilotRefDict[queue].setdefault('PilotList', [])
pilotRefDict[queue]['PilotList'].append(pilotReference)
pilotRefDict[queue]['GridType'] = gridType
# Do the work now queue by queue
ceFactory = ComputingElementFactory()
failed = []
for key, pilotDict in pilotRefDict.items():
owner, group, site, ce, queue = key.split('@@@')
result = Resources(group=group).getQueueDescription(queue)
if not result['OK']:
return result
queueDict = result['Value']
gridType = pilotDict['GridType']
result = ceFactory.getCE(gridType, ce, queueDict)
if not result['OK']:
return result
ce = result['Value']
if gridType in ["LCG", "gLite", "CREAM"]:
group = getGroupOption(group, 'VOMSRole', group)
ret = gProxyManager.getPilotProxyFromVOMSGroup(owner, group)
if not ret['OK']:
gLogger.error(ret['Message'])
gLogger.error('Could not get proxy:',
'User "%s", Group "%s"' % (owner, group))
return S_ERROR("Failed to get the pilot's owner proxy")
proxy = ret['Value']
ce.setProxy(proxy)
pilotList = pilotDict['PilotList']
result = ce.killJob(pilotList)
if not result['OK']:
failed.extend(pilotList)
if failed:
return S_ERROR('Failed to kill at least some pilots')
return S_OK()
class FileCatalog:
ro_methods = [
'exists', 'isLink', 'readLink', 'isFile', 'getFileMetadata',
'getReplicas', 'getReplicaStatus', 'getFileSize', 'isDirectory',
'getDirectoryReplicas', 'listDirectory', 'getDirectoryMetadata',
'getDirectorySize', 'getDirectoryContents', 'resolveDataset',
'getPathPermissions', 'getLFNForPFN', 'getUsers', 'getGroups',
'getFileUserMetadata'
]
write_methods = [
'createLink', 'removeLink', 'addFile', 'setFileStatus', 'addReplica',
'removeReplica', 'removeFile', 'setReplicaStatus', 'setReplicaHost',
'createDirectory', 'setDirectoryStatus', 'removeDirectory',
'removeDataset', 'removeFileFromDataset', 'createDataset'
]
def __init__(self, catalogs=[], vo=None):
""" Default constructor
"""
self.valid = True
self.timeout = 180
self.readCatalogs = []
self.writeCatalogs = []
self.vo = vo
if not vo:
result = getVOfromProxyGroup()
if not result['OK']:
return result
self.vo = result['Value']
self.opHelper = Operations(vo=self.vo)
self.reHelper = Resources(vo=self.vo)
if type(catalogs) in types.StringTypes:
catalogs = [catalogs]
if catalogs:
res = self._getSelectedCatalogs(catalogs)
else:
res = self._getCatalogs()
if not res['OK']:
self.valid = False
elif (len(self.readCatalogs) == 0) and (len(self.writeCatalogs) == 0):
self.valid = False
def isOK(self):
return self.valid
def getReadCatalogs(self):
return self.readCatalogs
def getWriteCatalogs(self):
return self.writeCatalogs
def __getattr__(self, name):
self.call = name
if name in FileCatalog.write_methods:
return self.w_execute
elif name in FileCatalog.ro_methods:
return self.r_execute
else:
raise AttributeError
def __checkArgumentFormat(self, path):
if type(path) in types.StringTypes:
urls = {path: False}
elif type(path) == types.ListType:
urls = {}
for url in path:
urls[url] = False
elif type(path) == types.DictType:
urls = path
else:
return S_ERROR(
"FileCatalog.__checkArgumentFormat: Supplied path is not of the correct format."
)
return S_OK(urls)
def w_execute(self, *parms, **kws):
""" Write method executor.
"""
successful = {}
failed = {}
failedCatalogs = []
fileInfo = parms[0]
res = self.__checkArgumentFormat(fileInfo)
if not res['OK']:
return res
fileInfo = res['Value']
allLfns = fileInfo.keys()
for catalogName, oCatalog, master in self.writeCatalogs:
method = getattr(oCatalog, self.call)
res = method(fileInfo, **kws)
if not res['OK']:
if master:
# If this is the master catalog and it fails we dont want to continue with the other catalogs
gLogger.error(
"FileCatalog.w_execute: Failed to execute %s on master catalog %s."
% (self.call, catalogName), res['Message'])
return res
else:
# Otherwise we keep the failed catalogs so we can update their state later
failedCatalogs.append((catalogName, res['Message']))
else:
for lfn, message in res['Value']['Failed'].items():
# Save the error message for the failed operations
if not failed.has_key(lfn):
failed[lfn] = {}
failed[lfn][catalogName] = message
if master:
# If this is the master catalog then we should not attempt the operation on other catalogs
fileInfo.pop(lfn)
for lfn, result in res['Value']['Successful'].items():
# Save the result return for each file for the successful operations
if not successful.has_key(lfn):
successful[lfn] = {}
successful[lfn][catalogName] = result
# This recovers the states of the files that completely failed i.e. when S_ERROR is returned by a catalog
for catalogName, errorMessage in failedCatalogs:
for lfn in allLfns:
if not failed.has_key(lfn):
failed[lfn] = {}
failed[lfn][catalogName] = errorMessage
resDict = {'Failed': failed, 'Successful': successful}
return S_OK(resDict)
def r_execute(self, *parms, **kws):
""" Read method executor.
"""
successful = {}
failed = {}
for catalogTuple in self.readCatalogs:
oCatalog = catalogTuple[1]
method = getattr(oCatalog, self.call)
res = method(*parms, **kws)
if res['OK']:
if 'Successful' in res['Value']:
for key, item in res['Value']['Successful'].items():
if not successful.has_key(key):
successful[key] = item
if failed.has_key(key):
failed.pop(key)
for key, item in res['Value']['Failed'].items():
if not successful.has_key(key):
failed[key] = item
if len(failed) == 0:
resDict = {'Failed': failed, 'Successful': successful}
return S_OK(resDict)
else:
return res
if (len(successful) == 0) and (len(failed) == 0):
return S_ERROR('Failed to perform %s from any catalog' % self.call)
resDict = {'Failed': failed, 'Successful': successful}
return S_OK(resDict)
###########################################################################################
#
# Below is the method for obtaining the objects instantiated for a provided catalogue configuration
#
def addCatalog(self, catalogName, mode="Write", master=False):
""" Add a new catalog with catalogName to the pool of catalogs in mode:
"Read","Write" or "ReadWrite"
"""
result = self._generateCatalogObject(catalogName)
if not result['OK']:
return result
oCatalog = result['Value']
if mode.lower().find("read") != -1:
self.readCatalogs.append((catalogName, oCatalog, master))
if mode.lower().find("write") != -1:
self.writeCatalogs.append((catalogName, oCatalog, master))
return S_OK()
def removeCatalog(self, catalogName):
""" Remove the specified catalog from the internal pool
"""
catalog_removed = False
for i in range(len(self.readCatalogs)):
catalog = self.readCatalogs[i][0]
if catalog == catalogName:
del self.readCatalogs[i]
catalog_removed = True
break
for i in range(len(self.writeCatalogs)):
catalog = self.writeCatalogs[i][0]
if catalog == catalogName:
del self.writeCatalogs[i]
catalog_removed = True
break
if catalog_removed:
return S_OK()
else:
return S_OK('Catalog does not exist')
def _getSelectedCatalogs(self, desiredCatalogs):
for catalogName in desiredCatalogs:
res = self._generateCatalogObject(catalogName)
if not res['OK']:
return res
oCatalog = res['Value']
self.readCatalogs.append((catalogName, oCatalog, True))
self.writeCatalogs.append((catalogName, oCatalog, True))
return S_OK()
def _getCatalogs(self):
# Get the eligible catalogs first
# First, look in the Operations, if nothing defined look in /Resources
result = self.opHelper.getSections('/Services/Catalogs')
fileCatalogs = []
operationsFlag = False
if result['OK']:
fileCatalogs = result['Value']
operationsFlag = True
else:
res = self.reHelper.getEligibleResources('Catalog')
if not res['OK']:
errStr = "FileCatalog._getCatalogs: Failed to get file catalog configuration."
gLogger.error(errStr, res['Message'])
return S_ERROR(errStr)
fileCatalogs = res['Value']
# Get the catalogs now
for catalogName in fileCatalogs:
res = self._getCatalogConfigDetails(catalogName)
if not res['OK']:
return res
catalogConfig = res['Value']
if operationsFlag:
result = self.opHelper.getOptionsDict('/Services/Catalogs/%s' %
catalogName)
if not result['OK']:
return result
catalogConfig.update(result['Value'])
if catalogConfig['Status'] == 'Active':
res = self._generateCatalogObject(catalogName)
if not res['OK']:
return res
oCatalog = res['Value']
master = catalogConfig['Master']
# If the catalog is read type
if re.search('Read', catalogConfig['AccessType']):
if master:
self.readCatalogs.insert(
0, (catalogName, oCatalog, master))
else:
self.readCatalogs.append(
(catalogName, oCatalog, master))
# If the catalog is write type
if re.search('Write', catalogConfig['AccessType']):
if master:
self.writeCatalogs.insert(
0, (catalogName, oCatalog, master))
else:
self.writeCatalogs.append(
(catalogName, oCatalog, master))
return S_OK()
def _getCatalogConfigDetails(self, catalogName):
# First obtain the options that are available
result = self.reHelper.getCatalogOptionsDict(catalogName)
if not result['OK']:
errStr = "FileCatalog._getCatalogConfigDetails: Failed to get catalog options"
gLogger.error(errStr, catalogName)
return S_ERROR(errStr)
catalogConfig = result['Value']
# The 'Status' option should be defined (default = 'Active')
if not catalogConfig.has_key('Status'):
warnStr = "FileCatalog._getCatalogConfigDetails: 'Status' option not defined"
gLogger.warn(warnStr, catalogName)
catalogConfig['Status'] = 'Active'
# The 'AccessType' option must be defined
if not catalogConfig.has_key('AccessType'):
errStr = "FileCatalog._getCatalogConfigDetails: Required option 'AccessType' not defined"
gLogger.error(errStr, catalogName)
return S_ERROR(errStr)
# Anything other than 'True' in the 'Master' option means it is not
if not catalogConfig.has_key('Master'):
catalogConfig['Master'] = False
elif catalogConfig['Master'] == 'True':
catalogConfig['Master'] = True
else:
catalogConfig['Master'] = False
return S_OK(catalogConfig)
def _generateCatalogObject(self, catalogName):
""" Create a file catalog object from its name and CS description
"""
useProxy = gConfig.getValue(
'/LocalSite/Catalogs/%s/UseProxy' % catalogName, False)
if not useProxy:
useProxy = self.opHelper.getValue(
'/Services/Catalogs/%s/UseProxy' % catalogName, False)
return FileCatalogFactory().createCatalog(catalogName, useProxy)