def __resolveFTSServer(self):
if not self.sourceSE:
return S_ERROR("Source SE not set")
if not self.targetSE:
return S_ERROR("Target SE not set")
res = getSitesForSE(self.sourceSE, 'LCG')
if not res['OK'] or not res['Value']:
return S_ERROR("Could not determine source site")
sourceSite = res['Value'][0]
res = getSitesForSE(self.targetSE, 'LCG')
if not res['OK'] or not res['Value']:
return S_ERROR("Could not determine target site")
targetSite = res['Value'][0]
if (sourceSite == 'LCG.CERN.ch') or (targetSite == 'LCG.CERN.ch'):
ep = 'LCG.CERN.ch'
else:
# Target site FTS server should be used
ep = targetSite
try:
configPath = '/Resources/FTSEndpoints/%s' % ep
endpointURL = gConfig.getValue(configPath)
if not endpointURL:
errStr = "FTSRequest.__resolveFTSEndpoint: Failed to find FTS endpoint, check CS entry for '%s'." % ep
return S_ERROR(errStr)
self.ftsServer = endpointURL
return S_OK(endpointURL)
except Exception, x:
return S_ERROR(
'FTSRequest.__resolveFTSEndpoint: Failed to obtain endpoint details from CS'
)
def __resolveFTSServer( self ):
if not self.sourceSE:
return S_ERROR( "Source SE not set" )
if not self.targetSE:
return S_ERROR( "Target SE not set" )
res = getSitesForSE( self.sourceSE, 'LCG' )
if not res['OK'] or not res['Value']:
return S_ERROR( "Could not determine source site" )
sourceSite = res['Value'][0]
res = getSitesForSE( self.targetSE, 'LCG' )
if not res['OK'] or not res['Value']:
return S_ERROR( "Could not determine target site" )
targetSite = res['Value'][0]
if ( sourceSite == 'LCG.CERN.ch' ) or ( targetSite == 'LCG.CERN.ch' ):
ep = 'LCG.CERN.ch'
else:
# Target site FTS server should be used
ep = targetSite
try:
configPath = '/Resources/FTSEndpoints/%s' % ep
endpointURL = gConfig.getValue( configPath )
if not endpointURL:
errStr = "FTSRequest.__resolveFTSEndpoint: Failed to find FTS endpoint, check CS entry for '%s'." % ep
return S_ERROR( errStr )
self.ftsServer = endpointURL
return S_OK( endpointURL )
except Exception, x:
return S_ERROR( 'FTSRequest.__resolveFTSEndpoint: Failed to obtain endpoint details from CS' )
def __resolveFTSServer(self):
"""
resolve FTS server to use, it should be the closest one from target SE
:param self: self reference
"""
from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getFTSServersForSites
if not self.targetSE:
return S_ERROR("Target SE not set")
res = getSitesForSE(self.targetSE)
if not res['OK'] or not res['Value']:
return S_ERROR("Could not determine target site")
targetSites = res['Value']
targetSite = ''
for targetSite in targetSites:
targetFTS = getFTSServersForSites([targetSite])
if targetFTS['OK']:
ftsTarget = targetFTS['Value'][targetSite]
if ftsTarget:
self.ftsServer = ftsTarget
return S_OK(self.ftsServer)
else:
return targetFTS
return S_ERROR('No FTS server found for %s' % targetSite)
def _BySE(self):
""" Matches using TargetSE. This is the standard plugin.
"""
destSites = set()
try:
seList = ['Unknown']
if self.params['TargetSE']:
if isinstance(self.params['TargetSE'], six.string_types):
seList = fromChar(self.params['TargetSE'])
elif isinstance(self.params['TargetSE'], list):
seList = self.params['TargetSE']
except KeyError:
pass
if not seList or seList == ['Unknown']:
return destSites
for se in seList:
res = getSitesForSE(se)
if not res['OK']:
gLogger.warn("Could not get Sites associated to SE",
res['Message'])
else:
thisSESites = res['Value']
if thisSESites:
# We make an OR of the possible sites
destSites.update(thisSESites)
gLogger.debug("Destinations: %s" % ','.join(destSites))
return destSites
def __resolveFTSServer( self ):
"""
resolve FTS server to use, it should be the closest one from target SE
:param self: self reference
"""
from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getFTSServersForSites
if not self.targetSE:
return S_ERROR( "Target SE not set" )
res = getSitesForSE( self.targetSE )
if not res['OK'] or not res['Value']:
return S_ERROR( "Could not determine target site" )
targetSites = res['Value']
targetSite = ''
for targetSite in targetSites:
targetFTS = getFTSServersForSites( [targetSite] )
if targetFTS['OK']:
ftsTarget = targetFTS['Value'][targetSite]
if ftsTarget:
self.ftsServer = ftsTarget
return S_OK( self.ftsServer )
else:
return targetFTS
return S_ERROR( 'No FTS server found for %s' % targetSite )
def _BySE( self ):
""" Matches using TargetSE. This is the standard plugin.
"""
destSites = set()
try:
seList = ['Unknown']
if self.params['TargetSE']:
if isinstance( self.params['TargetSE'], basestring ):
seList = fromChar( self.params['TargetSE'] )
elif isinstance( self.params['TargetSE'], list ):
seList = self.params['TargetSE']
except KeyError:
pass
if not seList or seList == ['Unknown']:
return destSites
for se in seList:
res = getSitesForSE( se )
if not res['OK']:
gLogger.warn( "Could not get Sites associated to SE", res['Message'] )
else:
thisSESites = res['Value']
if thisSESites:
# We make an OR of the possible sites
destSites.update( thisSESites )
gLogger.debug( "Destinations: %s" % ','.join ( destSites ) )
return destSites
def run(self):
'''
Checks it has the parameters it needs and tries to send an sms to the users
that apply.
'''
if self.url is None:
return S_ERROR('Slack URL not set')
# Minor security checks
element = self.decisionParams['element']
if element is None:
return S_ERROR('element should not be None')
name = self.decisionParams['name']
if name is None:
return S_ERROR('name should not be None')
statusType = self.decisionParams['statusType']
if statusType is None:
return S_ERROR('statusType should not be None')
previousStatus = self.decisionParams['status']
if previousStatus is None:
return S_ERROR('status should not be None')
status = self.enforcementResult['Status']
if status is None:
return S_ERROR('status should not be None')
reason = self.enforcementResult['Reason']
if reason is None:
return S_ERROR('reason should not be None')
if self.decisionParams['element'] == 'Site':
siteName = self.decisionParams['name']
else:
elementType = self.decisionParams['elementType']
if elementType == 'StorageElement':
siteName = getSitesForSE(name)
elif elementType == 'ComputingElement':
siteName = getSiteForCE(name)
else:
siteName = {'OK': True, 'Value': 'Unassigned'}
if not siteName['OK']:
self.log.error('Resource %s does not exist at any site: %s' % (name, siteName['Message']))
siteName = "Unassigned Resources"
elif not siteName['Value']:
siteName = "Unassigned Resources"
else:
siteName = siteName['Value'] if isinstance(siteName['Value'], basestring) else siteName['Value'][0]
message = "*{name}* _{statusType}_ --> _{status}_ \n{reason}".format(name=name,
statusType=statusType,
status=status,
reason=reason)
return self.sendSlackMessage(message)
def _getSiteForSE(cls, se):
"""Get site name for the given SE"""
result = getSitesForSE(se)
if not result["OK"]:
return result
if result["Value"]:
return S_OK(result["Value"][0])
return S_OK("")
def __getSitesForSE(self, seName):
result = self.__sitesForSE.get(seName)
if result == False:
result = getSitesForSE(seName)
if not result["OK"]:
return result
self.__sitesForSE.add(seName, 600, result)
return result
def _getSitesForSEs(cls, seList):
"""Get all the sites for the given SE list"""
sites = []
for se in seList:
result = getSitesForSE(se)
if result["OK"]:
sites += result["Value"]
return sites
def __getSitesForSE( self, seName ):
result = self.__sitesForSE.get( seName )
if result == False:
result = getSitesForSE( seName )
if not result['OK']:
return result
self.__sitesForSE.add( seName, 600, result )
return result
def run(self):
''' Checks it has the parameters it needs and writes the date to a cache file.
'''
# Minor security checks
element = self.decisionParams['element']
if element is None:
return S_ERROR('element should not be None')
name = self.decisionParams['name']
if name is None:
return S_ERROR('name should not be None')
statusType = self.decisionParams['statusType']
if statusType is None:
return S_ERROR('statusType should not be None')
previousStatus = self.decisionParams['status']
if previousStatus is None:
return S_ERROR('status should not be None')
status = self.enforcementResult['Status']
if status is None:
return S_ERROR('status should not be None')
reason = self.enforcementResult['Reason']
if reason is None:
return S_ERROR('reason should not be None')
if self.decisionParams['element'] == 'Site':
siteName = self.decisionParams['name']
else:
elementType = self.decisionParams['elementType']
if elementType == 'StorageElement':
siteName = getSitesForSE(name)
elif elementType == 'ComputingElement':
siteName = getSiteForCE(name)
else:
siteName = {'OK': True, 'Value': 'Unassigned'}
if not siteName['OK']:
self.log.error('Resource %s does not exist at any site: %s' % (name, siteName['Message']))
siteName = "Unassigned Resources"
elif not siteName['Value']:
siteName = "Unassigned Resources"
else:
siteName = siteName['Value'] if isinstance(siteName['Value'], basestring) else siteName['Value'][0]
# create record for insertion
recordDict = {}
recordDict['SiteName'] = siteName
recordDict['ResourceName'] = name
recordDict['Status'] = status
recordDict['PreviousStatus'] = previousStatus
recordDict['StatusType'] = statusType
return self.rsClient.insert('ResourceStatusCache', recordDict)
def __resolveFTSServer( self ):
"""
resolve FTS server to use, it should be the closest one from target SE
:param self: self reference
"""
if not self.sourceSE:
return S_ERROR( "Source SE not set" )
if not self.targetSE:
return S_ERROR( "Target SE not set" )
res = getSitesForSE( self.sourceSE, 'LCG' )
if not res['OK'] or not res['Value']:
return S_ERROR( "Could not determine source site" )
sourceSite = res['Value'][0]
res = getSitesForSE( self.targetSE, 'LCG' )
if not res['OK'] or not res['Value']:
return S_ERROR( "Could not determine target site" )
targetSite = res['Value'][0]
# CERN is a special case, handling incoming and outgoing transfers
if ( sourceSite == 'LCG.CERN.ch' ) or ( targetSite == 'LCG.CERN.ch' ):
res = self.__getFTSServer( 'LCG.CERN.ch' )
if res['OK']:
self.ftsServer = res['Value']
return S_OK( self.ftsServer )
else:
return res
else:
# Target site FTS server should be used
sourceFTS = self.__getFTSServer( sourceSite )
if sourceFTS['OK']:
ftsSource = sourceFTS['Value']
if 'fts3' in ftsSource:
self.ftsServer = ftsSource
return S_OK( self.ftsServer )
targetFTS = self.__getFTSServer( targetSite )
if targetFTS['OK']:
ftsTarget = targetFTS['Value']
if ftsTarget:
self.ftsServer = ftsTarget
return S_OK( self.ftsServer )
else:
return S_ERROR( 'No FTS server found for %s nor %s' % ( sourceSite, targetSite ) )
else:
return targetFTS
def _getSitesForSEs( self, seList ):
""" Get all the sites for the given SE list
"""
sites = []
for se in seList:
result = getSitesForSE( se, gridName = 'LCG' )
if result['OK']:
sites += result['Value']
return sites
def _getSiteForSE(cls, se):
""" Get site name for the given SE
"""
result = getSitesForSE(se)
if not result['OK']:
return result
if result['Value']:
return S_OK(result['Value'][0])
return S_OK('')
def _getSitesForSEs(cls, seList):
""" Get all the sites for the given SE list
"""
sites = []
for se in seList:
result = getSitesForSE(se, gridName="LCG")
if result["OK"]:
sites += result["Value"]
return sites
def _getSiteForSE( cls, se ):
""" Get site name for the given SE
"""
result = getSitesForSE( se )
if not result['OK']:
return result
if result['Value']:
return S_OK( result['Value'][0] )
return S_OK( '' )
def _getSitesForSEs( cls, seList ):
""" Get all the sites for the given SE list
"""
sites = []
for se in seList:
result = getSitesForSE( se )
if result['OK']:
sites += result['Value']
return sites
def _getSiteForSE(cls, se):
""" Get site name for the given SE
"""
result = getSitesForSE(se, gridName="LCG")
if not result["OK"]:
return result
if result["Value"]:
return S_OK(result["Value"][0])
return S_OK("")
def _getSitesForSEs(cls, seList):
""" Get all the sites for the given SE list
"""
sites = []
for se in seList:
result = getSitesForSE(se, gridName='LCG')
if result['OK']:
sites += result['Value']
return sites
def __getChannelSitesForSE(self, se):
res = getSitesForSE(se)
sites = []
if res['OK']:
for site in res['Value']:
s = site.split('.')
if len(s) > 1:
if not s[1] in sites:
sites.append(s[1])
return sites
def __getChannelSitesForSE( self, se ):
res = getSitesForSE( se )
sites = []
if res['OK']:
for site in res['Value']:
s = site.split( '.' )
if len( s ) > 1:
if not s[1] in sites:
sites.append( s[1] )
return sites
def _handleDestination(self, paramsDict, getSitesForSE=None):
""" Handle Sites and TargetSE in the parameters
"""
try:
sites = ['ANY']
if paramsDict['Site']:
# 'Site' comes from the XML and therefore is ; separated
sites = fromChar(paramsDict['Site'], sepChar=';')
except KeyError:
pass
try:
seList = ['Unknown']
if paramsDict['TargetSE']:
seList = fromChar(paramsDict['TargetSE'])
except KeyError:
pass
if not seList or seList == ['Unknown']:
return sites
# from now on we know there is some TargetSE requested
if not getSitesForSE:
from DIRAC.Core.Utilities.SiteSEMapping import getSitesForSE
seSites = []
for se in seList:
res = getSitesForSE(se)
if not res['OK']:
self.log.warn('Could not get Sites associated to SE',
res['Message'])
else:
thisSESites = res['Value']
if not thisSESites:
continue
if seSites == []:
seSites = copy.deepcopy(thisSESites)
else:
# We make an OR of the possible sites
for nSE in list(thisSESites):
if nSE not in seSites:
seSites.append(nSE)
# Now we need to make the AND with the sites, if defined
if sites == ['ANY']:
return seSites
else:
# Need to get the AND
for nSE in list(seSites):
if nSE not in sites:
seSites.remove(nSE)
return seSites
def _handleDestination( self, paramsDict, getSitesForSE = None ):
""" Handle Sites and TargetSE in the parameters
"""
try:
sites = ['ANY']
if paramsDict['Site']:
# 'Site' comes from the XML and therefore is ; separated
sites = fromChar( paramsDict['Site'], sepChar = ';' )
except KeyError:
pass
try:
seList = ['Unknown']
if paramsDict['TargetSE']:
seList = fromChar( paramsDict['TargetSE'] )
except KeyError:
pass
if not seList or seList == ['Unknown']:
return sites
# from now on we know there is some TargetSE requested
if not getSitesForSE:
from DIRAC.Core.Utilities.SiteSEMapping import getSitesForSE
seSites = []
for se in seList:
res = getSitesForSE( se )
if not res['OK']:
self.log.warn( 'Could not get Sites associated to SE', res['Message'] )
else:
thisSESites = res['Value']
if not thisSESites:
continue
if seSites == []:
seSites = copy.deepcopy( thisSESites )
else:
# We make an OR of the possible sites
for nSE in list( thisSESites ):
if nSE not in seSites:
seSites.append( nSE )
# Now we need to make the AND with the sites, if defined
if sites == ['ANY']:
return seSites
else:
# Need to get the AND
for nSE in list( seSites ):
if nSE not in sites:
seSites.remove( nSE )
return seSites
def __getChannelSitesForSE(self, storageElement):
"""Get sites for given storage element.
:param self: self reference
:param str storageElement: storage element name
"""
res = getSitesForSE(storageElement)
if not res["OK"]:
return []
sites = []
for site in res["Value"]:
siteName = site.split(".")
if len(siteName) > 1:
if not siteName[1] in sites:
sites.append(siteName[1])
return sites
def __getChannelSitesForSE( self, storageElement ):
"""Get sites for given storage element.
:param self: self reference
:param str storageElement: storage element name
"""
res = getSitesForSE( storageElement )
if not res["OK"]:
return []
sites = []
for site in res["Value"]:
siteName = site.split( "." )
if len( siteName ) > 1:
if not siteName[1] in sites:
sites.append( siteName[1] )
return sites
def __resolveFTSServer( self ):
"""
resolve FTS server to use, it should be the closest one from target SE
:param self: self reference
"""
if self.ftsVersion.upper() == 'FTS2':
from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getFTS2ServersForSites
if not self.targetSE:
return S_ERROR( "Target SE not set" )
res = getSitesForSE( self.targetSE )
if not res['OK'] or not res['Value']:
return S_ERROR( "Could not determine target site" )
targetSites = res['Value']
targetSite = ''
for targetSite in targetSites:
targetFTS = getFTS2ServersForSites( [targetSite] )
if targetFTS['OK']:
ftsTarget = targetFTS['Value'][targetSite]
if ftsTarget:
self.ftsServer = ftsTarget
return S_OK( self.ftsServer )
else:
return targetFTS
elif self.ftsVersion.upper() == 'FTS3':
from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getFTS3Servers
res = getFTS3Servers()
if not res['OK']:
return res
ftsServerList = res['Value']
if ftsServerList:
# Here we take the first one, regardless of the policy...
# Unclean but all this will disapear after refactoring the fts code
self.ftsServer = ftsServerList[0]
return S_OK( self.ftsServer )
else:
return S_ERROR( 'Unknown FTS version %s' % self.ftsVersion )
return S_ERROR( 'No FTS server found for %s' % targetSite )
def __resolveFTSServer( self ):
"""
resolve FTS server to use, it should be the closest one from target SE
:param self: self reference
"""
if self.ftsVersion.upper() == 'FTS2':
from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getFTS2ServersForSites
if not self.targetSE:
return S_ERROR( "Target SE not set" )
res = getSitesForSE( self.targetSE )
if not res['OK'] or not res['Value']:
return S_ERROR( "Could not determine target site" )
targetSites = res['Value']
targetSite = ''
for targetSite in targetSites:
targetFTS = getFTS2ServersForSites( [targetSite] )
if targetFTS['OK']:
ftsTarget = targetFTS['Value'][targetSite]
if ftsTarget:
self.ftsServer = ftsTarget
return S_OK( self.ftsServer )
else:
return targetFTS
elif self.ftsVersion.upper() == 'FTS3':
from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getFTS3Servers
res = getFTS3Servers()
if not res['OK']:
return res
ftsServerList = res['Value']
if ftsServerList:
# Here we take the first one, regardless of the policy...
# Unclean but all this will disapear after refactoring the fts code
self.ftsServer = ftsServerList[0]
return S_OK( self.ftsServer )
else:
return S_ERROR( 'Unknown FTS version %s' % self.ftsVersion )
return S_ERROR( 'No FTS server found for %s' % targetSite )
def __getSitesForSE(self, se):
""" Returns a list of sites having the given SE as a local one.
Uses the local cache of the site-se information
"""
# Empty the cache if too old
if (time.time() - self.lastCScheck) > self.cacheLength:
self.log.verbose('Resetting the SE to site mapping cache')
self.seToSiteMapping = {}
self.lastCScheck = time.time()
if se not in self.seToSiteMapping:
sites = getSitesForSE(se)
if sites['OK']:
self.seToSiteMapping[se] = list(sites['Value'])
return sites
else:
return S_OK(self.seToSiteMapping[se])
def __getSitesForSE( self, se ):
""" Returns a list of sites having the given SE as a local one.
Uses the local cache of the site-se information
"""
# Empty the cache if too old
if ( time.time() - self.lastCScheck ) > self.cacheLength:
self.log.verbose( 'Resetting the SE to site mapping cache' )
self.seToSiteMapping = {}
self.lastCScheck = time.time()
if se not in self.seToSiteMapping:
sites = getSitesForSE( se )
if sites['OK']:
self.seToSiteMapping[se] = list( sites['Value'] )
return sites
else:
return S_OK( self.seToSiteMapping[se] )
def __getSitesForSE(self, seName):
""" Returns a list of sites having the given SE as a local one.
Uses the local cache of the site-se information
"""
# Empty the cache if too old
now = time.time()
if (now - self.__lastCacheUpdate) > self.__cacheLifeTime:
self.log.verbose('Resetting the SE to site mapping cache')
self.__SEToSiteMap = {}
self.__lastCacheUpdate = now
if seName not in self.__SEToSiteMap:
result = getSitesForSE(seName)
if not result['OK']:
return result
self.__SEToSiteMap[seName] = list(result['Value'])
return S_OK(self.__SEToSiteMap[seName])
def __getSitesForSE( self, seName ):
""" Returns a list of sites having the given SE as a local one.
Uses the local cache of the site-se information
"""
# Empty the cache if too old
now = time.time()
if ( now - self.__lastCacheUpdate ) > self.__cacheLifeTime:
self.log.verbose( 'Resetting the SE to site mapping cache' )
self.__SEToSiteMap = {}
self.__lastCacheUpdate = now
if seName not in self.__SEToSiteMap:
result = getSitesForSE( seName )
if not result['OK']:
return result
self.__SEToSiteMap[ seName ] = list( result['Value'] )
return S_OK( self.__SEToSiteMap[ seName ] )
def _getExistingCounters( self, normalise = False, requestedSites = [] ):
res = self.transClient.getCounters( 'TransformationFiles', ['UsedSE'], {'TransformationID':self.params['TransformationID']} )
if not res['OK']:
return res
usageDict = {}
for usedDict, count in res['Value']:
usedSE = usedDict['UsedSE']
if usedSE != 'Unknown':
usageDict[usedSE] = count
if requestedSites:
siteDict = {}
for se, count in usageDict.items():
res = getSitesForSE( se, gridName = 'LCG' )
if not res['OK']:
return res
for site in res['Value']:
if site in requestedSites:
siteDict[site] = count
usageDict = siteDict.copy()
if normalise:
usageDict = self._normaliseShares( usageDict )
return S_OK( usageDict )
def _getExistingCounters(self, normalise=False, requestedSites=[]):
res = self.transClient.getCounters(
"TransformationFiles", ["UsedSE"], {"TransformationID": self.params["TransformationID"]}
)
if not res["OK"]:
return res
usageDict = {}
for usedDict, count in res["Value"]:
usedSE = usedDict["UsedSE"]
if usedSE != "Unknown":
usageDict[usedSE] = count
if requestedSites:
siteDict = {}
for se, count in usageDict.items():
res = getSitesForSE(se, gridName="LCG")
if not res["OK"]:
return res
for site in res["Value"]:
if site in requestedSites:
siteDict[site] = count
usageDict = siteDict.copy()
if normalise:
usageDict = self._normaliseShares(usageDict)
return S_OK(usageDict)
def getExistingCounters(self, normalise=False, requestedSites=[]):
res = self.transClient.getCounters(
"TransformationFiles", ["UsedSE"],
{"TransformationID": self.params["TransformationID"]})
if not res["OK"]:
return res
usageDict = {}
for usedDict, count in res["Value"]:
usedSE = usedDict["UsedSE"]
if usedSE != "Unknown":
usageDict[usedSE] = count
if requestedSites:
siteDict = {}
for se, count in usageDict.items():
res = getSitesForSE(se)
if not res["OK"]:
return res
for site in res["Value"]:
if site in requestedSites:
siteDict[site] = count
usageDict = siteDict.copy()
if normalise:
usageDict = self._normaliseShares(usageDict)
return S_OK(usageDict)
def run( self ):
''' Checks it has the parameters it needs and writes the date to a cache file.
'''
# Minor security checks
element = self.decisionParams[ 'element' ]
if element is None:
return S_ERROR( 'element should not be None' )
name = self.decisionParams[ 'name' ]
if name is None:
return S_ERROR( 'name should not be None' )
statusType = self.decisionParams[ 'statusType' ]
if statusType is None:
return S_ERROR( 'statusType should not be None' )
previousStatus = self.decisionParams[ 'status' ]
if previousStatus is None:
return S_ERROR( 'status should not be None' )
status = self.enforcementResult[ 'Status' ]
if status is None:
return S_ERROR( 'status should not be None' )
reason = self.enforcementResult[ 'Reason' ]
if reason is None:
return S_ERROR( 'reason should not be None' )
if self.decisionParams[ 'element' ] == 'Site':
siteName = self.decisionParams[ 'name' ]
else:
siteName = getSitesForSE(name)
if not siteName['OK']:
self.log.error('Resource %s does not exist at any site: %s' % (name, siteName['Message']))
siteName = "Unassigned Resources"
elif not siteName['Value']:
siteName = "Unassigned Resources"
else:
siteName = siteName['Value'][0]
with sqlite3.connect(self.cacheFile) as conn:
try:
conn.execute('''CREATE TABLE IF NOT EXISTS ResourceStatusCache(
SiteName VARCHAR(64) NOT NULL,
ResourceName VARCHAR(64) NOT NULL,
Status VARCHAR(8) NOT NULL DEFAULT "",
PreviousStatus VARCHAR(8) NOT NULL DEFAULT "",
StatusType VARCHAR(128) NOT NULL DEFAULT "all",
Time TIMESTAMP DEFAULT CURRENT_TIMESTAMP
);''')
except sqlite3.OperationalError:
self.log.error('Email cache database is locked')
conn.execute("INSERT INTO ResourceStatusCache (SiteName, ResourceName, Status, PreviousStatus, StatusType)"
" VALUES ('" + siteName + "', '" + name + "', '" + status + "', '" + previousStatus + "', '" + statusType + "' ); "
)
conn.commit()
return S_OK()
def run( self ):
''' Checks it has the parameters it needs and writes the date to a cache file.
'''
# Minor security checks
element = self.decisionParams[ 'element' ]
if element is None:
return S_ERROR( 'element should not be None' )
name = self.decisionParams[ 'name' ]
if name is None:
return S_ERROR( 'name should not be None' )
statusType = self.decisionParams[ 'statusType' ]
if statusType is None:
return S_ERROR( 'statusType should not be None' )
previousStatus = self.decisionParams[ 'status' ]
if previousStatus is None:
return S_ERROR( 'status should not be None' )
status = self.enforcementResult[ 'Status' ]
if status is None:
return S_ERROR( 'status should not be None' )
reason = self.enforcementResult[ 'Reason' ]
if reason is None:
return S_ERROR( 'reason should not be None' )
siteName = getSitesForSE(name)
if not siteName['OK']:
self.log.error('Resource %s does not exist at any site: %s' % (name, siteName['Message']))
siteName = "Unassigned Resources"
elif not siteName['Value']:
siteName = "Unassigned Resources"
else:
siteName = siteName['Value'][0]
with sqlite3.connect(self.cacheFile) as conn:
try:
conn.execute('''CREATE TABLE IF NOT EXISTS ResourceStatusCache(
SiteName VARCHAR(64) NOT NULL,
ResourceName VARCHAR(64) NOT NULL,
Status VARCHAR(8) NOT NULL DEFAULT "",
PreviousStatus VARCHAR(8) NOT NULL DEFAULT "",
StatusType VARCHAR(128) NOT NULL DEFAULT "all",
Time TIMESTAMP DEFAULT CURRENT_TIMESTAMP
);''')
except sqlite3.OperationalError:
self.log.error('Email cache database is locked')
conn.execute("INSERT INTO ResourceStatusCache (SiteName, ResourceName, Status, PreviousStatus, StatusType)"
" VALUES ('" + siteName + "', '" + name + "', '" + status + "', '" + previousStatus + "', '" + statusType + "' ); "
)
conn.commit()
return S_OK()
def getSitesForSE(se):
''' Get the Sites associated with this SE'''
from DIRAC.Core.Utilities.SiteSEMapping import getSitesForSE
result = getSitesForSE(storageElement=se)
return result
def run(self):
''' Checks it has the parameters it needs and writes the date to a cache file.
'''
# Minor security checks
element = self.decisionParams['element']
if element is None:
return S_ERROR('element should not be None')
name = self.decisionParams['name']
if name is None:
return S_ERROR('name should not be None')
statusType = self.decisionParams['statusType']
if statusType is None:
return S_ERROR('statusType should not be None')
previousStatus = self.decisionParams['status']
if previousStatus is None:
return S_ERROR('status should not be None')
status = self.enforcementResult['Status']
if status is None:
return S_ERROR('status should not be None')
reason = self.enforcementResult['Reason']
if reason is None:
return S_ERROR('reason should not be None')
if self.decisionParams['element'] == 'Site':
siteName = self.decisionParams['name']
else:
elementType = self.decisionParams['elementType']
if elementType == 'StorageElement':
siteName = getSitesForSE(name)
elif elementType == 'ComputingElement':
res = getCESiteMapping(name)
if not res['OK']:
return res
siteName = S_OK(res['Value'][name])
else:
siteName = {'OK': True, 'Value': 'Unassigned'}
if not siteName['OK']:
self.log.error('Resource %s does not exist at any site: %s' % (name, siteName['Message']))
siteName = "Unassigned Resources"
elif not siteName['Value']:
siteName = "Unassigned Resources"
else:
siteName = siteName['Value'] if isinstance(siteName['Value'], six.string_types) else siteName['Value'][0]
with sqlite3.connect(self.cacheFile) as conn:
try:
conn.execute('''CREATE TABLE IF NOT EXISTS ResourceStatusCache(
SiteName VARCHAR(64) NOT NULL,
ResourceName VARCHAR(64) NOT NULL,
Status VARCHAR(8) NOT NULL DEFAULT "",
PreviousStatus VARCHAR(8) NOT NULL DEFAULT "",
StatusType VARCHAR(128) NOT NULL DEFAULT "all",
Time TIMESTAMP DEFAULT CURRENT_TIMESTAMP
);''')
insertQuery = "INSERT INTO ResourceStatusCache (SiteName, ResourceName, Status, PreviousStatus, StatusType)"
insertQuery += " VALUES ('%s', '%s', '%s', '%s', '%s' ); " % (siteName, name, status,
previousStatus, statusType)
conn.execute(insertQuery)
conn.commit()
except sqlite3.OperationalError:
self.log.error('Email cache database is locked')
return S_OK()
def run(self):
"""
Checks it has the parameters it needs and tries to send an sms to the users
that apply.
"""
if self.url is None:
return S_ERROR("Slack URL not set")
# Minor security checks
element = self.decisionParams["element"]
if element is None:
return S_ERROR("element should not be None")
name = self.decisionParams["name"]
if name is None:
return S_ERROR("name should not be None")
statusType = self.decisionParams["statusType"]
if statusType is None:
return S_ERROR("statusType should not be None")
previousStatus = self.decisionParams["status"]
if previousStatus is None:
return S_ERROR("status should not be None")
status = self.enforcementResult["Status"]
if status is None:
return S_ERROR("status should not be None")
reason = self.enforcementResult["Reason"]
if reason is None:
return S_ERROR("reason should not be None")
if self.decisionParams["element"] == "Site":
siteName = self.decisionParams["name"]
else:
elementType = self.decisionParams["elementType"]
if elementType == "StorageElement":
siteName = getSitesForSE(name)
elif elementType == "ComputingElement":
res = getCESiteMapping(name)
if not res["OK"]:
self.log.error("Failure getting Site2CE mapping",
res["Message"])
siteName = "ERROR"
else:
siteName = res
else:
siteName = {"OK": True, "Value": "Unassigned"}
if not siteName["OK"]:
self.log.error("Resource %s does not exist at any site: %s" %
(name, siteName["Message"]))
siteName = "Unassigned Resources"
elif not siteName["Value"]:
siteName = "Unassigned Resources"
else:
siteName = (siteName["Value"] if isinstance(
siteName["Value"], six.string_types) else
siteName["Value"][0])
message = "*{name}* _{statusType}_ --> _{status}_ \n{reason}".format(
name=name, statusType=statusType, status=status, reason=reason)
return self.sendSlackMessage(message)
def splitInputDataBySize(self, lfns, maxSizePerJob=20, printOutput=False):
"""Split the supplied lfn list by the replicas present at the possible
destination sites, based on a maximum size.
An S_OK object will be returned containing a list of
lists in order to create the jobs.
Example usage:
>>> d.splitInputDataBySize(lfns,10)
{'OK': True, 'Value': [['<LFN>'], ['<LFN>']]}
@param lfns: Logical File Name(s) to split
@type lfns: list
@param maxSizePerJob: Maximum size (in GB) per bunch
@type maxSizePerJob: integer
@param printOutput: Optional flag to print result
@type printOutput: boolean
@return: S_OK,S_ERROR
"""
sitesForSE = {}
if isinstance(lfns, str):
lfns = [lfns.replace('LFN:', '')]
elif isinstance(lfns, list):
try:
lfns = [str(lfn.replace('LFN:', '')) for lfn in lfns]
except TypeError as x:
return self._errorReport(str(x), 'Expected strings for LFNs')
else:
return self._errorReport('Expected single string or list of strings for LFN(s)')
if not isinstance(maxSizePerJob, int):
try:
maxSizePerJob = int(maxSizePerJob)
except ValueError as x:
return self._errorReport(str(x), 'Expected integer for maxSizePerJob')
maxSizePerJob *= 1000 * 1000 * 1000
replicaDict = self.getReplicas(lfns)
if not replicaDict['OK']:
return replicaDict
replicas = replicaDict['Value']['Successful']
if not replicas:
return self._errorReport(replicaDict['Value']['Failed'].items()[0],
'Failed to get replica information')
siteLfns = {}
for lfn, reps in replicas.items():
possibleSites = set(site
for se in reps
for site in sitesForSE.setdefault(se, getSitesForSE(se).get('Value', [])))
siteLfns.setdefault(','.join(sorted(possibleSites)), []).append(lfn)
if '' in siteLfns:
# Some files don't have active replicas
return self._errorReport('No active replica found for', str(siteLfns['']))
# Get size of files
metadataDict = self.getLfnMetadata(lfns, printOutput)
if not metadataDict['OK']:
return metadataDict
fileSizes = dict((lfn, metadataDict['Value']['Successful'].get(lfn, {}).get('Size', maxSizePerJob))
for lfn in lfns)
lfnGroups = []
# maxSize is in GB
for files in siteLfns.values():
# Now get bunches of files,
# Sort in decreasing size
files.sort(cmp=(lambda f1, f2: fileSizes[f2] - fileSizes[f1]))
while files:
# print [( lfn, fileSizes[lfn] ) for lfn in files]
group = []
sizeTot = 0
for lfn in list(files):
size = fileSizes[lfn]
if size >= maxSizePerJob:
lfnGroups.append([lfn])
elif sizeTot + size < maxSizePerJob:
sizeTot += size
group.append(lfn)
files.remove(lfn)
if group:
lfnGroups.append(group)
if printOutput:
self.log.notice(self.pPrint.pformat(lfnGroups))
return S_OK(lfnGroups)
def run(self):
"""Checks it has the parameters it needs and writes the date to a cache file."""
# Minor security checks
element = self.decisionParams["element"]
if element is None:
return S_ERROR("element should not be None")
name = self.decisionParams["name"]
if name is None:
return S_ERROR("name should not be None")
statusType = self.decisionParams["statusType"]
if statusType is None:
return S_ERROR("statusType should not be None")
previousStatus = self.decisionParams["status"]
if previousStatus is None:
return S_ERROR("status should not be None")
status = self.enforcementResult["Status"]
if status is None:
return S_ERROR("status should not be None")
reason = self.enforcementResult["Reason"]
if reason is None:
return S_ERROR("reason should not be None")
if self.decisionParams["element"] == "Site":
siteName = self.decisionParams["name"]
else:
elementType = self.decisionParams["elementType"]
if elementType == "StorageElement":
siteName = getSitesForSE(name)
elif elementType == "ComputingElement":
res = getCESiteMapping(name)
if not res["OK"]:
return res
siteName = S_OK(res["Value"][name])
else:
siteName = {"OK": True, "Value": "Unassigned"}
if not siteName["OK"]:
self.log.error("Resource %s does not exist at any site: %s" % (name, siteName["Message"]))
siteName = "Unassigned Resources"
elif not siteName["Value"]:
siteName = "Unassigned Resources"
else:
siteName = (
siteName["Value"] if isinstance(siteName["Value"], six.string_types) else siteName["Value"][0]
)
# create record for insertion
recordDict = {}
recordDict["SiteName"] = siteName
recordDict["ResourceName"] = name
recordDict["Status"] = status
recordDict["PreviousStatus"] = previousStatus
recordDict["StatusType"] = statusType
return self.rsClient.insert("ResourceStatusCache", recordDict)
def getSitesForSE(se):
from DIRAC.Core.Utilities.SiteSEMapping import getSitesForSE
result = getSitesForSE(storageElement=se)
output(result)
def getSitesForSE(se):
''' Get the Sites associated with this SE'''
from DIRAC.Core.Utilities.SiteSEMapping import getSitesForSE
result = getSitesForSE(storageElement=se)
output(result)
def getSitesForSE(se):
from DIRAC.Core.Utilities.SiteSEMapping import getSitesForSE
result = getSitesForSE(storageElement=se)
output(result)