def getLocationOrderedCatalogs( siteName = '' ):
# First get a list of the active catalogs and their location
res = getActiveCatalogs()
if not res['OK']:
gLogger.error( "Failed to get list of active catalogs", res['Message'] )
return res
catalogDict = res['Value']
# Get the tier1 associated to the current location
if not siteName:
import DIRAC
siteName = DIRAC.siteName()
countryCode = siteName.split( '.' )[-1]
res = getCountryMappingTier1( countryCode )
if not res['OK']:
gLogger.error( "Failed to resolve closest Tier1", res['Message'] )
return res
tier1 = res['Value']
# Create a sorted list of the active readonly catalogs
catalogList = []
if catalogDict.has_key( tier1 ):
catalogList.append( catalogDict[tier1] )
catalogDict.pop( tier1 )
for catalogURL in randomize( catalogDict.values() ):
catalogList.append( catalogURL )
return S_OK( catalogList )
def getLocationOrderedCatalogs(siteName=''):
# First get a list of the active catalogs and their location
res = getActiveCatalogs()
if not res['OK']:
gLogger.error("Failed to get list of active catalogs", res['Message'])
return res
catalogDict = res['Value']
# Get the tier1 associated to the current location
if not siteName:
import DIRAC
siteName = DIRAC.siteName()
countryCode = siteName.split('.')[-1]
res = getCountryMappingTier1(countryCode)
if not res['OK']:
gLogger.error("Failed to resolve closest Tier1", res['Message'])
return res
tier1 = res['Value']
# Create a sorted list of the active readonly catalogs
catalogList = []
if catalogDict.has_key(tier1):
catalogList.append(catalogDict[tier1])
catalogDict.pop(tier1)
for catalogURL in randomize(catalogDict.values()):
catalogList.append(catalogURL)
return S_OK(catalogList)
def __init__(self, useCertificates = False):
""" Constructor of the RequestClient class
"""
self.localUrl = gConfig.getValue('/Systems/DataManagement/Development/Services/RequestDB/localURL')
self.centralUrl = gConfig.getValue('/Systems/DataManagement/Development/Services/RequestDB/centralURL')
voBoxUrls = gConfig.getValue('/Systems/DataManagement/Development/Services/RequestDB/voBoxURLs')
self.voBoxUrls = randomize(voBoxUrls).remove(self.localUrl)
def __init__( self, useCertificates = False ):
"""c'tor
:param self: self reference
:param bool useCertificates: flag to enable/disable certificates
"""
Client.__init__( self )
## setup logger
self.log = gLogger.getSubLogger( "RequestManagement/RequestClient" )
## dict to store all RPC clients for easy reuse
self.__requestRPCClientsDict = {}
## local if any defined
local = PathFinder.getServiceURL( "RequestManagement/localURL" )
if local:
self.__requestRPCClientsDict.setdefault( "local" , [ self.__requestRPCClient( local ) ] )
## central if any defined
central = PathFinder.getServiceURL( "RequestManagement/centralURL" )
if central:
self.__requestRPCClientsDict.setdefault( "central", [ self.__requestRPCClient( central ) ] )
## voboxes if any defined
voBoxUrls = fromChar( PathFinder.getServiceURL( "RequestManagement/voBoxURLs" ) )
if voBoxUrls:
self.__requestRPCClientsDict.setdefault( "voboxes", [] )
for voBoxURL in randomize( voBoxUrls ):
self.__requestRPCClientsDict["voboxes"].append( self.__requestRPCClient( voBoxURL ) )
self.setServer( 'RequestManagement/centralURL' )
def __applyReduction(self, lfns, maxFiles=None):
"""eventually remove the number of files to be considered"""
if maxFiles is None:
maxFiles = self.maxFiles
if not maxFiles or len(lfns) <= maxFiles:
return lfns
return randomize(lfns)[:maxFiles]
def __init__(self, useCertificates=False):
"""c'tor
:param self: self reference
:param bool useCertificates: flag to enable/disable certificates
"""
Client.__init__(self)
## setup logger
self.log = gLogger.getSubLogger("RequestManagement/RequestClient")
## dict to store all RPC clients for easy reuse
self.__requestRPCClientsDict = {}
## local if any defined
local = PathFinder.getServiceURL("RequestManagement/localURL")
if local:
self.__requestRPCClientsDict.setdefault(
"local", [self.__requestRPCClient(local)])
## central if any defined
central = PathFinder.getServiceURL("RequestManagement/centralURL")
if central:
self.__requestRPCClientsDict.setdefault(
"central", [self.__requestRPCClient(central)])
## voboxes if any defined
voBoxUrls = fromChar(
PathFinder.getServiceURL("RequestManagement/voBoxURLs"))
if voBoxUrls:
self.__requestRPCClientsDict.setdefault("voboxes", [])
for voBoxURL in randomize(voBoxUrls):
self.__requestRPCClientsDict["voboxes"].append(
self.__requestRPCClient(voBoxURL))
self.setServer('RequestManagement/centralURL')
def __applyReduction(self, lfns, maxFiles=None):
""" eventually remove the number of files to be considered
"""
if maxFiles is None:
maxFiles = self.maxFiles
if not maxFiles or len(lfns) <= maxFiles:
return lfns
return randomize(lfns)[:maxFiles]
def _Broadcast( self ):
""" This plug-in takes files found at the sourceSE and broadcasts to all (or a selection of) targetSEs.
"""
if not self.params:
return S_ERROR( "TransformationPlugin._Broadcast: The 'Broadcast' plugin requires additional parameters." )
sourceseParam = self.params['SourceSE']
targetseParam = self.params['TargetSE']
sourceSEs = []
targetSEs = []
if sourceseParam.count( '[' ):#assume it's an array
sourceSEs = eval( sourceseParam )
else:
sourceSEs = [sourceseParam]
sourceSEs = eval( self.params['SourceSE'] )
if targetseParam.count( '[' ):
targetSEs = eval( targetseParam )
elif type(targetseParam)==type([]):
targetSEs = targetseParam
else:
targetSEs = [targetseParam]
#sourceSEs = eval(self.params['SourceSE'])
#targetSEs = eval(self.params['TargetSE'])
destinations = int( self.params.get( 'Destinations', 0 ) )
if destinations and ( destinations >= len(targetSEs) ):
destinations = 0
fileGroups = self._getFileGroups( self.data )
targetSELfns = {}
for replicaSE, lfns in fileGroups.items():
ses = replicaSE.split( ',' )
#sourceSites = self._getSitesForSEs(ses)
atSource = False
for se in ses:
if se in sourceSEs:
atSource = True
if not atSource:
continue
for lfn in lfns:
targets = []
sources = self._getSitesForSEs( ses )
for targetSE in randomize( targetSEs ):
site = self._getSiteForSE( targetSE )['Value']
if not site in sources:
if ( destinations ) and ( len( targets ) >= destinations ):
continue
targets.append( targetSE )
sources.append( site )
strTargetSEs = str.join( ',', sortList( targets ) )
if not targetSELfns.has_key( strTargetSEs ):
targetSELfns[strTargetSEs] = []
targetSELfns[strTargetSEs].append( lfn )
tasks = []
for ses, lfns in targetSELfns.items():
tasks.append( ( ses, lfns ) )
return S_OK( tasks )
def _Broadcast( self ):
""" This plug-in takes files found at the sourceSE and broadcasts to all (or a selection of) targetSEs. """
if not self.params:
return S_ERROR( "TransformationPlugin._Broadcast: The 'Broadcast' plugin requires additional parameters." )
sourceseParam = self.params['SourceSE']
targetseParam = self.params['TargetSE']
sourceSEs = []
targetSEs = []
if sourceseParam.count( '[' ):#assume it's an array
sourceSEs = eval( sourceseParam )
else:
sourceSEs = [sourceseParam]
sourceSEs = eval( self.params['SourceSE'] )
if targetseParam.count( '[' ):
targetSEs = eval( targetseParam )
elif type(targetseParam)==type([]):
targetSEs = targetseParam
else:
targetSEs = [targetseParam]
#sourceSEs = eval(self.params['SourceSE'])
#targetSEs = eval(self.params['TargetSE'])
destinations = int( self.params.get( 'Destinations', 0 ) )
if destinations and ( destinations >= len(targetSEs) ):
destinations = 0
fileGroups = self._getFileGroups( self.data )
targetSELfns = {}
for replicaSE, lfns in fileGroups.items():
ses = replicaSE.split( ',' )
#sourceSites = self._getSitesForSEs(ses)
atSource = False
for se in ses:
if se in sourceSEs:
atSource = True
if not atSource:
continue
for lfn in lfns:
targets = []
sources = self._getSitesForSEs( ses )
for targetSE in randomize( targetSEs ):
site = self._getSiteForSE( targetSE )['Value']
if not site in sources:
if ( destinations ) and ( len( targets ) >= destinations ):
continue
targets.append( targetSE )
sources.append( site )
strTargetSEs = str.join( ',', sortList( targets ) )
if not targetSELfns.has_key( strTargetSEs ):
targetSELfns[strTargetSEs] = []
targetSELfns[strTargetSEs].append( lfn )
tasks = []
for ses, lfns in targetSELfns.items():
tasks.append( ( ses, lfns ) )
return S_OK( tasks )
def putRequest(self, request, useFailoverProxy=True, retryMainService=0):
"""Put request to RequestManager
:param self: self reference
:param ~Request.Request request: Request instance
:param bool useFailoverProxy: if False, will not attempt to forward the request to ReqProxies
:param int retryMainService: Amount of time we retry on the main ReqHandler in case of failures
:return: S_OK/S_ERROR
"""
errorsDict = {"OK": False}
valid = self.requestValidator().validate(request)
if not valid["OK"]:
self.log.error("putRequest: request not valid", "%s" % valid["Message"])
return valid
# # dump to json
requestJSON = request.toJSON()
if not requestJSON["OK"]:
return requestJSON
requestJSON = requestJSON["Value"]
retryMainService += 1
while retryMainService:
retryMainService -= 1
setRequestMgr = self._getRPC().putRequest(requestJSON)
if setRequestMgr["OK"]:
return setRequestMgr
errorsDict["RequestManager"] = setRequestMgr["Message"]
# sleep a bit
time.sleep(random.randint(1, 5))
self.log.warn("putRequest: unable to set request '%s' at RequestManager" %
request.RequestName, setRequestMgr["Message"])
proxies = self.requestProxies() if useFailoverProxy else {}
for proxyURL in randomize(proxies.keys()):
proxyClient = proxies[proxyURL]
self.log.debug("putRequest: trying RequestProxy at %s" % proxyURL)
setRequestProxy = proxyClient.putRequest(requestJSON)
if setRequestProxy["OK"]:
if setRequestProxy["Value"]["set"]:
self.log.info("putRequest: request '%s' successfully set using RequestProxy %s" % (request.RequestName,
proxyURL))
elif setRequestProxy["Value"]["saved"]:
self.log.info("putRequest: request '%s' successfully forwarded to RequestProxy %s" % (request.RequestName,
proxyURL))
return setRequestProxy
else:
self.log.warn("putRequest: unable to set request using RequestProxy %s: %s" % (proxyURL,
setRequestProxy["Message"]))
errorsDict["RequestProxy(%s)" % proxyURL] = setRequestProxy["Message"]
# # if we're here neither requestManager nor requestProxy were successful
self.log.error("putRequest: unable to set request", "'%s'" % request.RequestName)
errorsDict["Message"] = "ReqClient.putRequest: unable to set request '%s'" % request.RequestName
return errorsDict
def requestProxies( self, timeout = 120 ):
""" get request proxies dict """
if not self.__requestProxiesDict:
self.__requestProxiesDict = {}
proxiesURLs = fromChar( PathFinder.getServiceURL( "RequestManagement/RequestProxyURLs" ) )
if not proxiesURLs:
self.log.warn( "CS option RequestManagement/RequestProxyURLs is not set!" )
for proxyURL in randomize( proxiesURLs ):
self.log.debug( "creating RequestProxy for url = %s" % proxyURL )
self.__requestProxiesDict[proxyURL] = RPCClient( proxyURL, timeout = timeout )
return self.__requestProxiesDict
def __init__(self, useCertificates=False):
""" Constructor of the RequestClient class
"""
self.localUrl = gConfig.getValue(
'/Systems/DataManagement/Development/Services/RequestDB/localURL')
self.centralUrl = gConfig.getValue(
'/Systems/DataManagement/Development/Services/RequestDB/centralURL'
)
voBoxUrls = gConfig.getValue(
'/Systems/DataManagement/Development/Services/RequestDB/voBoxURLs')
self.voBoxUrls = randomize(voBoxUrls).remove(self.localUrl)
def putRequest(self, request):
""" put request to RequestManager
:param self: self reference
:param Request request: Request instance
"""
errorsDict = {"OK": False}
valid = self.requestValidator().validate(request)
if not valid["OK"]:
self.log.error("putRequest: request not valid",
"%s" % valid["Message"])
return valid
# # dump to json
requestJSON = request.toJSON()
if not requestJSON["OK"]:
return requestJSON
requestJSON = requestJSON["Value"]
setRequestMgr = self.requestManager().putRequest(requestJSON)
if setRequestMgr["OK"]:
return setRequestMgr
errorsDict["RequestManager"] = setRequestMgr["Message"]
self.log.warn(
"putRequest: unable to set request '%s' at RequestManager" %
request.RequestName, setRequestMgr["Message"])
proxies = self.requestProxies()
for proxyURL in randomize(proxies.keys()):
proxyClient = proxies[proxyURL]
self.log.debug("putRequest: trying RequestProxy at %s" % proxyURL)
setRequestProxy = proxyClient.putRequest(requestJSON)
if setRequestProxy["OK"]:
if setRequestProxy["Value"]["set"]:
self.log.info(
"putRequest: request '%s' successfully set using RequestProxy %s"
% (request.RequestName, proxyURL))
elif setRequestProxy["Value"]["saved"]:
self.log.info(
"putRequest: request '%s' successfully forwarded to RequestProxy %s"
% (request.RequestName, proxyURL))
return setRequestProxy
else:
self.log.warn(
"putRequest: unable to set request using RequestProxy %s: %s"
% (proxyURL, setRequestProxy["Message"]))
errorsDict["RequestProxy(%s)" %
proxyURL] = setRequestProxy["Message"]
# # if we're here neither requestManager nor requestProxy were successful
self.log.error("putRequest: unable to set request",
"'%s'" % request.RequestName)
errorsDict[
"Message"] = "ReqClient.putRequest: unable to set request '%s'" % request.RequestName
return errorsDict
def __init__( self, useCertificates = False ):
""" Constructor of the RequestClient class
"""
voBoxUrls = fromChar( PathFinder.getServiceURL( "RequestManagement/voBoxURLs" ) )
self.voBoxUrls = []
if voBoxUrls:
self.voBoxUrls = randomize( voBoxUrls )
local = PathFinder.getServiceURL( "RequestManagement/localURL" )
self.local = False
if local:
self.local = local
central = PathFinder.getServiceURL( "RequestManagement/centralURL" )
self.central = False
if central:
self.central = central
self.setServer( 'RequestManagement/centralURL' )
def putRequest( self, request ):
""" put request to RequestManager
:param self: self reference
:param Request request: Request instance
"""
errorsDict = { "OK" : False }
valid = self.requestValidator().validate( request )
if not valid["OK"]:
self.log.error( "putRequest: request not valid: %s" % valid["Message"] )
return valid
# # dump to json
requestJSON = request.toJSON()
if not requestJSON["OK"]:
return requestJSON
requestJSON = requestJSON["Value"]
setRequestMgr = self.requestManager().putRequest( requestJSON )
if setRequestMgr["OK"]:
return setRequestMgr
errorsDict["RequestManager"] = setRequestMgr["Message"]
self.log.warn( "putRequest: unable to set request '%s' at RequestManager" % request.RequestName )
proxies = self.requestProxies()
for proxyURL in randomize( proxies.keys() ):
proxyClient = proxies[proxyURL]
self.log.debug( "putRequest: trying RequestProxy at %s" % proxyURL )
setRequestProxy = proxyClient.putRequest( requestJSON )
if setRequestProxy["OK"]:
if setRequestProxy["Value"]["set"]:
self.log.info( "putRequest: request '%s' successfully set using RequestProxy %s" % ( request.RequestName,
proxyURL ) )
elif setRequestProxy["Value"]["saved"]:
self.log.info( "putRequest: request '%s' successfully forwarded to RequestProxy %s" % ( request.RequestName,
proxyURL ) )
return setRequestProxy
else:
self.log.warn( "putRequest: unable to set request using RequestProxy %s: %s" % ( proxyURL,
setRequestProxy["Message"] ) )
errorsDict["RequestProxy(%s)" % proxyURL] = setRequestProxy["Message"]
# # if we're here neither requestManager nor requestProxy were successful
self.log.error( "putRequest: unable to set request '%s'" % request.RequestName )
errorsDict["Message"] = "ReqClient.putRequest: unable to set request '%s'" % request.RequestName
return errorsDict
def __init__(self, useCertificates=False):
""" Constructor of the RequestClient class
"""
voBoxUrls = fromChar(
PathFinder.getServiceURL("RequestManagement/voBoxURLs"))
self.voBoxUrls = []
if voBoxUrls:
self.voBoxUrls = randomize(voBoxUrls)
local = PathFinder.getServiceURL("RequestManagement/localURL")
self.local = False
if local:
self.local = local
central = PathFinder.getServiceURL("RequestManagement/centralURL")
self.central = False
if central:
self.central = central
self.setServer('RequestManagement/centralURL')
def __applyReduction( self, lfns ):
""" eventually remove the number of files to be considered
"""
if len( lfns ) <= self.maxFiles:
return lfns
return randomize( lfns )[:self.maxFiles]
def _Broadcast(self):
""" This plug-in takes files found at the sourceSE and broadcasts to all (or a selection of) targetSEs.
"""
if not self.params:
return S_ERROR(
"TransformationPlugin._Broadcast: The 'Broadcast' plugin requires additional parameters."
)
sourceseParam = self.params['SourceSE']
targetseParam = self.params['TargetSE']
sourceSEs = []
targetSEs = []
if sourceseParam.count('['): #assume it's an array
sourceSEs = eval(sourceseParam)
else:
sourceSEs = [sourceseParam]
sourceSEs = eval(self.params['SourceSE'])
if targetseParam.count('['):
targetSEs = eval(targetseParam)
elif type(targetseParam) == type([]):
targetSEs = targetseParam
else:
targetSEs = [targetseParam]
#sourceSEs = eval(self.params['SourceSE'])
#targetSEs = eval(self.params['TargetSE'])
destinations = int(self.params.get('Destinations', 0))
if destinations and (destinations >= len(targetSEs)):
destinations = 0
status = self.params['Status']
groupSize = self.params['GroupSize'] #Number of files per tasks
fileGroups = self._getFileGroups(self.data) #groups by SE
targetSELfns = {}
for replicaSE, lfns in fileGroups.items():
ses = replicaSE.split(',')
#sourceSites = self._getSitesForSEs(ses)
atSource = False
for se in ses:
if se in sourceSEs:
atSource = True
if not atSource:
continue
for lfn in lfns:
targets = []
sources = self._getSitesForSEs(ses)
for targetSE in randomize(targetSEs):
site = self._getSiteForSE(targetSE)['Value']
if not site in sources:
if (destinations) and (len(targets) >= destinations):
continue
sources.append(site)
targets.append(
targetSE
) #after all, if someone wants to copy to the source, it's his choice
strTargetSEs = str.join(',', sortList(targets))
if not targetSELfns.has_key(strTargetSEs):
targetSELfns[strTargetSEs] = []
targetSELfns[strTargetSEs].append(lfn)
tasks = []
for ses, lfns in targetSELfns.items():
tasksLfns = breakListIntoChunks(lfns, groupSize)
for taskLfns in tasksLfns:
if (status == 'Flush') or (len(taskLfns) >= int(groupSize)):
#do not allow groups smaller than the groupSize, except if transformation is in flush state
tasks.append((ses, taskLfns))
return S_OK(tasks)
def _Broadcast( self ):
""" This plug-in takes files found at the sourceSE and broadcasts to all (or a selection of) targetSEs.
"""
if not self.params:
return S_ERROR( "TransformationPlugin._Broadcast: The 'Broadcast' plugin requires additional parameters." )
sourceseParam = self.params['SourceSE']
targetseParam = self.params['TargetSE']
sourceSEs = []
targetSEs = []
if sourceseParam.count( '[' ):#assume it's an array
sourceSEs = eval( sourceseParam )
else:
sourceSEs = [sourceseParam]
sourceSEs = eval( self.params['SourceSE'] )
if targetseParam.count( '[' ):
targetSEs = eval( targetseParam )
elif type(targetseParam)==type([]):
targetSEs = targetseParam
else:
targetSEs = [targetseParam]
#sourceSEs = eval(self.params['SourceSE'])
#targetSEs = eval(self.params['TargetSE'])
destinations = int( self.params.get( 'Destinations', 0 ) )
if destinations and ( destinations >= len(targetSEs) ):
destinations = 0
status = self.params['Status']
groupSize = self.params['GroupSize']#Number of files per tasks
fileGroups = self._getFileGroups( self.data )#groups by SE
targetSELfns = {}
for replicaSE, lfns in fileGroups.items():
ses = replicaSE.split( ',' )
#sourceSites = self._getSitesForSEs(ses)
atSource = False
for se in ses:
if se in sourceSEs:
atSource = True
if not atSource:
continue
for lfn in lfns:
targets = []
sources = self._getSitesForSEs( ses )
for targetSE in randomize( targetSEs ):
site = self._getSiteForSE( targetSE )['Value']
if not site in sources:
if ( destinations ) and ( len( targets ) >= destinations ):
continue
sources.append( site )
targets.append( targetSE )#after all, if someone wants to copy to the source, it's his choice
strTargetSEs = str.join( ',', sortList( targets ) )
if not targetSELfns.has_key( strTargetSEs ):
targetSELfns[strTargetSEs] = []
targetSELfns[strTargetSEs].append( lfn )
tasks = []
for ses, lfns in targetSELfns.items():
tasksLfns = breakListIntoChunks(lfns, groupSize)
for taskLfns in tasksLfns:
if ( status == 'Flush' ) or ( len( taskLfns ) >= int( groupSize ) ):
#do not allow groups smaller than the groupSize, except if transformation is in flush state
tasks.append( ( ses, taskLfns ) )
return S_OK( tasks )
def __applyReduction(self, lfns):
""" eventually remove the number of files to be considered
"""
if len(lfns) <= self.maxFiles:
return lfns
return randomize(lfns)[:self.maxFiles]
