def getLocationOrderedCatalogs( siteName = '' ): # First get a list of the active catalogs and their location res = getActiveCatalogs() if not res['OK']: gLogger.error( "Failed to get list of active catalogs", res['Message'] ) return res catalogDict = res['Value'] # Get the tier1 associated to the current location if not siteName: import DIRAC siteName = DIRAC.siteName() countryCode = siteName.split( '.' )[-1] res = getCountryMappingTier1( countryCode ) if not res['OK']: gLogger.error( "Failed to resolve closest Tier1", res['Message'] ) return res tier1 = res['Value'] # Create a sorted list of the active readonly catalogs catalogList = [] if catalogDict.has_key( tier1 ): catalogList.append( catalogDict[tier1] ) catalogDict.pop( tier1 ) for catalogURL in randomize( catalogDict.values() ): catalogList.append( catalogURL ) return S_OK( catalogList )
def getLocationOrderedCatalogs(siteName=''): # First get a list of the active catalogs and their location res = getActiveCatalogs() if not res['OK']: gLogger.error("Failed to get list of active catalogs", res['Message']) return res catalogDict = res['Value'] # Get the tier1 associated to the current location if not siteName: import DIRAC siteName = DIRAC.siteName() countryCode = siteName.split('.')[-1] res = getCountryMappingTier1(countryCode) if not res['OK']: gLogger.error("Failed to resolve closest Tier1", res['Message']) return res tier1 = res['Value'] # Create a sorted list of the active readonly catalogs catalogList = [] if catalogDict.has_key(tier1): catalogList.append(catalogDict[tier1]) catalogDict.pop(tier1) for catalogURL in randomize(catalogDict.values()): catalogList.append(catalogURL) return S_OK(catalogList)
def __init__(self, useCertificates = False): """ Constructor of the RequestClient class """ self.localUrl = gConfig.getValue('/Systems/DataManagement/Development/Services/RequestDB/localURL') self.centralUrl = gConfig.getValue('/Systems/DataManagement/Development/Services/RequestDB/centralURL') voBoxUrls = gConfig.getValue('/Systems/DataManagement/Development/Services/RequestDB/voBoxURLs') self.voBoxUrls = randomize(voBoxUrls).remove(self.localUrl)
def __init__( self, useCertificates = False ): """c'tor :param self: self reference :param bool useCertificates: flag to enable/disable certificates """ Client.__init__( self ) ## setup logger self.log = gLogger.getSubLogger( "RequestManagement/RequestClient" ) ## dict to store all RPC clients for easy reuse self.__requestRPCClientsDict = {} ## local if any defined local = PathFinder.getServiceURL( "RequestManagement/localURL" ) if local: self.__requestRPCClientsDict.setdefault( "local" , [ self.__requestRPCClient( local ) ] ) ## central if any defined central = PathFinder.getServiceURL( "RequestManagement/centralURL" ) if central: self.__requestRPCClientsDict.setdefault( "central", [ self.__requestRPCClient( central ) ] ) ## voboxes if any defined voBoxUrls = fromChar( PathFinder.getServiceURL( "RequestManagement/voBoxURLs" ) ) if voBoxUrls: self.__requestRPCClientsDict.setdefault( "voboxes", [] ) for voBoxURL in randomize( voBoxUrls ): self.__requestRPCClientsDict["voboxes"].append( self.__requestRPCClient( voBoxURL ) ) self.setServer( 'RequestManagement/centralURL' )
def __applyReduction(self, lfns, maxFiles=None): """eventually remove the number of files to be considered""" if maxFiles is None: maxFiles = self.maxFiles if not maxFiles or len(lfns) <= maxFiles: return lfns return randomize(lfns)[:maxFiles]
def __init__(self, useCertificates=False): """c'tor :param self: self reference :param bool useCertificates: flag to enable/disable certificates """ Client.__init__(self) ## setup logger self.log = gLogger.getSubLogger("RequestManagement/RequestClient") ## dict to store all RPC clients for easy reuse self.__requestRPCClientsDict = {} ## local if any defined local = PathFinder.getServiceURL("RequestManagement/localURL") if local: self.__requestRPCClientsDict.setdefault( "local", [self.__requestRPCClient(local)]) ## central if any defined central = PathFinder.getServiceURL("RequestManagement/centralURL") if central: self.__requestRPCClientsDict.setdefault( "central", [self.__requestRPCClient(central)]) ## voboxes if any defined voBoxUrls = fromChar( PathFinder.getServiceURL("RequestManagement/voBoxURLs")) if voBoxUrls: self.__requestRPCClientsDict.setdefault("voboxes", []) for voBoxURL in randomize(voBoxUrls): self.__requestRPCClientsDict["voboxes"].append( self.__requestRPCClient(voBoxURL)) self.setServer('RequestManagement/centralURL')
def __applyReduction(self, lfns, maxFiles=None): """ eventually remove the number of files to be considered """ if maxFiles is None: maxFiles = self.maxFiles if not maxFiles or len(lfns) <= maxFiles: return lfns return randomize(lfns)[:maxFiles]
def _Broadcast( self ): """ This plug-in takes files found at the sourceSE and broadcasts to all (or a selection of) targetSEs. """ if not self.params: return S_ERROR( "TransformationPlugin._Broadcast: The 'Broadcast' plugin requires additional parameters." ) sourceseParam = self.params['SourceSE'] targetseParam = self.params['TargetSE'] sourceSEs = [] targetSEs = [] if sourceseParam.count( '[' ):#assume it's an array sourceSEs = eval( sourceseParam ) else: sourceSEs = [sourceseParam] sourceSEs = eval( self.params['SourceSE'] ) if targetseParam.count( '[' ): targetSEs = eval( targetseParam ) elif type(targetseParam)==type([]): targetSEs = targetseParam else: targetSEs = [targetseParam] #sourceSEs = eval(self.params['SourceSE']) #targetSEs = eval(self.params['TargetSE']) destinations = int( self.params.get( 'Destinations', 0 ) ) if destinations and ( destinations >= len(targetSEs) ): destinations = 0 fileGroups = self._getFileGroups( self.data ) targetSELfns = {} for replicaSE, lfns in fileGroups.items(): ses = replicaSE.split( ',' ) #sourceSites = self._getSitesForSEs(ses) atSource = False for se in ses: if se in sourceSEs: atSource = True if not atSource: continue for lfn in lfns: targets = [] sources = self._getSitesForSEs( ses ) for targetSE in randomize( targetSEs ): site = self._getSiteForSE( targetSE )['Value'] if not site in sources: if ( destinations ) and ( len( targets ) >= destinations ): continue targets.append( targetSE ) sources.append( site ) strTargetSEs = str.join( ',', sortList( targets ) ) if not targetSELfns.has_key( strTargetSEs ): targetSELfns[strTargetSEs] = [] targetSELfns[strTargetSEs].append( lfn ) tasks = [] for ses, lfns in targetSELfns.items(): tasks.append( ( ses, lfns ) ) return S_OK( tasks )
def _Broadcast( self ): """ This plug-in takes files found at the sourceSE and broadcasts to all (or a selection of) targetSEs. """ if not self.params: return S_ERROR( "TransformationPlugin._Broadcast: The 'Broadcast' plugin requires additional parameters." ) sourceseParam = self.params['SourceSE'] targetseParam = self.params['TargetSE'] sourceSEs = [] targetSEs = [] if sourceseParam.count( '[' ):#assume it's an array sourceSEs = eval( sourceseParam ) else: sourceSEs = [sourceseParam] sourceSEs = eval( self.params['SourceSE'] ) if targetseParam.count( '[' ): targetSEs = eval( targetseParam ) elif type(targetseParam)==type([]): targetSEs = targetseParam else: targetSEs = [targetseParam] #sourceSEs = eval(self.params['SourceSE']) #targetSEs = eval(self.params['TargetSE']) destinations = int( self.params.get( 'Destinations', 0 ) ) if destinations and ( destinations >= len(targetSEs) ): destinations = 0 fileGroups = self._getFileGroups( self.data ) targetSELfns = {} for replicaSE, lfns in fileGroups.items(): ses = replicaSE.split( ',' ) #sourceSites = self._getSitesForSEs(ses) atSource = False for se in ses: if se in sourceSEs: atSource = True if not atSource: continue for lfn in lfns: targets = [] sources = self._getSitesForSEs( ses ) for targetSE in randomize( targetSEs ): site = self._getSiteForSE( targetSE )['Value'] if not site in sources: if ( destinations ) and ( len( targets ) >= destinations ): continue targets.append( targetSE ) sources.append( site ) strTargetSEs = str.join( ',', sortList( targets ) ) if not targetSELfns.has_key( strTargetSEs ): targetSELfns[strTargetSEs] = [] targetSELfns[strTargetSEs].append( lfn ) tasks = [] for ses, lfns in targetSELfns.items(): tasks.append( ( ses, lfns ) ) return S_OK( tasks )
def putRequest(self, request, useFailoverProxy=True, retryMainService=0): """Put request to RequestManager :param self: self reference :param ~Request.Request request: Request instance :param bool useFailoverProxy: if False, will not attempt to forward the request to ReqProxies :param int retryMainService: Amount of time we retry on the main ReqHandler in case of failures :return: S_OK/S_ERROR """ errorsDict = {"OK": False} valid = self.requestValidator().validate(request) if not valid["OK"]: self.log.error("putRequest: request not valid", "%s" % valid["Message"]) return valid # # dump to json requestJSON = request.toJSON() if not requestJSON["OK"]: return requestJSON requestJSON = requestJSON["Value"] retryMainService += 1 while retryMainService: retryMainService -= 1 setRequestMgr = self._getRPC().putRequest(requestJSON) if setRequestMgr["OK"]: return setRequestMgr errorsDict["RequestManager"] = setRequestMgr["Message"] # sleep a bit time.sleep(random.randint(1, 5)) self.log.warn("putRequest: unable to set request '%s' at RequestManager" % request.RequestName, setRequestMgr["Message"]) proxies = self.requestProxies() if useFailoverProxy else {} for proxyURL in randomize(proxies.keys()): proxyClient = proxies[proxyURL] self.log.debug("putRequest: trying RequestProxy at %s" % proxyURL) setRequestProxy = proxyClient.putRequest(requestJSON) if setRequestProxy["OK"]: if setRequestProxy["Value"]["set"]: self.log.info("putRequest: request '%s' successfully set using RequestProxy %s" % (request.RequestName, proxyURL)) elif setRequestProxy["Value"]["saved"]: self.log.info("putRequest: request '%s' successfully forwarded to RequestProxy %s" % (request.RequestName, proxyURL)) return setRequestProxy else: self.log.warn("putRequest: unable to set request using RequestProxy %s: %s" % (proxyURL, setRequestProxy["Message"])) errorsDict["RequestProxy(%s)" % proxyURL] = setRequestProxy["Message"] # # if we're here neither requestManager nor requestProxy were successful self.log.error("putRequest: unable to set request", "'%s'" % request.RequestName) errorsDict["Message"] = "ReqClient.putRequest: unable to set request '%s'" % request.RequestName return errorsDict
def requestProxies( self, timeout = 120 ): """ get request proxies dict """ if not self.__requestProxiesDict: self.__requestProxiesDict = {} proxiesURLs = fromChar( PathFinder.getServiceURL( "RequestManagement/RequestProxyURLs" ) ) if not proxiesURLs: self.log.warn( "CS option RequestManagement/RequestProxyURLs is not set!" ) for proxyURL in randomize( proxiesURLs ): self.log.debug( "creating RequestProxy for url = %s" % proxyURL ) self.__requestProxiesDict[proxyURL] = RPCClient( proxyURL, timeout = timeout ) return self.__requestProxiesDict
def __init__(self, useCertificates=False): """ Constructor of the RequestClient class """ self.localUrl = gConfig.getValue( '/Systems/DataManagement/Development/Services/RequestDB/localURL') self.centralUrl = gConfig.getValue( '/Systems/DataManagement/Development/Services/RequestDB/centralURL' ) voBoxUrls = gConfig.getValue( '/Systems/DataManagement/Development/Services/RequestDB/voBoxURLs') self.voBoxUrls = randomize(voBoxUrls).remove(self.localUrl)
def putRequest(self, request): """ put request to RequestManager :param self: self reference :param Request request: Request instance """ errorsDict = {"OK": False} valid = self.requestValidator().validate(request) if not valid["OK"]: self.log.error("putRequest: request not valid", "%s" % valid["Message"]) return valid # # dump to json requestJSON = request.toJSON() if not requestJSON["OK"]: return requestJSON requestJSON = requestJSON["Value"] setRequestMgr = self.requestManager().putRequest(requestJSON) if setRequestMgr["OK"]: return setRequestMgr errorsDict["RequestManager"] = setRequestMgr["Message"] self.log.warn( "putRequest: unable to set request '%s' at RequestManager" % request.RequestName, setRequestMgr["Message"]) proxies = self.requestProxies() for proxyURL in randomize(proxies.keys()): proxyClient = proxies[proxyURL] self.log.debug("putRequest: trying RequestProxy at %s" % proxyURL) setRequestProxy = proxyClient.putRequest(requestJSON) if setRequestProxy["OK"]: if setRequestProxy["Value"]["set"]: self.log.info( "putRequest: request '%s' successfully set using RequestProxy %s" % (request.RequestName, proxyURL)) elif setRequestProxy["Value"]["saved"]: self.log.info( "putRequest: request '%s' successfully forwarded to RequestProxy %s" % (request.RequestName, proxyURL)) return setRequestProxy else: self.log.warn( "putRequest: unable to set request using RequestProxy %s: %s" % (proxyURL, setRequestProxy["Message"])) errorsDict["RequestProxy(%s)" % proxyURL] = setRequestProxy["Message"] # # if we're here neither requestManager nor requestProxy were successful self.log.error("putRequest: unable to set request", "'%s'" % request.RequestName) errorsDict[ "Message"] = "ReqClient.putRequest: unable to set request '%s'" % request.RequestName return errorsDict
def __init__( self, useCertificates = False ): """ Constructor of the RequestClient class """ voBoxUrls = fromChar( PathFinder.getServiceURL( "RequestManagement/voBoxURLs" ) ) self.voBoxUrls = [] if voBoxUrls: self.voBoxUrls = randomize( voBoxUrls ) local = PathFinder.getServiceURL( "RequestManagement/localURL" ) self.local = False if local: self.local = local central = PathFinder.getServiceURL( "RequestManagement/centralURL" ) self.central = False if central: self.central = central self.setServer( 'RequestManagement/centralURL' )
def putRequest( self, request ): """ put request to RequestManager :param self: self reference :param Request request: Request instance """ errorsDict = { "OK" : False } valid = self.requestValidator().validate( request ) if not valid["OK"]: self.log.error( "putRequest: request not valid: %s" % valid["Message"] ) return valid # # dump to json requestJSON = request.toJSON() if not requestJSON["OK"]: return requestJSON requestJSON = requestJSON["Value"] setRequestMgr = self.requestManager().putRequest( requestJSON ) if setRequestMgr["OK"]: return setRequestMgr errorsDict["RequestManager"] = setRequestMgr["Message"] self.log.warn( "putRequest: unable to set request '%s' at RequestManager" % request.RequestName ) proxies = self.requestProxies() for proxyURL in randomize( proxies.keys() ): proxyClient = proxies[proxyURL] self.log.debug( "putRequest: trying RequestProxy at %s" % proxyURL ) setRequestProxy = proxyClient.putRequest( requestJSON ) if setRequestProxy["OK"]: if setRequestProxy["Value"]["set"]: self.log.info( "putRequest: request '%s' successfully set using RequestProxy %s" % ( request.RequestName, proxyURL ) ) elif setRequestProxy["Value"]["saved"]: self.log.info( "putRequest: request '%s' successfully forwarded to RequestProxy %s" % ( request.RequestName, proxyURL ) ) return setRequestProxy else: self.log.warn( "putRequest: unable to set request using RequestProxy %s: %s" % ( proxyURL, setRequestProxy["Message"] ) ) errorsDict["RequestProxy(%s)" % proxyURL] = setRequestProxy["Message"] # # if we're here neither requestManager nor requestProxy were successful self.log.error( "putRequest: unable to set request '%s'" % request.RequestName ) errorsDict["Message"] = "ReqClient.putRequest: unable to set request '%s'" % request.RequestName return errorsDict
def __init__(self, useCertificates=False): """ Constructor of the RequestClient class """ voBoxUrls = fromChar( PathFinder.getServiceURL("RequestManagement/voBoxURLs")) self.voBoxUrls = [] if voBoxUrls: self.voBoxUrls = randomize(voBoxUrls) local = PathFinder.getServiceURL("RequestManagement/localURL") self.local = False if local: self.local = local central = PathFinder.getServiceURL("RequestManagement/centralURL") self.central = False if central: self.central = central self.setServer('RequestManagement/centralURL')
def __applyReduction( self, lfns ): """ eventually remove the number of files to be considered """ if len( lfns ) <= self.maxFiles: return lfns return randomize( lfns )[:self.maxFiles]
def _Broadcast(self): """ This plug-in takes files found at the sourceSE and broadcasts to all (or a selection of) targetSEs. """ if not self.params: return S_ERROR( "TransformationPlugin._Broadcast: The 'Broadcast' plugin requires additional parameters." ) sourceseParam = self.params['SourceSE'] targetseParam = self.params['TargetSE'] sourceSEs = [] targetSEs = [] if sourceseParam.count('['): #assume it's an array sourceSEs = eval(sourceseParam) else: sourceSEs = [sourceseParam] sourceSEs = eval(self.params['SourceSE']) if targetseParam.count('['): targetSEs = eval(targetseParam) elif type(targetseParam) == type([]): targetSEs = targetseParam else: targetSEs = [targetseParam] #sourceSEs = eval(self.params['SourceSE']) #targetSEs = eval(self.params['TargetSE']) destinations = int(self.params.get('Destinations', 0)) if destinations and (destinations >= len(targetSEs)): destinations = 0 status = self.params['Status'] groupSize = self.params['GroupSize'] #Number of files per tasks fileGroups = self._getFileGroups(self.data) #groups by SE targetSELfns = {} for replicaSE, lfns in fileGroups.items(): ses = replicaSE.split(',') #sourceSites = self._getSitesForSEs(ses) atSource = False for se in ses: if se in sourceSEs: atSource = True if not atSource: continue for lfn in lfns: targets = [] sources = self._getSitesForSEs(ses) for targetSE in randomize(targetSEs): site = self._getSiteForSE(targetSE)['Value'] if not site in sources: if (destinations) and (len(targets) >= destinations): continue sources.append(site) targets.append( targetSE ) #after all, if someone wants to copy to the source, it's his choice strTargetSEs = str.join(',', sortList(targets)) if not targetSELfns.has_key(strTargetSEs): targetSELfns[strTargetSEs] = [] targetSELfns[strTargetSEs].append(lfn) tasks = [] for ses, lfns in targetSELfns.items(): tasksLfns = breakListIntoChunks(lfns, groupSize) for taskLfns in tasksLfns: if (status == 'Flush') or (len(taskLfns) >= int(groupSize)): #do not allow groups smaller than the groupSize, except if transformation is in flush state tasks.append((ses, taskLfns)) return S_OK(tasks)
def _Broadcast( self ): """ This plug-in takes files found at the sourceSE and broadcasts to all (or a selection of) targetSEs. """ if not self.params: return S_ERROR( "TransformationPlugin._Broadcast: The 'Broadcast' plugin requires additional parameters." ) sourceseParam = self.params['SourceSE'] targetseParam = self.params['TargetSE'] sourceSEs = [] targetSEs = [] if sourceseParam.count( '[' ):#assume it's an array sourceSEs = eval( sourceseParam ) else: sourceSEs = [sourceseParam] sourceSEs = eval( self.params['SourceSE'] ) if targetseParam.count( '[' ): targetSEs = eval( targetseParam ) elif type(targetseParam)==type([]): targetSEs = targetseParam else: targetSEs = [targetseParam] #sourceSEs = eval(self.params['SourceSE']) #targetSEs = eval(self.params['TargetSE']) destinations = int( self.params.get( 'Destinations', 0 ) ) if destinations and ( destinations >= len(targetSEs) ): destinations = 0 status = self.params['Status'] groupSize = self.params['GroupSize']#Number of files per tasks fileGroups = self._getFileGroups( self.data )#groups by SE targetSELfns = {} for replicaSE, lfns in fileGroups.items(): ses = replicaSE.split( ',' ) #sourceSites = self._getSitesForSEs(ses) atSource = False for se in ses: if se in sourceSEs: atSource = True if not atSource: continue for lfn in lfns: targets = [] sources = self._getSitesForSEs( ses ) for targetSE in randomize( targetSEs ): site = self._getSiteForSE( targetSE )['Value'] if not site in sources: if ( destinations ) and ( len( targets ) >= destinations ): continue sources.append( site ) targets.append( targetSE )#after all, if someone wants to copy to the source, it's his choice strTargetSEs = str.join( ',', sortList( targets ) ) if not targetSELfns.has_key( strTargetSEs ): targetSELfns[strTargetSEs] = [] targetSELfns[strTargetSEs].append( lfn ) tasks = [] for ses, lfns in targetSELfns.items(): tasksLfns = breakListIntoChunks(lfns, groupSize) for taskLfns in tasksLfns: if ( status == 'Flush' ) or ( len( taskLfns ) >= int( groupSize ) ): #do not allow groups smaller than the groupSize, except if transformation is in flush state tasks.append( ( ses, taskLfns ) ) return S_OK( tasks )
def __applyReduction(self, lfns): """ eventually remove the number of files to be considered """ if len(lfns) <= self.maxFiles: return lfns return randomize(lfns)[:self.maxFiles]