Пример #1
0
    def createCatalog(self, catalogName, useProxy=False):
        """ Create a file catalog object from its name and CS description
    """
        catalogPath = getCatalogPath(catalogName)
        catalogType = gConfig.getValue(catalogPath + '/CatalogType',
                                       catalogName)
        catalogURL = gConfig.getValue(catalogPath + '/CatalogURL',
                                      "DataManagement/" + catalogType)
        optionsDict = {}
        result = gConfig.getOptionsDict(catalogPath)
        if result['OK']:
            optionsDict = result['Value']

        if useProxy:
            result = self.__getCatalogClass(catalogType)
            if not result['OK']:
                return result
            catalogClass = result['Value']
            methods = catalogClass.getInterfaceMethods()
            catalog = FileCatalogProxyClient(catalogName)
            catalog.setInterfaceMethods(methods)
            return S_OK(catalog)

        return self.__createCatalog(catalogName, catalogType, catalogURL,
                                    optionsDict)
Пример #2
0
    def createCatalog(self, catalogName):
        """ Create a file catalog object from its name and CS description
    """
        # get the CS description first
        catalogPath = getCatalogPath(catalogName)
        catalogType = gConfig.getValue(catalogPath + "/CatalogType", catalogName)
        catalogURL = gConfig.getValue(catalogPath + "/CatalogURL", "")

        self.log.verbose("Creating %s client" % catalogName)
        moduleRootPaths = getInstalledExtensions()
        for moduleRootPath in moduleRootPaths:
            gLogger.verbose("Trying to load from root path %s" % moduleRootPath)
            # moduleFile = os.path.join( rootPath, moduleRootPath, "Resources", "Catalog", "%sClient.py" % catalogType )
            # gLogger.verbose( "Looking for file %s" % moduleFile )
            # if not os.path.isfile( moduleFile ):
            #  continue
            try:
                # This enforces the convention that the plug in must be named after the file catalog
                moduleName = "%sClient" % (catalogType)
                catalogModule = __import__(
                    "%s.Resources.Catalog.%s" % (moduleRootPath, moduleName), globals(), locals(), [moduleName]
                )
            except ImportError, x:
                if "No module" in str(x):
                    gLogger.debug("Catalog module %s not found in %s" % (catalogType, moduleRootPath))
                else:
                    errStr = "Failed attempt to import %s from the path %s: %s" % (catalogType, moduleRootPath, x)
                    gLogger.error(errStr)
                continue
            except Exception, x:
                errStr = "Failed attempt to import %s from the path %s: %s" % (catalogType, moduleRootPath, x)
                gLogger.error(errStr)
                continue
Пример #3
0
    def createCatalog(self, catalogName, useProxy=False):
        """ Create a file catalog object from its name and CS description
    """
        if useProxy:
            catalog = FileCatalogProxyClient(catalogName)
            return S_OK(catalog)

        # get the CS description first
        catalogPath = getCatalogPath(catalogName)
        catalogType = gConfig.getValue(catalogPath + '/CatalogType',
                                       catalogName)
        catalogURL = gConfig.getValue(catalogPath + '/CatalogURL',
                                      "DataManagement/" + catalogType)

        self.log.debug('Creating %s client' % catalogName)

        objectLoader = ObjectLoader.ObjectLoader()
        result = objectLoader.loadObject(
            'Resources.Catalog.%sClient' % catalogType, catalogType + 'Client')
        if not result['OK']:
            gLogger.error('Failed to load catalog object: %s' %
                          result['Message'])
            return result

        catalogClass = result['Value']

        try:
            # FIXME: is it really needed? This is the factory, can't this be moved out?
            if catalogType in ['LcgFileCatalogCombined', 'LcgFileCatalog']:
                # The LFC special case
                infoSys = gConfig.getValue(catalogPath + '/LcgGfalInfosys', '')
                host = gConfig.getValue(catalogPath + '/MasterHost', '')
                catalog = catalogClass(infoSys, host)
            else:
                if catalogURL:
                    catalog = catalogClass(url=catalogURL)
                else:
                    catalog = catalogClass()
            self.log.debug('Loaded module %sClient' % catalogType)
            return S_OK(catalog)
        except Exception, x:
            errStr = "Failed to instantiate %s()" % (catalogType)
            gLogger.exception(errStr, lException=x)
            return S_ERROR(errStr)
Пример #4
0
    def createCatalog(self, catalogName, useProxy=False):
        """ Create a file catalog object from its name and CS description
    """
        if useProxy:
            catalog = FileCatalogProxyClient(catalogName)
            return S_OK(catalog)

        # get the CS description first
        catalogPath = getCatalogPath(catalogName)
        catalogType = gConfig.getValue(catalogPath + '/CatalogType',
                                       catalogName)
        catalogURL = gConfig.getValue(catalogPath + '/CatalogURL', '')

        self.log.verbose('Creating %s client' % catalogName)
        moduleRootPaths = getInstalledExtensions()
        for moduleRootPath in moduleRootPaths:
            gLogger.verbose("Trying to load from root path %s" %
                            moduleRootPath)
            #moduleFile = os.path.join( rootPath, moduleRootPath, "Resources", "Catalog", "%sClient.py" % catalogType )
            #gLogger.verbose( "Looking for file %s" % moduleFile )
            #if not os.path.isfile( moduleFile ):
            #  continue
            try:
                # This enforces the convention that the plug in must be named after the file catalog
                moduleName = "%sClient" % (catalogType)
                catalogModule = __import__(
                    '%s.Resources.Catalog.%s' % (moduleRootPath, moduleName),
                    globals(), locals(), [moduleName])
            except ImportError, x:
                if "No module" in str(x):
                    gLogger.debug('Catalog module %s not found in %s' %
                                  (catalogType, moduleRootPath))
                else:
                    errStr = "Failed attempt to import %s from the path %s: %s" % (
                        catalogType, moduleRootPath, x)
                    gLogger.error(errStr)
                continue
            except Exception, x:
                errStr = "Failed attempt to import %s from the path %s: %s" % (
                    catalogType, moduleRootPath, x)
                gLogger.error(errStr)
                continue
Пример #5
0
  def createCatalog( self, catalogName, useProxy = False ):
    """ Create a file catalog object from its name and CS description
    """
    if useProxy:
      catalog = FileCatalogProxyClient( catalogName )
      return S_OK( catalog )

    # get the CS description first
    catalogPath = getCatalogPath( catalogName )
    catalogType = gConfig.getValue( catalogPath + '/CatalogType', catalogName )
    catalogURL = gConfig.getValue( catalogPath + '/CatalogURL', "DataManagement/" + catalogType )

    self.log.debug( 'Creating %s client' % catalogName )

    objectLoader = ObjectLoader.ObjectLoader()
    result = objectLoader.loadObject( 'Resources.Catalog.%sClient' % catalogType, catalogType + 'Client' )
    if not result['OK']:
      gLogger.error( 'Failed to load catalog object', '%s' % result['Message'] )
      return result

    catalogClass = result['Value']

    try:
      # FIXME: is it really needed? This is the factory, can't this be moved out?
      if catalogType in [ 'LcgFileCatalogCombined', 'LcgFileCatalog' ]:
        # The LFC special case
        infoSys = gConfig.getValue( catalogPath + '/LcgGfalInfosys', '' )
        host = gConfig.getValue( catalogPath + '/MasterHost', '' )
        catalog = catalogClass( infoSys, host )
      else:
        if catalogURL:
          catalog = catalogClass( url = catalogURL )
        else:
          catalog = catalogClass()
      self.log.debug( 'Loaded module %sClient' % catalogType )
      return S_OK( catalog )
    except Exception, x:
      errStr = "Failed to instantiate %s()" % ( catalogType )
      gLogger.exception( errStr, lException = x )
      return S_ERROR( errStr )
Пример #6
0
  def createCatalog( self, catalogName, useProxy = False ):
    """ Create a file catalog object from its name and CS description
    """
    catalogPath = getCatalogPath( catalogName )
    catalogType = gConfig.getValue( catalogPath + '/CatalogType', catalogName )
    catalogURL = gConfig.getValue( catalogPath + '/CatalogURL', "DataManagement/" + catalogType )
    optionsDict = {}
    result = gConfig.getOptionsDict( catalogPath )
    if result['OK']:
      optionsDict = result['Value']

    if useProxy:
      result = self.__getCatalogClass( catalogType )
      if not result['OK']:
        return result
      catalogClass = result['Value']
      methods = catalogClass.getInterfaceMethods()
      catalog = FileCatalogProxyClient( catalogName )
      catalog.setInterfaceMethods( methods )
      return S_OK( catalog )

    return self.__createCatalog( catalogName, catalogType, catalogURL, optionsDict )
Пример #7
0
    def createCatalog(self, catalogName, useProxy=False):
        """ Create a file catalog object from its name and CS description
    """
        if useProxy:
            catalog = FileCatalogProxyClient(catalogName)
            return S_OK(catalog)

        # get the CS description first
        catalogPath = getCatalogPath(catalogName)
        catalogType = gConfig.getValue(catalogPath + "/CatalogType", catalogName)
        catalogURL = gConfig.getValue(catalogPath + "/CatalogURL", "DataManagement/" + catalogName)

        self.log.verbose("Creating %s client" % catalogName)

        objectLoader = ObjectLoader.ObjectLoader()
        result = objectLoader.loadObject("Resources.Catalog.%sClient" % catalogType, catalogType + "Client")
        if not result["OK"]:
            gLogger.error("Failed to load catalog object: %s" % result["Message"])
            return result

        catalogClass = result["Value"]

        try:
            if catalogType in ["LcgFileCatalogCombined", "LcgFileCatalog"]:
                # The LFC special case
                infoSys = gConfig.getValue(catalogPath + "/LcgGfalInfosys", "")
                host = gConfig.getValue(catalogPath + "/MasterHost", "")
                catalog = catalogClass(infoSys, host)
            else:
                if catalogURL:
                    catalog = catalogClass(url=catalogURL)
                else:
                    catalog = catalogClass()
            self.log.debug("Loaded module %sClient" % catalogType)
            return S_OK(catalog)
        except Exception, x:
            errStr = "Failed to instantiate %s()" % (catalogType)
            gLogger.exception(errStr, lException=x)
            return S_ERROR(errStr)
Пример #8
0
def configHelper(voList):
    """
    A helper function to gather necessary Rucio client options from the CS.

    :param volist: list of VO names, or  a VO name  (str)
    :return: a dictionary of a form {vo: params, vo: params,}
    :rtype: dict
    """
    log = gLogger.getLocalSubLogger("RucioSynchronizerHelper")

    if isinstance(voList, str):
        voList = [voList]
    clientConfig = {}
    log.debug("VO list to consider for synchronization: ", voList)
    # locate RucioFileCatalog type in resources first
    result = gConfig.getSections("/Resources/FileCatalogs")
    catNames = []
    if result["OK"]:
        catalogs = result["Value"]
        log.debug("File catalogs defined in Resources", catalogs)
        for catalog in catalogs:
            result = gConfig.getOptionsDict(getCatalogPath(catalog))
            if result["OK"]:
                options = result["Value"]
                log.debug("Rucio Catalog candidate options", options)
                if options.get("Status", None) == "Active" and options.get(
                        "CatalogType", None) == "RucioFileCatalog":
                    catNames.append(catalog)
    else:
        log.error("No catalogs defined in Resources.")
        return S_ERROR("No catalogs defined in Resources.")

    log.info(
        "Active FileCatalogs candidates of type RucioFileCatalog found in Resources:",
        catNames)
    # we found (possibly more that one) candidate, now we look for it in Operations
    # to find out which one is used by which VO. There can be only one
    # Rucio catalog per VO.

    for vo in voList:
        opHelper = Operations(vo=vo)
        result = opHelper.getSections("/Services/Catalogs")
        if result["OK"]:
            catSections = set(result["Value"])
        else:
            log.warn("No Services/Catalogs section in Operations, for ",
                     "VO=%s (skipped)" % vo)
            continue

        selectedCatalog = list(catSections.intersection(catNames))

        if len(selectedCatalog) > 1:
            log.error(
                "VO %s: Services/Catalogs section mis-configured."
                " More that one Rucio file catalog",
                "[VO: %s, Catalogs: %s]" % (vo, selectedCatalog),
            )
            continue

        if not selectedCatalog:
            log.warn("VO is not using RucioFileCatalog  (VO skipped)",
                     "[VO: %s]" % vo)
            continue

        # check if the section name is in the catalog list to use.
        # if the list is not empty it has to contain the selected catalog.
        fileCatalogs = opHelper.getValue("/Services/Catalogs/CatalogList", [])

        if fileCatalogs and selectedCatalog[0] not in fileCatalogs:
            log.warn(
                "VO is not using RucioFileCatalog - it is not in the catalog list",
                "[VO: %s]" % vo)
            continue
        # now collect Rucio specific parameters for the VO
        params = {}
        result = gConfig.getOptionsDict(getCatalogPath(selectedCatalog[0]))
        if result["OK"]:
            optDict = result["Value"]
            params["rucioHost"] = optDict.get("RucioHost", None)
            params["authHost"] = optDict.get("AuthHost", None)
            params["privilegedAccount"] = optDict.get("PrivilegedAccount",
                                                      "root")
            clientConfig[vo] = params
            log.info("RSEs and users will be configured in Rucio for the VO:",
                     vo)
        else:
            log.error(result["Message"])
    return clientConfig