Ejemplo n.º 1
0
  def __generateStorageObject(self, storageName, pluginName, parameters, hideExceptions=False):
    """
      Generate a Storage Element from parameters collected

      :param storageName: is the storage section to check in the CS
      :param pluginName: name of the plugin used. Example: GFAL2_XROOT, GFAL2_SRM2...
      :param parameters: dictionary of protocol details.
    """

    storageType = pluginName
    if self.proxy:
      storageType = 'Proxy'

    objectLoader = ObjectLoader()
    result = objectLoader.loadObject('Resources.Storage.%sStorage' % storageType, storageType + 'Storage',
                                     hideExceptions=hideExceptions)
    if not result['OK']:
      gLogger.error('Failed to load storage object: %s' % result['Message'])
      return result

    storageClass = result['Value']
    try:
      storage = storageClass(storageName, parameters)
    except Exception as x:
      errStr = "StorageFactory._generateStorageObject: Failed to instantiate %s: %s" % (storageName, x)
      gLogger.exception(errStr)
      return S_ERROR(errStr)

    return S_OK(storage)
Ejemplo n.º 2
0
    def initializeHandler(cls, svcInfoDict):
        """WMS AdministratorService initialization"""
        try:
            result = ObjectLoader().loadObject(
                "WorkloadManagementSystem.DB.JobDB", "JobDB")
            if not result["OK"]:
                return result
            cls.jobDB = result["Value"](parentLogger=cls.log)
        except RuntimeError as excp:
            return S_ERROR(f"Can't connect to DB: {excp!r}")

        cls.elasticJobParametersDB = None
        useESForJobParametersFlag = Operations().getValue(
            "/Services/JobMonitoring/useESForJobParametersFlag", False)
        if useESForJobParametersFlag:
            try:
                result = ObjectLoader().loadObject(
                    "WorkloadManagementSystem.DB.ElasticJobParametersDB",
                    "ElasticJobParametersDB")
                if not result["OK"]:
                    return result
                cls.elasticJobParametersDB = result["Value"]()
            except RuntimeError as excp:
                return S_ERROR(f"Can't connect to DB: {excp!r}")

        cls.pilotManager = PilotManagerClient()

        return S_OK()
Ejemplo n.º 3
0
    def initializeHandler(cls, serviceInfoDict):
        """Initialization of DB objects"""

        try:
            result = ObjectLoader().loadObject(
                "WorkloadManagementSystem.DB.PilotAgentsDB", "PilotAgentsDB")
            if not result["OK"]:
                return result
            cls.pilotAgentsDB = result["Value"](parentLogger=cls.log)

        except RuntimeError as excp:
            return S_ERROR("Can't connect to DB: %s" % excp)

        cls.pilotsLoggingDB = None
        enablePilotsLogging = Operations().getValue(
            "/Services/JobMonitoring/usePilotsLoggingFlag", False)
        if enablePilotsLogging:
            try:
                result = ObjectLoader().loadObject(
                    "WorkloadManagementSystem.DB.PilotsLoggingDB",
                    "PilotsLoggingDB")
                if not result["OK"]:
                    return result
                cls.pilotsLoggingDB = result["Value"](parentLogger=cls.log)
            except RuntimeError as excp:
                return S_ERROR("Can't connect to DB: %s" % excp)

        return S_OK()
Ejemplo n.º 4
0
    def initialize(self):
        """Standard initialize method"""
        res = TaskManagerAgentBase.initialize(self)
        if not res["OK"]:
            return res

        objLoader = ObjectLoader()
        _class = objLoader.loadObject(
            "TransformationSystem.Client.RequestTasks", "RequestTasks")

        if not _class["OK"]:
            raise Exception(_class["Message"])

        self.requestTasksCls = _class["Value"]

        # clients
        self.taskManager = self.requestTasksCls(transClient=self.transClient)

        agentTSTypes = self.am_getOption("TransType", [])
        if agentTSTypes:
            self.transType = agentTSTypes
        else:
            self.transType = Operations().getValue(
                "Transformations/DataManipulation", ["Replication", "Removal"])

        return S_OK()
Ejemplo n.º 5
0
    def initializeHandler(cls, svcInfoDict):
        """
        Determines the switching of ElasticSearch and MySQL backends
        """
        try:
            result = ObjectLoader().loadObject(
                "WorkloadManagementSystem.DB.JobDB", "JobDB")
            if not result["OK"]:
                return result
            cls.jobDB = result["Value"]()

            result = ObjectLoader().loadObject(
                "WorkloadManagementSystem.DB.JobLoggingDB", "JobLoggingDB")
            if not result["OK"]:
                return result
            cls.jobLoggingDB = result["Value"]()

        except RuntimeError as excp:
            return S_ERROR("Can't connect to DB: %s" % excp)

        cls.elasticJobParametersDB = None
        useESForJobParametersFlag = Operations().getValue(
            "/Services/JobMonitoring/useESForJobParametersFlag", False)
        if useESForJobParametersFlag:
            try:
                result = ObjectLoader().loadObject(
                    "WorkloadManagementSystem.DB.ElasticJobParametersDB",
                    "ElasticJobParametersDB")
                if not result["OK"]:
                    return result
                cls.elasticJobParametersDB = result["Value"]()
            except RuntimeError as excp:
                return S_ERROR("Can't connect to DB: %s" % excp)
        return S_OK()
Ejemplo n.º 6
0
    def initialize(self):
        """Sets default parameters and creates CE instance"""
        super(PushJobAgent, self).initialize()

        result = self._initializeComputingElement("Pool")
        if not result["OK"]:
            return result

        # on-the fly imports
        ol = ObjectLoader()
        res = ol.loadModule("ConfigurationSystem.Client.Helpers.Resources")
        if not res["OK"]:
            sys.exit(res["Message"])
        self.resourcesModule = res["Value"]
        self.opsHelper = Operations()

        # Disable Watchdog: we don't need it as pre/post processing occurs locally
        setup = gConfig.getValue("/DIRAC/Setup", "")
        if not setup:
            return S_ERROR("Cannot get the DIRAC Setup value")
        wms_instance = getSystemInstance("WorkloadManagement")
        if not wms_instance:
            return S_ERROR("Cannot get the WorkloadManagement system instance")
        section = "/Systems/WorkloadManagement/%s/JobWrapper" % wms_instance
        self._updateConfiguration("CheckWallClockFlag", 0, path=section)
        self._updateConfiguration("CheckDiskSpaceFlag", 0, path=section)
        self._updateConfiguration("CheckLoadAvgFlag", 0, path=section)
        self._updateConfiguration("CheckCPUConsumedFlag", 0, path=section)
        self._updateConfiguration("CheckCPULimitFlag", 0, path=section)
        self._updateConfiguration("CheckMemoryLimitFlag", 0, path=section)
        self._updateConfiguration("CheckTimeLeftFlag", 0, path=section)

        return S_OK()
Ejemplo n.º 7
0
    def initializeHandler(cls, serviceInfoDict):
        try:
            result = ObjectLoader().loadObject("WorkloadManagementSystem.DB.JobDB", "JobDB")
            if not result["OK"]:
                return result
            cls.jobDB = result["Value"]()

            result = ObjectLoader().loadObject("WorkloadManagementSystem.DB.JobLoggingDB", "JobLoggingDB")
            if not result["OK"]:
                return result
            cls.jobLoggingDB = result["Value"]()

            result = ObjectLoader().loadObject("WorkloadManagementSystem.DB.TaskQueueDB", "TaskQueueDB")
            if not result["OK"]:
                return result
            cls.taskQueueDB = result["Value"]()

            result = ObjectLoader().loadObject("WorkloadManagementSystem.DB.PilotAgentsDB", "PilotAgentsDB")
            if not result["OK"]:
                return result
            cls.pilotAgentsDB = result["Value"]()

        except RuntimeError as excp:
            return S_ERROR("Can't connect to DB: %s" % excp)

        cls.limiter = Limiter(jobDB=cls.jobDB)

        gMonitor.registerActivity("matchTime", "Job matching time", "Matching", "secs", gMonitor.OP_MEAN, 300)
        gMonitor.registerActivity("matchesDone", "Job Match Request", "Matching", "matches", gMonitor.OP_RATE, 300)
        gMonitor.registerActivity("matchesOK", "Matched jobs", "Matching", "matches", gMonitor.OP_RATE, 300)
        gMonitor.registerActivity("numTQs", "Number of Task Queues", "Matching", "tqsk queues", gMonitor.OP_MEAN, 300)

        return S_OK()
Ejemplo n.º 8
0
    def initialize(self):
        """Standard initialize."""

        res = ObjectLoader().loadObject(
            "DIRAC.ResourceStatusSystem.Client.ResourceManagementClient")
        if not res["OK"]:
            self.log.error(
                "Failed to load ResourceManagementClient class: %s" %
                res["Message"])
            return res
        rmClass = res["Value"]

        res = ObjectLoader().loadObject(
            "DIRAC.ResourceStatusSystem.Client.ResourceStatusClient")
        if not res["OK"]:
            self.log.error("Failed to load ResourceStatusClient class: %s" %
                           res["Message"])
            return res
        rsClass = res["Value"]

        self.rsClient = rsClass()
        self.clients["ResourceStatusClient"] = rsClass()
        self.clients["ResourceManagementClient"] = rmClass()

        maxNumberOfThreads = self.am_getOption("maxNumberOfThreads", 15)
        self.log.info("Multithreaded with %d threads" % maxNumberOfThreads)
        self.threadPoolExecutor = concurrent.futures.ThreadPoolExecutor(
            max_workers=maxNumberOfThreads)

        return S_OK()
Ejemplo n.º 9
0
    def initialize(self):
        """ Standard initialize.
    """

        maxNumberOfThreads = self.am_getOption('maxNumberOfThreads',
                                               self.__maxNumberOfThreads)
        self.threadPool = ThreadPool(maxNumberOfThreads, maxNumberOfThreads)

        res = ObjectLoader().loadObject(
            'DIRAC.ResourceStatusSystem.Client.SiteStatus', 'SiteStatus')
        if not res['OK']:
            self.log.error('Failed to load SiteStatus class: %s' %
                           res['Message'])
            return res
        siteStatusClass = res['Value']

        res = ObjectLoader().loadObject(
            'DIRAC.ResourceStatusSystem.Client.ResourceManagementClient',
            'ResourceManagementClient')
        if not res['OK']:
            self.log.error(
                'Failed to load ResourceManagementClient class: %s' %
                res['Message'])
            return res
        rmClass = res['Value']

        self.siteClient = siteStatusClass()
        self.clients['SiteStatus'] = siteStatusClass()
        self.clients['ResourceManagementClient'] = rmClass()

        return S_OK()
Ejemplo n.º 10
0
  def registerBackend(self, desiredBackend, backendOptions=None):
    """
    Attach a backend to the Logging object.
    Convert backend name to backend class name to a Backend object and add it to the Logging object

    :params desiredBackend: a name attaching to a backend type.
                            list of the possible values: ['stdout', 'stderr', 'file', 'server']
    :params backendOptions: dictionary of different backend options.
                            example: FileName='/tmp/log.txt'
    """
    # import ObjectLoader here to avoid a dependancy loop
    from DIRAC.Core.Utilities.ObjectLoader import ObjectLoader
    objLoader = ObjectLoader()

    # Remove white space and capitalize the first letter
    desiredBackend = desiredBackend.strip()
    desiredBackend = desiredBackend[0].upper() + desiredBackend[1:]

    # lock to avoid problem in ObjectLoader which is a singleton not
    # thread-safe
    self._lockObjectLoader.acquire()
    try:
      # load the Backend class
      _class = objLoader.loadObject('Resources.LogBackends.%sBackend' % desiredBackend)
    finally:
      self._lockObjectLoader.release()

    if _class['OK']:
      # add the backend instance to the Logging
      self._addBackend(_class['Value'](), backendOptions)
      self._generateBackendFormat()
    else:
      self._generateBackendFormat()
      self.warn("%s is not a valid backend name." % desiredBackend)
Ejemplo n.º 11
0
    def __generateStorageObject(self,
                                storageName,
                                pluginName,
                                parameters,
                                hideExceptions=False):

        storageType = pluginName
        if self.proxy:
            storageType = 'Proxy'

        objectLoader = ObjectLoader()
        result = objectLoader.loadObject('Resources.Storage.%sStorage' %
                                         storageType,
                                         storageType + 'Storage',
                                         hideExceptions=hideExceptions)
        if not result['OK']:
            gLogger.error('Failed to load storage object: %s' %
                          result['Message'])
            return result

        storageClass = result['Value']
        try:
            storage = storageClass(storageName, parameters)
        except Exception, x:
            errStr = "StorageFactory._generateStorageObject: Failed to instantiate %s: %s" % (
                storageName, x)
            gLogger.exception(errStr)
            return S_ERROR(errStr)
Ejemplo n.º 12
0
    def initializeHandler(cls, serviceInfoDict):
        try:
            result = ObjectLoader().loadObject(
                "WorkloadManagementSystem.DB.JobDB", "JobDB")
            if not result["OK"]:
                return result
            cls.jobDB = result["Value"](parentLogger=cls.log)

            result = ObjectLoader().loadObject(
                "WorkloadManagementSystem.DB.JobLoggingDB", "JobLoggingDB")
            if not result["OK"]:
                return result
            cls.jobLoggingDB = result["Value"](parentLogger=cls.log)

            result = ObjectLoader().loadObject(
                "WorkloadManagementSystem.DB.TaskQueueDB", "TaskQueueDB")
            if not result["OK"]:
                return result
            cls.taskQueueDB = result["Value"](parentLogger=cls.log)

            result = ObjectLoader().loadObject(
                "WorkloadManagementSystem.DB.PilotAgentsDB", "PilotAgentsDB")
            if not result["OK"]:
                return result
            cls.pilotAgentsDB = result["Value"](parentLogger=cls.log)

        except RuntimeError as excp:
            return S_ERROR("Can't connect to DB: %s" % excp)

        cls.limiter = Limiter(jobDB=cls.jobDB)

        return S_OK()
Ejemplo n.º 13
0
    def registerBackend(self, desiredBackend, backendOptions=None):
        """
    Attach a backend to the Logging object.
    Convert backend name to backend class name to a Backend object and add it to the Logging object

    :params desiredBackend: a name attaching to a backend type.
                            list of the possible values: ['stdout', 'stderr', 'file', 'server']
    :params backendOptions: dictionary of different backend options.
                            example: FileName='/tmp/log.txt'
    """
        # import ObjectLoader here to avoid a dependancy loop
        from DIRAC.Core.Utilities.ObjectLoader import ObjectLoader
        objLoader = ObjectLoader()

        # Remove white space and capitalize the first letter
        desiredBackend = desiredBackend.strip()
        desiredBackend = desiredBackend[0].upper() + desiredBackend[1:]

        # lock to avoid problem in ObjectLoader which is a singleton not thread-safe
        self._lockObjectLoader.acquire()
        try:
            # load the Backend class
            _class = objLoader.loadObject(
                'DIRAC.Resources.LogBackends.%sBackend' % desiredBackend)
        finally:
            self._lockObjectLoader.release()

        if _class['OK']:
            # add the backend instance to the Logging
            self._addBackend(_class['Value'](), backendOptions)
            self._generateBackendFormat()
        else:
            self._generateBackendFormat()
            self.warn("%s is not a valid backend name." % desiredBackend)
Ejemplo n.º 14
0
  def initialize(self):
    """ Define the commands to be executed, and instantiate the clients that will be used.
    """

    res = ObjectLoader().loadObject('DIRAC.ResourceStatusSystem.Client.ResourceStatusClient',
                                    'ResourceStatusClient')
    if not res['OK']:
      self.log.error('Failed to load ResourceStatusClient class: %s' % res['Message'])
      return res
    rsClass = res['Value']

    res = ObjectLoader().loadObject('DIRAC.ResourceStatusSystem.Client.ResourceManagementClient',
                                    'ResourceManagementClient')
    if not res['OK']:
      self.log.error('Failed to load ResourceManagementClient class: %s' % res['Message'])
      return res
    rmClass = res['Value']

    self.commands['Downtime'] = [{'Downtime': {}}]
    self.commands['GOCDBSync'] = [{'GOCDBSync': {}}]
    self.commands['FreeDiskSpace'] = [{'FreeDiskSpace': {}}]

    # PilotsCommand
#    self.commands[ 'Pilots' ] = [
#                                 { 'PilotsWMS' : { 'element' : 'Site', 'siteName' : None } },
#                                 { 'PilotsWMS' : { 'element' : 'Resource', 'siteName' : None } }
#                                 ]

    # FIXME: do not forget about hourly vs Always ...etc
    # AccountingCacheCommand
#    self.commands[ 'AccountingCache' ] = [
#                                          {'SuccessfullJobsBySiteSplitted'    :{'hours' :24, 'plotType' :'Job' }},
#                                          {'FailedJobsBySiteSplitted'         :{'hours' :24, 'plotType' :'Job' }},
#                                          {'SuccessfullPilotsBySiteSplitted'  :{'hours' :24, 'plotType' :'Pilot' }},
#                                          {'FailedPilotsBySiteSplitted'       :{'hours' :24, 'plotType' :'Pilot' }},
#                                          {'SuccessfullPilotsByCESplitted'    :{'hours' :24, 'plotType' :'Pilot' }},
#                                          {'FailedPilotsByCESplitted'         :{'hours' :24, 'plotType' :'Pilot' }},
#                                          {'RunningJobsBySiteSplitted'        :{'hours' :24, 'plotType' :'Job' }},
# #                                          {'RunningJobsBySiteSplitted'        :{'hours' :168, 'plotType' :'Job' }},
# #                                          {'RunningJobsBySiteSplitted'        :{'hours' :720, 'plotType' :'Job' }},
# #                                          {'RunningJobsBySiteSplitted'        :{'hours' :8760, 'plotType' :'Job' }},
#                                          ]

    # VOBOXAvailability
#    self.commands[ 'VOBOXAvailability' ] = [
#                                            { 'VOBOXAvailability' : {} }
#

    # Reuse clients for the commands
    self.clients['GOCDBClient'] = GOCDBClient()
    self.clients['ReportsClient'] = ReportsClient()
    self.clients['ResourceStatusClient'] = rsClass()
    self.clients['ResourceManagementClient'] = rmClass()
    self.clients['WMSAdministrator'] = WMSAdministratorClient()
    self.clients['Pilots'] = PilotManagerClient()

    self.cCaller = CommandCaller

    return S_OK()
Ejemplo n.º 15
0
    def __init__(self, clients=dict()):
        """ Constructor

    examples:
      >>> pep = PEP()
      >>> pep1 = PEP( { 'ResourceStatusClient' : ResourceStatusClient() } )
      >>> pep2 = PEP( { 'ResourceStatusClient' : ResourceStatusClient(), 'ClientY' : None } )

    :Parameters:
      **clients** - [ None, `dict` ]
        dictionary with clients to be used in the commands issued by the policies.
        If not defined, the commands will import them. It is a measure to avoid
        opening the same connection every time a policy is evaluated.

    """

        self.clients = dict(clients)

        # Creating the client in the PEP is a convenience for the PDP, that uses internally the RSS clients

        res = ObjectLoader().loadObject(
            'DIRAC.ResourceStatusSystem.Client.ResourceStatusClient',
            'ResourceStatusClient')
        if not res['OK']:
            self.log.error('Failed to load ResourceStatusClient class: %s' %
                           res['Message'])
            raise ImportError(res['Message'])
        rsClass = res['Value']

        res = ObjectLoader().loadObject(
            'DIRAC.ResourceStatusSystem.Client.ResourceManagementClient',
            'ResourceManagementClient')
        if not res['OK']:
            self.log.error(
                'Failed to load ResourceManagementClient class: %s' %
                res['Message'])
            raise ImportError(res['Message'])
        rmClass = res['Value']

        res = ObjectLoader().loadObject(
            'DIRAC.ResourceStatusSystem.Client.SiteStatus', 'SiteStatus')
        if not res['OK']:
            self.log.error('Failed to load SiteStatus class: %s' %
                           res['Message'])
            raise ImportError(res['Message'])
        ssClass = res['Value']

        if 'ResourceStatusClient' not in clients:
            self.clients['ResourceStatusClient'] = rsClass()
        if 'ResourceManagementClient' not in clients:
            self.clients['ResourceManagementClient'] = rmClass()
        if 'SiteStatus' not in clients:
            self.clients['SiteStatus'] = ssClass()

        # Pass to the PDP the clients that are going to be used on the Commands
        self.pdp = PDP(self.clients)

        self.log = gLogger
Ejemplo n.º 16
0
 def __calculateRoutes( self ):
   """
   Load all handlers and generate the routes
   """
   ol = ObjectLoader( [ 'WebAppDIRAC' ] )
   origin = "WebApp.handler"
   result = ol.getObjects( origin, parentClass = WebHandler, recurse = True )
   if not result[ 'OK' ]:
     return result
   self.__handlers = result[ 'Value' ]
   staticPaths = self.getPaths( "static" )
   self.log.verbose( "Static paths found:\n - %s" % "\n - ".join( staticPaths ) )
   self.__routes = []
   for pattern in ( ( r"/static/(.*)", r"/(favicon\.ico)", r"/(robots\.txt)" ) ):
     if self.__baseURL:
       pattern = "/%s%s" % ( self.__baseURL, pattern )
     self.__routes.append( ( pattern, StaticHandler, dict( pathList = staticPaths ) ) )
   for hn in self.__handlers:
     self.log.info( "Found handler %s" % hn  )
     handler = self.__handlers[ hn ]
     #CHeck it has AUTH_PROPS
     if type( handler.AUTH_PROPS ) == None:
       return S_ERROR( "Handler %s does not have AUTH_PROPS defined. Fix it!" % hn )
     #Get the root for the handler
     if handler.LOCATION:
       handlerRoute = handler.LOCATION.strip( "/")
     else:
       handlerRoute = hn[ len( origin ): ].replace( ".", "/" ).replace( "Handler", "" )
     #Add the setup group RE before
     baseRoute = self.__setupGroupRE
     #IF theres a base url like /DIRAC add it
     if self.__baseURL:
       baseRoute = "/%s%s" % ( self.__baseURL, baseRoute )
     #Set properly the LOCATION after calculating where it is with helpers to add group and setup later
     handler.LOCATION = handlerRoute
     handler.PATH_RE = re.compile( "%s(%s/.*)" % ( baseRoute, handlerRoute ) )
     handler.URLSCHEMA = "/%s%%(setup)s%%(group)s%%(location)s/%%(action)s" % ( self.__baseURL )
     #Look for methods that are exported
     for mName, mObj in inspect.getmembers( handler ):
       if inspect.ismethod( mObj ) and mName.find( "web_" ) == 0:
         if mName == "web_index":
           #Index methods have the bare url
           self.log.verbose( " - Route %s -> %s.web_index" % ( handlerRoute, hn ) )
           route = "%s(%s/)" % ( baseRoute, handlerRoute )
           self.__routes.append( ( route, handler ) )
           self.__routes.append( ( route.rstrip( "/" ), CoreHandler, dict( action = 'addSlash' ) ) )
         else:
           #Normal methods get the method appeded without web_
           self.log.verbose( " - Route %s/%s ->  %s.%s" % ( handlerRoute, mName[4:], hn, mName ) )
           route = "%s(%s/%s)" % ( baseRoute, handlerRoute, mName[4:] )
           self.__routes.append( ( route, handler ) )
         self.log.debug( "  * %s" % route )
   #Send to root
   self.__routes.append( ( "%s(/?)" % self.__setupGroupRE, CoreHandler, dict( action = "sendToRoot" ) ) )
   if self.__baseURL:
     self.__routes.append( ( "/%s%s()" % ( self.__baseURL, self.__setupGroupRE ),
                             CoreHandler, dict( action = "sendToRoot" ) ) )
   return S_OK()
Ejemplo n.º 17
0
    def createCatalog(self,
                      catalogName,
                      useProxy=False,
                      vo=None,
                      catalogConfig={}):
        """ Create a file catalog object from its name and CS description
    """
        if useProxy:
            catalog = FileCatalogProxyClient(catalogName)
            return S_OK(catalog)

        # get the CS description first
        catConfig = catalogConfig
        if not catConfig:
            if not vo:
                result = getVOfromProxyGroup()
                if not result['OK']:
                    return result
                vo = result['Value']
            reHelper = Resources(vo=vo)
            result = reHelper.getCatalogOptionsDict(catalogName)
            if not result['OK']:
                return result
            catConfig = result['Value']

        catalogType = catConfig.get('CatalogType', catalogName)
        catalogURL = catConfig.get('CatalogURL', '')

        self.log.verbose('Creating %s client' % catalogName)

        objectLoader = ObjectLoader()
        result = objectLoader.loadObject(
            'Resources.Catalog.%sClient' % catalogType, catalogType + 'Client')
        if not result['OK']:
            gLogger.error('Failed to load catalog object: %s' %
                          result['Message'])
            return result

        catalogClass = result['Value']

        try:
            if catalogType in ['LcgFileCatalogCombined', 'LcgFileCatalog']:
                # The LFC special case
                infoSys = catConfig.get('LcgGfalInfosys', '')
                host = catConfig.get('MasterHost', '')
                catalog = catalogClass(infoSys, host)
            else:
                if catalogURL:
                    catalog = catalogClass(url=catalogURL)
                else:
                    catalog = catalogClass()
            self.log.debug('Loaded module %sClient' % catalogType)
            return S_OK(catalog)
        except Exception, x:
            errStr = "Failed to instantiate %s()" % (catalogType)
            gLogger.exception(errStr, lException=x)
            return S_ERROR(errStr)
Ejemplo n.º 18
0
 def loadBatchSystem( self ):
   """ Instantiate object representing the backend batch system
   """
   self.batchSystem = self.ceParameters['BatchSystem']
   objectLoader = ObjectLoader()
   result = objectLoader.loadObject( 'Resources.Computing.BatchSystems.%s' % self.batchSystem, self.batchSystem )
   if not result['OK']:
     gLogger.error( 'Failed to load batch object: %s' % result['Message'] )
     return result
   batchClass = result['Value']
   self.batchModuleFile = result['ModuleFile']
   self.batch = batchClass()
   self.log.info( "Batch system class from module: ", self.batchModuleFile )
Ejemplo n.º 19
0
 def loadBatchSystem( self ):
   """ Instantiate object representing the backend batch system
   """
   self.batchSystem = self.ceParameters['BatchSystem']
   objectLoader = ObjectLoader()
   result = objectLoader.loadObject( 'Resources.Computing.BatchSystems.%s' % self.batchSystem, self.batchSystem )
   if not result['OK']:
     gLogger.error( 'Failed to load batch object: %s' % result['Message'] )
     return result
   batchClass = result['Value']
   self.batchModuleFile = result['ModuleFile']
   self.batch = batchClass()
   self.log.info( "Batch system class from module: ", self.batchModuleFile )
Ejemplo n.º 20
0
 def createCatalog( self, catalogName, useProxy = False, vo = None, catalogConfig = {} ):
   """ Create a file catalog object from its name and CS description
   """    
   if useProxy:
     catalog = FileCatalogProxyClient( catalogName )
     return S_OK( catalog )
   
   # get the CS description first
   catConfig = catalogConfig
   if not catConfig:
     if not vo:
       result = getVOfromProxyGroup()
       if not result['OK']:
         return result
       vo = result['Value']
     reHelper = Resources( vo = vo )
     result = reHelper.getCatalogOptionsDict( catalogName )
     if not result['OK']:
       return result
     catConfig = result['Value']
   
   catalogType = catConfig.get('CatalogType',catalogName)
   catalogURL = catConfig.get('CatalogURL','')
   
   self.log.verbose( 'Creating %s client' % catalogName )
   
   objectLoader = ObjectLoader()
   result = objectLoader.loadObject( 'Resources.Catalog.%sClient' % catalogType, catalogType+'Client' )
   if not result['OK']:
     gLogger.error( 'Failed to load catalog object: %s' % result['Message'] )
     return result
   
   catalogClass = result['Value']
    
   try:
     if catalogType in ['LcgFileCatalogCombined','LcgFileCatalog']:
       # The LFC special case
       infoSys = catConfig.get('LcgGfalInfosys','')
       host = catConfig.get('MasterHost','')
       catalog = catalogClass( infoSys, host )
     else:  
       if catalogURL:
         catalog = catalogClass( url = catalogURL )  
       else:  
         catalog = catalogClass()
     self.log.debug('Loaded module %sClient' % catalogType )
     return S_OK( catalog )
   except Exception, x:
     errStr = "Failed to instantiate %s()" % ( catalogType )
     gLogger.exception( errStr, lException = x )
     return S_ERROR( errStr )
Ejemplo n.º 21
0
 def __loadLogClass(self, modulePath):
     """Load class thread-safe."""
     # import ObjectLoader here to avoid a dependancy loop
     from DIRAC.Core.Utilities.ObjectLoader import ObjectLoader
     objLoader = ObjectLoader()
     # lock to avoid problem in ObjectLoader which is a singleton not
     # thread-safe
     self._lockObjectLoader.acquire()
     try:
         # load the Backend class
         return objLoader.loadObject(modulePath)
     finally:
         self._lockObjectLoader.release()
     return S_ERROR()
Ejemplo n.º 22
0
 def __loadLogClass(self, modulePath):
   """Load class thread-safe."""
   # import ObjectLoader here to avoid a dependancy loop
   from DIRAC.Core.Utilities.ObjectLoader import ObjectLoader
   objLoader = ObjectLoader()
   # lock to avoid problem in ObjectLoader which is a singleton not
   # thread-safe
   self._lockObjectLoader.acquire()
   try:
     # load the Backend class
     return objLoader.loadObject(modulePath)
   finally:
     self._lockObjectLoader.release()
   return S_ERROR()
Ejemplo n.º 23
0
    def initializeHandler(cls, serviceInfoDict):
        """Initialization of DB objects and OptimizationMind"""
        try:
            result = ObjectLoader().loadObject(
                "WorkloadManagementSystem.DB.JobDB", "JobDB")
            if not result["OK"]:
                return result
            cls.jobDB = result["Value"](parentLogger=cls.log)

            result = ObjectLoader().loadObject(
                "WorkloadManagementSystem.DB.JobLoggingDB", "JobLoggingDB")
            if not result["OK"]:
                return result
            cls.jobLoggingDB = result["Value"](parentLogger=cls.log)

            result = ObjectLoader().loadObject(
                "WorkloadManagementSystem.DB.TaskQueueDB", "TaskQueueDB")
            if not result["OK"]:
                return result
            cls.taskQueueDB = result["Value"](parentLogger=cls.log)

            result = ObjectLoader().loadObject(
                "WorkloadManagementSystem.DB.PilotAgentsDB", "PilotAgentsDB")
            if not result["OK"]:
                return result
            cls.pilotAgentsDB = result["Value"](parentLogger=cls.log)

        except RuntimeError as excp:
            return S_ERROR("Can't connect to DB: %s" % excp)

        cls.pilotsLoggingDB = None
        enablePilotsLogging = Operations().getValue(
            "/Services/JobMonitoring/usePilotsLoggingFlag", False)
        if enablePilotsLogging:
            try:
                result = ObjectLoader().loadObject(
                    "WorkloadManagementSystem.DB.PilotsLoggingDB",
                    "PilotsLoggingDB")
                if not result["OK"]:
                    return result
                cls.pilotsLoggingDB = result["Value"](parentLogger=cls.log)
            except RuntimeError as excp:
                return S_ERROR("Can't connect to DB: %s" % excp)

        cls.msgClient = MessageClient("WorkloadManagement/OptimizationMind")
        result = cls.msgClient.connect(JobManager=True)
        if not result["OK"]:
            cls.log.warn("Cannot connect to OptimizationMind!",
                         result["Message"])
        return S_OK()
Ejemplo n.º 24
0
 def initializeOptimizer( cls ):
   objLoader = ObjectLoader()
   result = objLoader.getObjects( "WorkloadManagementSystem.Splitters",
                                  reFilter = ".*Splitter",
                                  parentClass = BaseSplitter )
   if not result[ 'OK' ]:
     return result
   data = result[ 'Value' ]
   cls.__splitters = {}
   for k in data:
     spClass = data[k]
     spName = k.split(".")[-1][:-8]
     cls.__splitters[ spName ] = spClass
     cls.log.notice( "Found %s splitter" % spName )
   cls.ex_setOption( "FailedStatus", "Invalid split" )
   return S_OK()
Ejemplo n.º 25
0
class ObjectLoaderMainSuccessScenario(unittest.TestCase):

  def setUp(self):
    self.ol = ObjectLoader()

  def __check(self, result):
    if not result['OK']:
      self.fail(result['Message'])
    return result['Value']

  def test_load(self):
    self.__check(self.ol.loadObject("Core.Utilities.List", 'fromChar'))
    self.__check(self.ol.loadObject("Core.Utilities.ObjectLoader", "ObjectLoader"))
    dataFilter = self.__check(self.ol.getObjects("WorkloadManagementSystem.Service", ".*Handler"))
    dataClass = self.__check(self.ol.getObjects("WorkloadManagementSystem.Service", parentClass=RequestHandler))
    self.assertEqual(sorted(dataFilter), sorted(dataClass))
Ejemplo n.º 26
0
    def initialize(self, jobDB=None, logDB=None):
        """Initialization of the Optimizer Agent."""
        self.jobDB = JobDB() if jobDB is None else jobDB
        if not self.jobDB.isValid():
            dExit(1)

        useESForJobParametersFlag = Operations().getValue(
            "/Services/JobMonitoring/useESForJobParametersFlag", False)
        if useESForJobParametersFlag:
            try:
                result = ObjectLoader().loadObject(
                    "WorkloadManagementSystem.DB.ElasticJobParametersDB",
                    "ElasticJobParametersDB")
                if not result["OK"]:
                    return result
                self.elasticJobParametersDB = result["Value"]()
            except RuntimeError as excp:
                return S_ERROR("Can't connect to DB: %s" % excp)

        self.logDB = JobLoggingDB() if logDB is None else logDB

        optimizerName = self.am_getModuleParam("agentName")
        if optimizerName.endswith("Agent"):
            optimizerName = optimizerName[:-len("Agent")]
        self.am_setModuleParam("optimizerName", optimizerName)

        self.startingMinorStatus = self.am_getModuleParam("optimizerName")
        self.failedStatus = self.am_getOption("FailedJobStatus",
                                              JobStatus.FAILED)
        self.am_setOption("PollingTime", 30)

        return self.initializeOptimizer()
Ejemplo n.º 27
0
 def initializeOptimizer(cls):
     objLoader = ObjectLoader()
     result = objLoader.getObjects("WorkloadManagementSystem.Splitters",
                                   reFilter=".*Splitter",
                                   parentClass=BaseSplitter)
     if not result['OK']:
         return result
     data = result['Value']
     cls.__splitters = {}
     for k in data:
         spClass = data[k]
         spName = k.split(".")[-1][:-8]
         cls.__splitters[spName] = spClass
         cls.log.notice("Found %s splitter" % spName)
     cls.ex_setOption("FailedStatus", "Invalid split")
     return S_OK()
Ejemplo n.º 28
0
    def initializeHandler(cls, serviceInfo):
        try:
            result = ObjectLoader().loadObject(
                "MonitoringSystem.DB.MonitoringDB", "MonitoringDB")
            if not result["OK"]:
                return result
            cls.__db = result["Value"]()
        except RuntimeError as excp:
            return S_ERROR("Can't connect to DB: %s" % excp)

        reportSection = serviceInfo["serviceSectionPath"]
        dataPath = gConfig.getValue("%s/DataLocation" % reportSection,
                                    "data/monitoringPlots")
        gLogger.info("Data will be written into %s" % dataPath)
        mkDir(dataPath)
        try:
            testFile = "%s/moni.plot.test" % dataPath
            with open(testFile, "w") as _:
                os.unlink(testFile)
        except IOError as err:
            gLogger.fatal("Can't write to %s" % dataPath, err)
            return S_ERROR("Data location is not writable: %s" % repr(err))
        gDataCache.setGraphsLocation(dataPath)

        return S_OK()
Ejemplo n.º 29
0
    def initializeHandler(cls, serviceInfoDict):
        useMyProxy = cls.srv_getCSOption("UseMyProxy", False)
        try:
            result = ObjectLoader().loadObject('FrameworkSystem.DB.ProxyDB',
                                               'ProxyDB')
            if not result['OK']:
                gLogger.error('Failed to load ProxyDB class: %s' %
                              result['Message'])
                return result
            dbClass = result['Value']

            cls.__proxyDB = dbClass(useMyProxy=useMyProxy)

        except RuntimeError as excp:
            return S_ERROR("Can't connect to ProxyDB: %s" % excp)
        gThreadScheduler.addPeriodicTask(900,
                                         cls.__proxyDB.purgeExpiredTokens,
                                         elapsedTime=900)
        gThreadScheduler.addPeriodicTask(900,
                                         cls.__proxyDB.purgeExpiredRequests,
                                         elapsedTime=900)
        gThreadScheduler.addPeriodicTask(21600, cls.__proxyDB.purgeLogs)
        gThreadScheduler.addPeriodicTask(3600,
                                         cls.__proxyDB.purgeExpiredProxies)
        gLogger.info("MyProxy: %s\n MyProxy Server: %s" %
                     (useMyProxy, cls.__proxyDB.getMyProxyServer()))
        return S_OK()
Ejemplo n.º 30
0
 def __getCatalogClass(self, catalogType):
     result = ObjectLoader().loadObject("Resources.Catalog.%sClient" %
                                        catalogType)
     if not result["OK"]:
         gLogger.error("Failed to load catalog object",
                       "%s" % result["Message"])
     return result
Ejemplo n.º 31
0
    def initializeHandler(cls, serviceInfoDict):
        """Initialization of Pilots Logging service"""
        cls.consumersSet = set()
        try:
            result = ObjectLoader().loadObject(
                "WorkloadManagementSystem.DB.PilotsLoggingDB",
                "PilotsLoggingDB")
            if not result["OK"]:
                return result
            cls.pilotsLoggingDB = result["Value"](parentLogger=cls.log)

        except RuntimeError as excp:
            return S_ERROR("Can't connect to DB: %s" % excp)

        queue = cls.srv_getCSOption("PilotsLoggingQueue")
        # This is pretty awful hack. Somehow, for uknown reason, I cannot access CS with srv_getCSOption.
        # The only way is using full CS path, so I'm using it as a backup solution.
        if not queue:
            queue = gConfig.getValue(serviceInfoDict["serviceSectionPath"] +
                                     "/PilotsLoggingQueue")
        result = createConsumer(queue, callback=cls.consumingCallback)
        if result["OK"]:
            cls.consumersSet.add(result["Value"])
        else:
            return result
        return S_OK()
Ejemplo n.º 32
0
class ObjectLoaderMainSuccessScenario( unittest.TestCase ):

  def setUp( self ):
    self.ol = ObjectLoader()

  def __check( self, result ):
    if not result[ 'OK' ]:
      self.fail( result[ 'Message' ] )
    return result[ 'Value' ]
    

  def test_load( self ):
    self.__check( self.ol.loadObject( "Core.Utilities.List", 'fromChar' ) )
    self.__check( self.ol.loadObject( "Core.Utilities.ObjectLoader", "ObjectLoader" ) )
    dataFilter = self.__check( self.ol.getObjects( "WorkloadManagementSystem.Service", ".*Handler" ) )
    dataClass = self.__check( self.ol.getObjects( "WorkloadManagementSystem.Service", parentClass = RequestHandler )  )
    self.assertEqual( sorted( dataFilter.keys() ), sorted( dataClass.keys() ) )
Ejemplo n.º 33
0
    def __init__(
        self,
        vo,
        autoModifyUsers=True,
        autoAddUsers=True,
        autoDeleteUsers=False,
        autoLiftSuspendedStatus=False,
        syncPluginName=None,
    ):
        """VOMS2CSSynchronizer class constructor

        :param str vo: VO to be synced
        :param boolean autoModifyUsers: flag to automatically modify user data in CS
        :param autoAddUsers: flag to automatically add new users to CS
        :param autoDeleteUsers: flag to automatically delete users from CS if no more in VOMS
        :param autoLiftSuspendedStatus: flag to automatically remove Suspended status in CS
        :param syncPluginName: name of the plugin to validate or extend users' info

        :return: None
        """

        self.log = gLogger.getSubLogger(self.__class__.__name__)
        self.csapi = CSAPI()
        self.vo = vo
        self.vomsVOName = getVOOption(vo, "VOMSName", "")
        if not self.vomsVOName:
            raise Exception("VOMS name not defined for VO %s" % vo)
        self.adminMsgs = {"Errors": [], "Info": []}
        self.vomsUserDict = {}
        self.autoModifyUsers = autoModifyUsers
        self.autoAddUsers = autoAddUsers
        self.autoDeleteUsers = autoDeleteUsers
        self.autoLiftSuspendedStatus = autoLiftSuspendedStatus
        self.voChanged = False
        self.syncPlugin = None

        if syncPluginName:
            objLoader = ObjectLoader()
            _class = objLoader.loadObject(
                "ConfigurationSystem.Client.SyncPlugins.%sSyncPlugin" % syncPluginName, "%sSyncPlugin" % syncPluginName
            )

            if not _class["OK"]:
                raise Exception(_class["Message"])

            self.syncPlugin = _class["Value"]()
Ejemplo n.º 34
0
    def loadBatchSystem(self, batchSystemName):
        """Instantiate object representing the backend batch system

        :param str batchSystemName: name of the batch system
        """
        if batchSystemName is None:
            batchSystemName = self.ceParameters["BatchSystem"]

        objectLoader = ObjectLoader()
        result = objectLoader.loadObject("Resources.Computing.BatchSystems.%s" % batchSystemName, batchSystemName)
        if not result["OK"]:
            self.log.error("Failed to load batch object: %s" % result["Message"])
            return result
        batchClass = result["Value"]
        batchModuleFile = result["ModuleFile"]
        self.batchSystem = batchClass()
        self.log.info("Batch system class from module: ", batchModuleFile)
        return S_OK()
Ejemplo n.º 35
0
def loadResourceStatusComponent(moduleName, className):
    """
  Create an object of a given database component.

  :param moduleName: module name to be loaded
  :param className: class name
  :return: object instance wrapped in a standard Dirac return object.
  """

    objectLoader = ObjectLoader()
    componentModule = 'ResourceStatusSystem.DB.%s' % (moduleName, )
    result = objectLoader.loadObject(componentModule, className)
    if not result['OK']:
        gLogger.error('Failed to load RSS component',
                      '%s: %s' % (moduleName, result['Message']))
        return result
    componentClass = result['Value']
    component = componentClass()
    return S_OK(component)
Ejemplo n.º 36
0
  def enableCS(self):
    """
    Force the connection the Configuration Server

    (And incidentaly reinitialize the ObjectLoader and logger)
    """
    res = gRefresher.enable()

    # This is quite ugly but necessary for the logging
    # We force the reinitialization of the ObjectLoader
    # so that it also takes into account the extensions
    # (since the first time it is loaded by the logger BEFORE the full CS init)
    # And then we regenerate all the backend
    if res['OK']:
      from DIRAC.Core.Utilities.ObjectLoader import ObjectLoader
      objLoader = ObjectLoader()
      objLoader.reloadRootModules()
      self.__initLogger(self.componentName, self.loggingSection, forceInit=True)
    return res
Ejemplo n.º 37
0
  def setConfig(self, databaseConfig):

    self.directories = {}
    # In memory storage of the various parameters
    self.users = {}
    self.uids = {}
    self.groups = {}
    self.gids = {}
    self.seNames = {}
    self.seids = {}

    # Obtain some general configuration of the database
    self.uniqueGUID = databaseConfig['UniqueGUID']
    self.globalReadAccess = databaseConfig['GlobalReadAccess']
    self.lfnPfnConvention = databaseConfig['LFNPFNConvention']
    if self.lfnPfnConvention == "None":
      self.lfnPfnConvention = False
    self.resolvePfn = databaseConfig['ResolvePFN']
    self.umask = databaseConfig['DefaultUmask']
    self.validFileStatus = databaseConfig['ValidFileStatus']
    self.validReplicaStatus = databaseConfig['ValidReplicaStatus']
    self.visibleFileStatus = databaseConfig['VisibleFileStatus']
    self.visibleReplicaStatus = databaseConfig['VisibleReplicaStatus']

    # Obtain the plugins to be used for DB interaction
    self.objectLoader = ObjectLoader()

    # Load the configured components
    for compAttribute, componentType in [("ugManager", "UserGroupManager"),
                                         ("seManager", "SEManager"),
                                         ("securityManager", "SecurityManager"),
                                         ("dtree", "DirectoryManager"),
                                         ("fileManager", "FileManager"),
                                         ("datasetManager", "DatasetManager"),
                                         ("dmeta", "DirectoryMetadata"),
                                         ("fmeta", "FileMetadata")]:

      result = self.__loadCatalogComponent(componentType, databaseConfig[componentType])
      if not result['OK']:
        return result
      self.__setattr__(compAttribute, result['Value'])

    return S_OK()
Ejemplo n.º 38
0
  def __generateStorageObject( self, storageName, pluginName, parameters ):

    storageType = pluginName
    if self.proxy:
      storageType = 'Proxy'

    objectLoader = ObjectLoader()
    result = objectLoader.loadObject( 'Resources.Storage.%sStorage' % storageType, storageType + 'Storage' )
    if not result['OK']:
      gLogger.error( 'Failed to load storage object: %s' % result['Message'] )
      return result

    storageClass = result['Value']
    try:
      storage = storageClass( storageName, parameters )
    except Exception, x:
      errStr = "StorageFactory._generateStorageObject: Failed to instantiate %s: %s" % ( storageName, x )
      gLogger.exception( errStr )
      return S_ERROR( errStr )
Ejemplo n.º 39
0
 def __loadCatalogComponent(self, componentType, componentName):
     """Create an object of a given catalog component"""
     componentModule = "DataManagementSystem.DB.FileCatalogComponents.%s.%s" % (componentType, componentName)
     result = ObjectLoader().loadObject(componentModule)
     if not result["OK"]:
         gLogger.error("Failed to load catalog component", "%s: %s" % (componentName, result["Message"]))
         return result
     componentClass = result["Value"]
     component = componentClass(self)
     return S_OK(component)
Ejemplo n.º 40
0
    def __init__( self, tokens ):
      """
          :param tokens: [ pluginName, =, conditions ]
                 the pluginName is automatically prepended with 'Plugin'


      """

      self.pluginName = "%sPlugin" % tokens[0].strip( ' ' )
      self.conditions = tokens[2].strip( ' ' )

      # Load the plugin, and give it the condition
      objLoader = ObjectLoader()
      _class = objLoader.loadObject( 'Resources.Catalog.ConditionPlugins.%s' % self.pluginName, self.pluginName )

      if not _class['OK']:
        raise Exception( _class['Message'] )


      self._pluginInst = _class['Value']( self.conditions )
Ejemplo n.º 41
0
  def bootstrap( self ):
    gLogger.always( "\n  === Bootstrapping REST Server ===  \n" )
    ol = ObjectLoader( [ 'DIRAC', 'RESTDIRAC' ] )
    result = ol.getObjects( "RESTSystem.API", parentClass = RESTHandler, recurse = True )
    if not result[ 'OK' ]:
      return result

    self.__handlers = result[ 'Value' ]
    if not self.__handlers:
      return S_ERROR( "No handlers found" )

    self.__routes = [ ( self.__handlers[ k ].getRoute(), self.__handlers[k] ) for k in self.__handlers if self.__handlers[ k ].getRoute()  ]
    gLogger.info( "Routes found:" )
    for t in sorted( self.__routes ):
      gLogger.info( " - %s : %s" % ( t[0], t[1].__name__ ) )

    balancer = RESTConf.balancer()
    kw = dict( debug = RESTConf.debug(), log_function = self._logRequest )
    if balancer and RESTConf.numProcesses not in ( 0, 1 ):
      process.fork_processes( RESTConf.numProcesses(), max_restarts = 0 )
      kw[ 'debug' ] = False
    if kw[ 'debug' ]:
      gLogger.always( "Starting in debug mode" )
    self.__app = web.Application( self.__routes, **kw )
    port = RESTConf.port()
    if balancer:
      gLogger.notice( "Configuring REST HTTP service for balancer %s on port %s" % ( balancer, port ) )
      self.__sslops = False
    else:
      gLogger.notice( "Configuring REST HTTPS service on port %s" % port )
      self.__sslops = dict( certfile = RESTConf.cert(),
                            keyfile = RESTConf.key(),
                            cert_reqs = ssl.CERT_OPTIONAL,
                            ca_certs = RESTConf.generateCAFile() )
    self.__httpSrv = httpserver.HTTPServer( self.__app, ssl_options = self.__sslops )
    self.__httpSrv.listen( port )
    return S_OK()
Ejemplo n.º 42
0
class FileCatalogDB(DB):

  __tables = {}
  __tables["FC_Statuses"] = { "Fields":
                              {
                                "StatusID": "INT AUTO_INCREMENT",
                                "Status": "VARCHAR(32)"
                              },
                              "UniqueIndexes": { "Status": ["Status"] },
                              "PrimaryKey":"StatusID" 
                            }

  def __init__( self, databaseLocation='DataManagement/FileCatalogDB', maxQueueSize=10 ):
    """ Standard Constructor
    """
    
    # The database location can be specified in System/Database form or in just the Database name
    # in the DataManagement system 
    db = databaseLocation
    if db.find('/') == -1:
      db = 'DataManagement/' + db
    DB.__init__(self,'FileCatalogDB',db,maxQueueSize)
    
    result = self._createTables( self.__tables )
    if not result['OK']:
      gLogger.error( "Failed to create tables", str( self.__tables.keys() ) )
    elif result['Value']:
      gLogger.info( "Tables created: %s" % ','.join( result['Value'] ) )    
    
    self.ugManager = None
    self.seManager = None
    self.securityManager = None
    self.dtree = None
    self.fileManager = None
    self.dmeta = None
    self.fmeta = None
    self.statusDict = {}

  def setConfig(self,databaseConfig):

    self.directories = {}
    # In memory storage of the various parameters
    self.users = {}
    self.uids = {}
    self.groups = {}
    self.gids = {}
    self.seNames = {}
    self.seids = {}
    self.seDefinitions = {}

    # Obtain some general configuration of the database
    self.uniqueGUID = databaseConfig['UniqueGUID']
    self.globalReadAccess = databaseConfig['GlobalReadAccess']
    self.lfnPfnConvention = databaseConfig['LFNPFNConvention']
    if self.lfnPfnConvention == "None":
      self.lfnPfnConvention = False
    self.resolvePfn = databaseConfig['ResolvePFN']
    self.umask = databaseConfig['DefaultUmask']
    self.validFileStatus = databaseConfig['ValidFileStatus']
    self.validReplicaStatus = databaseConfig['ValidReplicaStatus']
    self.visibleFileStatus = databaseConfig['VisibleFileStatus']
    self.visibleReplicaStatus = databaseConfig['VisibleReplicaStatus']

    # Obtain the plugins to be used for DB interaction
    self. objectLoader = ObjectLoader()
    
    result = self.__loadCatalogComponent( databaseConfig['UserGroupManager'] )
    if not result['OK']:
      return result
    self.ugManager = result['Value']
    
    result = self.__loadCatalogComponent( databaseConfig['SEManager'] )
    if not result['OK']:
      return result
    self.seManager = result['Value']
    
    result = self.__loadCatalogComponent( databaseConfig['SecurityManager'] )
    if not result['OK']:
      return result
    self.securityManager = result['Value']
    
    result = self.__loadCatalogComponent( databaseConfig['DirectoryManager'] )
    if not result['OK']:
      return result
    self.dtree = result['Value']
    
    result = self.__loadCatalogComponent( databaseConfig['FileManager'] )
    if not result['OK']:
      return result
    self.fileManager = result['Value']
    
    result = self.__loadCatalogComponent( databaseConfig['DatasetManager'] )
    if not result['OK']:
      return result
    self.datasetManager = result['Value']
    
    result = self.__loadCatalogComponent( databaseConfig['DirectoryMetadata'] )
    if not result['OK']:
      return result
    self.dmeta = result['Value']
    
    result = self.__loadCatalogComponent( databaseConfig['FileMetadata'] )
    if not result['OK']:
      return result
    self.fmeta = result['Value']

    return S_OK()
  
  def __loadCatalogComponent( self, componentName ):
    """ Create an object of a given catalog component
    """
    moduleName = componentName
    # some modules contain several implementation classes
    for m in ['SEManager','UserAndGroupManager','SecurityManager']:
      if m in componentName:
        moduleName = m
    componentPath = 'DataManagementSystem.DB.FileCatalogComponents'
    result = self.objectLoader.loadObject( '%s.%s' % ( componentPath, moduleName ), 
                                      componentName )
    if not result['OK']:
      gLogger.error( 'Failed to load catalog component', result['Message'] )
      return result
    componentClass = result['Value']
    component = componentClass( self )
    return S_OK( component )
    
  def setUmask(self,umask):
    self.umask = umask

  ########################################################################
  #
  #  General purpose utility methods

  def getStatusInt( self, status, connection = False ):
    
    """ Get integer ID of the given status string
    """
    connection = self._getConnection( connection )
    req = "SELECT StatusID FROM FC_Statuses WHERE Status = '%s';" % status
    res = self.db._query( req, connection )
    if not res['OK']:
      return res
    if res['Value']:
      return S_OK( res['Value'][0][0] )
    req = "INSERT INTO FC_Statuses (Status) VALUES ('%s');" % status
    res = self.db._update( req, connection )
    if not res['OK']:
      return res
    return S_OK( res['lastRowId'] )

  def getIntStatus(self,statusID,connection=False):
    """ Get status string for a given integer status ID
    """
    if statusID in self.statusDict:
      return S_OK(self.statusDict[statusID])
    connection = self._getConnection(connection)
    req = "SELECT StatusID,Status FROM FC_Statuses" 
    res = self.db._query(req,connection)
    if not res['OK']:
      return res
    if res['Value']:
      for row in res['Value']:
        self.statusDict[int(row[0])] = row[1]
    if statusID in self.statusDict:
      return S_OK(self.statusDict[statusID])
    return S_OK('Unknown')

  ########################################################################
  #
  #  SE based write methods
  #
  
  def addSE(self,seName,credDict):
    res = self._checkAdminPermission(credDict)
    if not res['OK']:
      return res
    if not res['Value']:
      return S_ERROR("Permission denied")
    return self.seManager.addSE(seName)
    
  def deleteSE(self,seName,credDict):
    res = self._checkAdminPermission(credDict)
    if not res['OK']:
      return res
    if not res['Value']:
      return S_ERROR("Permission denied")
    return self.seManager.deleteSE(seName)

  ########################################################################
  #
  #  User/groups based write methods
  #

  def addUser(self,userName,credDict):
    res = self._checkAdminPermission(credDict)
    if not res['OK']:
      return res
    if not res['Value']:
      return S_ERROR("Permission denied")
    return self.ugManager.addUser(userName)

  def deleteUser(self,userName,credDict):
    res = self._checkAdminPermission(credDict)
    if not res['OK']:
      return res
    if not res['Value']:
      return S_ERROR("Permission denied")
    return self.ugManager.deleteUser(userName)

  def addGroup(self,groupName,credDict):
    res = self._checkAdminPermission(credDict)
    if not res['OK']:
      return res
    if not res['Value']:
      return S_ERROR("Permission denied")
    return self.ugManager.addGroup(groupName)
  
  def deleteGroup(self,groupName,credDict):
    res = self._checkAdminPermission(credDict)
    if not res['OK']:
      return res
    if not res['Value']:
      return S_ERROR("Permission denied")
    return self.ugManager.deleteGroup(groupName)
  
  ########################################################################
  #
  #  User/groups based read methods
  #

  def getUsers(self,credDict):
    res = self._checkAdminPermission(credDict)
    if not res['OK']:
      return res
    if not res['Value']:
      return S_ERROR("Permission denied")
    return self.ugManager.getUsers()

  def getGroups(self,credDict):
    res = self._checkAdminPermission(credDict)
    if not res['OK']:
      return res
    if not res['Value']:
      return S_ERROR("Permission denied")
    return self.ugManager.getGroups()

  ########################################################################
  #
  #  Path based read methods
  #

  def exists(self, lfns, credDict):
    res = self._checkPathPermissions('Read', lfns, credDict)
    if not res['OK']:
      return res
    failed = res['Value']['Failed']
    res = self.fileManager.exists(res['Value']['Successful'])
    if not res['OK']:
      return res
    failed.update(res['Value']['Failed'])
    successful = res['Value']['Successful']
    notExist = []
    for lfn in res['Value']['Successful'].keys():
      if not successful[lfn]:
        notExist.append(lfn)
        successful.pop(lfn)
    if notExist:
      res = self.dtree.exists(notExist)
      if not res['OK']:
        return res    
      failed.update(res['Value']['Failed'])
      successful.update(res['Value']['Successful'])
    return S_OK( {'Successful':successful,'Failed':failed} )
  
  def getPathPermissions(self, lfns, credDict):
    """ Get permissions for the given user/group to manipulate the given lfns 
    """
    res = checkArgumentFormat(lfns)
    if not res['OK']:
      return res
    lfns = res['Value']

    return self.securityManager.getPathPermissions( lfns.keys(), credDict )
  
  ########################################################################
  #
  #  Path based read methods
  #

  def changePathOwner(self, lfns, credDict, recursive=False):
    """ Change the owner of the given list of paths
    """
    res = self._checkPathPermissions('Write', lfns, credDict)
    if not res['OK']:
      return res
    failed = res['Value']['Failed']
    res = self.fileManager.changePathOwner(res['Value']['Successful'],credDict, recursive)
    if not res['OK']:
      return res
    failed.update(res['Value']['Failed'])
    successful = res['Value']['Successful']   
    return S_OK({'Successful':successful,'Failed':failed}) 
  
  def changePathGroup(self, lfns, credDict, recursive=False):
    """ Change the group of the given list of paths
    """
    res = self._checkPathPermissions('Write', lfns, credDict)
    if not res['OK']:
      return res
    failed = res['Value']['Failed']
    res = self.fileManager.changePathGroup(res['Value']['Successful'],credDict, recursive)
    if not res['OK']:
      return res
    failed.update(res['Value']['Failed'])
    successful = res['Value']['Successful']   
    return S_OK({'Successful':successful,'Failed':failed}) 

  def changePathMode(self, lfns, credDict, recursive=False):
    """ Change the mode of the given list of paths
    """
    res = self._checkPathPermissions('Write', lfns, credDict)    
    if not res['OK']:
      return res
    failed = res['Value']['Failed']
    res = self.fileManager.changePathMode(res['Value']['Successful'],credDict, recursive)    
    if not res['OK']:
      return res
    failed.update(res['Value']['Failed'])
    successful = res['Value']['Successful']   
    return S_OK({'Successful':successful,'Failed':failed}) 

  ########################################################################
  #
  #  File based write methods
  #

  def addFile(self, lfns, credDict):
    res = self._checkPathPermissions('Write', lfns, credDict)
    if not res['OK']:
      return res
    failed = res['Value']['Failed']
    res = self.fileManager.addFile(res['Value']['Successful'],credDict)
    if not res['OK']:
      return res
    failed.update(res['Value']['Failed'])
    successful = res['Value']['Successful']
    return S_OK( {'Successful':successful,'Failed':failed} )
  
  def setFileStatus(self, lfns, credDict):
    res = self._checkPathPermissions('Write', lfns, credDict)
    if not res['OK']:
      return res
    failed = res['Value']['Failed']
    res = self.fileManager.setFileStatus( res['Value']['Successful'], credDict )
    if not res['OK']:
      return res
    failed.update(res['Value']['Failed'])
    successful = res['Value']['Successful']
    return S_OK( {'Successful':successful,'Failed':failed} )
  
  def removeFile(self, lfns, credDict):
    res = self._checkPathPermissions('Write', lfns, credDict)
    if not res['OK']:
      return res
    failed = res['Value']['Failed']
    res = self.fileManager.removeFile(res['Value']['Successful'])
    if not res['OK']:
      return res
    failed.update(res['Value']['Failed'])
    successful = res['Value']['Successful']
    return S_OK( {'Successful':successful,'Failed':failed} )
  
  def addReplica(self, lfns, credDict):
    res = self._checkPathPermissions('Write', lfns, credDict)
    if not res['OK']:
      return res
    failed = res['Value']['Failed']
    res = self.fileManager.addReplica(res['Value']['Successful'])
    if not res['OK']:
      return res
    failed.update(res['Value']['Failed'])
    successful = res['Value']['Successful']
    return S_OK( {'Successful':successful,'Failed':failed} )
  
  def removeReplica(self, lfns, credDict):
    res = self._checkPathPermissions('Write', lfns, credDict)
    if not res['OK']:
      return res
    failed = res['Value']['Failed']
    res = self.fileManager.removeReplica(res['Value']['Successful'])
    if not res['OK']:
      return res
    failed.update(res['Value']['Failed'])
    successful = res['Value']['Successful']
    return S_OK( {'Successful':successful,'Failed':failed} )

  def setReplicaStatus(self, lfns, credDict):
    res = self._checkPathPermissions('Write', lfns, credDict)
    if not res['OK']:
      return res
    failed = res['Value']['Failed']
    res = self.fileManager.setReplicaStatus(res['Value']['Successful'])
    if not res['OK']:
      return res
    failed.update(res['Value']['Failed'])
    successful = res['Value']['Successful']
    return S_OK( {'Successful':successful,'Failed':failed} )

  def setReplicaHost(self, lfns, credDict):
    res = self._checkPathPermissions('Write', lfns, credDict)
    if not res['OK']:
      return res
    failed = res['Value']['Failed']
    res = self.fileManager.setReplicaHost(res['Value']['Successful'])
    if not res['OK']:
      return res
    failed.update(res['Value']['Failed'])
    successful = res['Value']['Successful']
    return S_OK( {'Successful':successful,'Failed':failed} )

  def setFileOwner(self,lfns,credDict):
    res = self._checkPathPermissions('Write', lfns, credDict)
    if not res['OK']:
      return res
    failed = res['Value']['Failed']
    res = self.fileManager.setFileOwner(res['Value']['Successful'])
    if not res['OK']:
      return res
    failed.update(res['Value']['Failed'])
    successful = res['Value']['Successful']
    return S_OK( {'Successful':successful,'Failed':failed} )
  
  def setFileGroup(self,lfns,credDict):  
    res = self._checkPathPermissions('Write', lfns, credDict)
    if not res['OK']:
      return res
    failed = res['Value']['Failed']
    res = self.fileManager.setFileGroup(res['Value']['Successful'])
    if not res['OK']:
      return res
    failed.update(res['Value']['Failed'])
    successful = res['Value']['Successful']
    return S_OK( {'Successful':successful,'Failed':failed} )
    
  def setFileMode(self,lfns,credDict):  
    res = self._checkPathPermissions('Write', lfns, credDict)
    if not res['OK']:
      return res
    failed = res['Value']['Failed']
    res = self.fileManager.setFileMode(res['Value']['Successful'])
    if not res['OK']:
      return res
    failed.update(res['Value']['Failed'])
    successful = res['Value']['Successful']
    return S_OK( {'Successful':successful,'Failed':failed} )
    
  def addFileAncestors(self,lfns,credDict):
    """ Add ancestor information for the given LFNs
    """        
    res = self._checkPathPermissions('Write', lfns, credDict)
    if not res['OK']:
      return res
    failed = res['Value']['Failed']
    res = self.fileManager.addFileAncestors(res['Value']['Successful'])
    if not res['OK']:
      return res
    failed.update(res['Value']['Failed'])
    successful = res['Value']['Successful']
    return S_OK( {'Successful':successful,'Failed':failed} )
  
  ########################################################################
  #
  #  File based read methods
  #

  def isFile(self, lfns, credDict):
    res = self._checkPathPermissions('Read', lfns, credDict)
    if not res['OK']:
      return res
    failed = res['Value']['Failed']
    res = self.fileManager.isFile(res['Value']['Successful'])
    if not res['OK']:
      return res
    failed.update(res['Value']['Failed'])
    successful = res['Value']['Successful']
    return S_OK( {'Successful':successful,'Failed':failed} )

  def getFileSize(self, lfns, credDict):
    res = self._checkPathPermissions('Read', lfns, credDict)
    if not res['OK']:
      return res
    failed = res['Value']['Failed']
    res = self.fileManager.getFileSize(res['Value']['Successful'])
    if not res['OK']:
      return res
    failed.update(res['Value']['Failed'])
    successful = res['Value']['Successful']
    return S_OK( {'Successful':successful,'Failed':failed} )
  
  def getFileMetadata(self, lfns, credDict):
    res = self._checkPathPermissions('Read', lfns, credDict)
    if not res['OK']:
      return res
    failed = res['Value']['Failed']
    res = self.fileManager.getFileMetadata(res['Value']['Successful'])
    if not res['OK']:
      return res
    failed.update(res['Value']['Failed'])
    successful = res['Value']['Successful']
    return S_OK( {'Successful':successful,'Failed':failed} )

  def getReplicas(self, lfns, allStatus, credDict):
    res = self._checkPathPermissions('Read', lfns, credDict)
    if not res['OK']:
      return res
    failed = res['Value']['Failed']
    res = self.fileManager.getReplicas(res['Value']['Successful'],allStatus=allStatus)
    if not res['OK']:
      return res
    failed.update(res['Value']['Failed'])
    successful = res['Value']['Successful']
    return S_OK( {'Successful':successful, 'Failed':failed, 'SEPrefixes': res['Value'].get( 'SEPrefixes', {} ) } )

  def getReplicaStatus(self, lfns, credDict):
    res = self._checkPathPermissions('Read', lfns, credDict)
    if not res['OK']:
      return res
    failed = res['Value']['Failed']
    res = self.fileManager.getReplicaStatus(res['Value']['Successful'])
    if not res['OK']:
      return res
    failed.update(res['Value']['Failed'])
    successful = res['Value']['Successful']
    return S_OK( {'Successful':successful,'Failed':failed} )  
    
  def getFileAncestors(self, lfns, depths, credDict):
    res = self._checkPathPermissions('Read', lfns, credDict)
    if not res['OK']:
      return res
    failed = res['Value']['Failed']
    res = self.fileManager.getFileAncestors(res['Value']['Successful'],depths)
    if not res['OK']:
      return res
    failed.update(res['Value']['Failed'])
    successful = res['Value']['Successful']
    return S_OK( {'Successful':successful,'Failed':failed} )        
    
  def getFileDescendents(self, lfns, depths, credDict):
    res = self._checkPathPermissions('Read', lfns, credDict)
    if not res['OK']:
      return res
    failed = res['Value']['Failed']
    res = self.fileManager.getFileDescendents(res['Value']['Successful'],depths)
    if not res['OK']:
      return res
    failed.update(res['Value']['Failed'])
    successful = res['Value']['Successful']
    return S_OK( {'Successful':successful,'Failed':failed} )     
  
  def getFileDetails( self, lfnList, credDict ):
    """ Get all the metadata for the given files
    """  
    connection = False
    result = self.fileManager._findFiles( lfnList, connection=connection )
    if not result['OK']:
      return result
    resultDict = {}
    fileIDDict = {}
    lfnDict = result['Value']['Successful']
    for lfn in lfnDict:
      fileIDDict[lfnDict[lfn]['FileID']] = lfn
      
    result = self.fileManager._getFileMetadataByID( fileIDDict.keys(), connection=connection )
    if not result['OK']:
      return result
    for fileID in result['Value']:
      resultDict[ fileIDDict[fileID] ] = result['Value'][fileID]
      
    result = self.fmeta._getFileUserMetadataByID( fileIDDict.keys(), credDict, connection=connection )
    if not result['OK']:
      return result
    for fileID in fileIDDict:
      resultDict[ fileIDDict[fileID] ].setdefault( 'Metadata', {} )
      if fileID in result['Value']:
        resultDict[ fileIDDict[fileID] ]['Metadata'] = result['Value'][fileID]    
      
    return S_OK(resultDict) 

  ########################################################################
  #
  #  Directory based Write methods
  #

  def createDirectory(self,lfns,credDict):
    res = self._checkPathPermissions('Write', lfns, credDict)
    if not res['OK']:
      return res
    failed = res['Value']['Failed']
    res = self.dtree.createDirectory(res['Value']['Successful'],credDict)
    if not res['OK']:
      return res
    failed.update(res['Value']['Failed'])
    successful = res['Value']['Successful']
    return S_OK( {'Successful':successful,'Failed':failed} )

  def removeDirectory(self,lfns,credDict):
    res = self._checkPathPermissions('Write', lfns, credDict)
    if not res['OK']:
      return res
    failed = res['Value']['Failed']
    successful = res['Value']['Successful']
    if successful:
      res = self.dtree.removeDirectory(res['Value']['Successful'],credDict)
      if not res['OK']:
        return res
      failed.update(res['Value']['Failed'])
      successful = res['Value']['Successful']
      if not successful:
        return S_OK( {'Successful':successful,'Failed':failed} )
    else:
      return S_OK( {'Successful':successful,'Failed':failed} )
    
    # Remove the directory metadata now
    dirIdList = [ successful[p]['DirID'] for p in successful ]
    result = self.dmeta.removeMetadataForDirectory( dirIdList,credDict )
    if not result['OK']:
      return result
    failed.update(result['Value']['Failed'])
    successful = result['Value']['Successful']
    return S_OK( {'Successful':successful,'Failed':failed} )

  ########################################################################
  #
  #  Directory based read methods
  #

  def listDirectory(self,lfns,credDict,verbose=False):
    res = self._checkPathPermissions('Read', lfns, credDict)
    if not res['OK']:
      return res
    failed = res['Value']['Failed']
    res = self.dtree.listDirectory(res['Value']['Successful'],verbose=verbose)
    if not res['OK']:
      return res
    failed.update(res['Value']['Failed'])
    successful = res['Value']['Successful']
    return S_OK( {'Successful':successful,'Failed':failed} )
  
  def isDirectory(self,lfns,credDict):
    res = self._checkPathPermissions('Read', lfns, credDict)
    if not res['OK']:
      return res
    failed = res['Value']['Failed']
    res = self.dtree.isDirectory(res['Value']['Successful'])
    if not res['OK']:
      return res
    failed.update(res['Value']['Failed'])
    successful = res['Value']['Successful']
    return S_OK( {'Successful':successful,'Failed':failed} )

  def getDirectoryReplicas(self,lfns,allStatus,credDict):
    res = self._checkPathPermissions('Read', lfns, credDict)
    if not res['OK']:
      return res
    failed = res['Value']['Failed']
    res = self.dtree.getDirectoryReplicas(res['Value']['Successful'],allStatus)
    if not res['OK']:
      return res
    failed.update(res['Value']['Failed'])
    successful = res['Value']['Successful']
    return S_OK( { 'Successful':successful, 'Failed':failed, 'SEPrefixes': res['Value'].get( 'SEPrefixes', {} )} )

  def getDirectorySize(self,lfns,longOutput,fromFiles,credDict):
    res = self._checkPathPermissions('Read', lfns, credDict)
    if not res['OK']:
      return res
    failed = res['Value']['Failed']
    res = self.dtree.getDirectorySize(res['Value']['Successful'],longOutput,fromFiles)
    if not res['OK']:
      return res
    failed.update(res['Value']['Failed'])
    successful = res['Value']['Successful']
    queryTime = res['Value'].get('QueryTime',-1.)
    return S_OK( {'Successful':successful,'Failed':failed,'QueryTime':queryTime} )
  
  def rebuildDirectoryUsage(self):
    """ Rebuild DirectoryUsage table from scratch
    """
    
    result = self.dtree._rebuildDirectoryUsage()
    return result

  def repairCatalog( self, directoryFlag=True, credDict={} ):
    """ Repair catalog inconsistencies
    """
    result = S_OK()
    if directoryFlag:
      result = self.dtree.recoverOrphanDirectories( credDict )
      
    return result 
    
  #######################################################################
  #
  #  Catalog metadata methods
  #  
  
  def setMetadata(self, path, metadataDict, credDict):
    """ Add metadata to the given path
    """
    res = self._checkPathPermissions('Write', path, credDict)   
    if not res['OK']:
      return res
    if not res['Value']['Successful']:
      return S_ERROR('Permission denied')
    if not res['Value']['Successful'][path]:
      return S_ERROR('Permission denied') 
      
    result = self.dtree.isDirectory({path:True})
    if not result['OK']:
      return result
    if not result['Value']['Successful']:
      return S_ERROR('Failed to determine the path type')
    if result['Value']['Successful'][path]:
      # This is a directory
      return self.dmeta.setMetadata(path,metadataDict,credDict)
    else:
      # This is a file      
      return self.fmeta.setMetadata(path,metadataDict,credDict)      
    
  def setMetadataBulk( self, pathMetadataDict, credDict ):
    """  Add metadata for the given paths
    """  
    successful = {}
    failed = {}
    for path, metadataDict in pathMetadataDict.items():
      result = self.setMetadata( path, metadataDict, credDict )
      if result['OK']:
        successful[path] = True
      else:
        failed[path] = result['Message']
        
    return S_OK( { 'Successful': successful, 'Failed': failed } )      
    
  def removeMetadata(self, path, metadata, credDict):
    """ Add metadata to the given path
    """
    res = self._checkPathPermissions('Write', path, credDict)   
    if not res['OK']:
      return res
    if not res['Value']['Successful']:
      return S_ERROR('Permission denied')
    if not res['Value']['Successful'][path]:
      return S_ERROR('Permission denied') 
      
    result = self.dtree.isDirectory({path:True})
    if not result['OK']:
      return result
    if not result['Value']['Successful']:
      return S_ERROR('Failed to determine the path type')
    if result['Value']['Successful'][path]:
      # This is a directory
      return self.dmeta.removeMetadata(path,metadata,credDict)
    else:
      # This is a file      
      return self.fmeta.removeMetadata(path,metadata,credDict)                                  
    
  #######################################################################
  #
  #  Catalog admin methods
  #

  def getCatalogCounters(self,credDict):
    counterDict = {}
    res = self._checkAdminPermission(credDict)
    if not res['OK']:
      return res
    if not res['Value']:
      return S_ERROR("Permission denied")
    #res = self.dtree.getDirectoryCounters()
    #if not res['OK']:
    #  return res
    #counterDict.update(res['Value'])
    res = self.fileManager.getFileCounters()
    if not res['OK']:
      return res
    counterDict.update(res['Value'])
    res = self.fileManager.getReplicaCounters() 
    if not res['OK']:
      return res
    counterDict.update(res['Value'])
    res = self.dtree.getDirectoryCounters() 
    if not res['OK']:
      return res
    counterDict.update(res['Value'])
    return S_OK(counterDict)

  ########################################################################
  #
  #  Security based methods
  #

  def _checkAdminPermission(self,credDict):
    return self.securityManager.hasAdminAccess(credDict)

  def _checkPathPermissions(self,operation,lfns,credDict):
    res = checkArgumentFormat(lfns)
    if not res['OK']:
      return res
    lfns = res['Value']
    res = self.securityManager.hasAccess(operation,lfns.keys(),credDict)   
    if not res['OK']:
      return res
    # Do not consider those paths for which we failed to determine access
    failed = res['Value']['Failed']
    for lfn in failed.keys():
      lfns.pop(lfn)
    # Do not consider those paths for which access is denied
    successful = {}
    for lfn,access in res['Value']['Successful'].items():
      if not access:
        failed[lfn] = 'Permission denied'
      else:  
        successful[lfn] = lfns[lfn]
    return S_OK( {'Successful':successful,'Failed':failed} )
Ejemplo n.º 43
0
  def setConfig(self,databaseConfig):

    self.directories = {}
    # In memory storage of the various parameters
    self.users = {}
    self.uids = {}
    self.groups = {}
    self.gids = {}
    self.seNames = {}
    self.seids = {}
    self.seDefinitions = {}

    # Obtain some general configuration of the database
    self.uniqueGUID = databaseConfig['UniqueGUID']
    self.globalReadAccess = databaseConfig['GlobalReadAccess']
    self.lfnPfnConvention = databaseConfig['LFNPFNConvention']
    if self.lfnPfnConvention == "None":
      self.lfnPfnConvention = False
    self.resolvePfn = databaseConfig['ResolvePFN']
    self.umask = databaseConfig['DefaultUmask']
    self.validFileStatus = databaseConfig['ValidFileStatus']
    self.validReplicaStatus = databaseConfig['ValidReplicaStatus']
    self.visibleFileStatus = databaseConfig['VisibleFileStatus']
    self.visibleReplicaStatus = databaseConfig['VisibleReplicaStatus']

    # Obtain the plugins to be used for DB interaction
    self. objectLoader = ObjectLoader()
    
    result = self.__loadCatalogComponent( databaseConfig['UserGroupManager'] )
    if not result['OK']:
      return result
    self.ugManager = result['Value']
    
    result = self.__loadCatalogComponent( databaseConfig['SEManager'] )
    if not result['OK']:
      return result
    self.seManager = result['Value']
    
    result = self.__loadCatalogComponent( databaseConfig['SecurityManager'] )
    if not result['OK']:
      return result
    self.securityManager = result['Value']
    
    result = self.__loadCatalogComponent( databaseConfig['DirectoryManager'] )
    if not result['OK']:
      return result
    self.dtree = result['Value']
    
    result = self.__loadCatalogComponent( databaseConfig['FileManager'] )
    if not result['OK']:
      return result
    self.fileManager = result['Value']
    
    result = self.__loadCatalogComponent( databaseConfig['DatasetManager'] )
    if not result['OK']:
      return result
    self.datasetManager = result['Value']
    
    result = self.__loadCatalogComponent( databaseConfig['DirectoryMetadata'] )
    if not result['OK']:
      return result
    self.dmeta = result['Value']
    
    result = self.__loadCatalogComponent( databaseConfig['FileMetadata'] )
    if not result['OK']:
      return result
    self.fmeta = result['Value']

    return S_OK()
Ejemplo n.º 44
0
 def setUp( self ):
   self.ol = ObjectLoader()