Beispiel #1
0
    def _ex_initialize(cls, exeName, loadName):
        cls.__properties = {
            'fullName': exeName,
            'loadName': loadName,
            'section': PathFinder.getExecutorSection(exeName),
            'loadSection': PathFinder.getExecutorSection(loadName),
            'messagesProcessed': 0,
            'reconnects': 0,
            'setup': gConfig.getValue("/DIRAC/Setup", "Unknown")
        }
        cls.__basePath = gConfig.getValue('/LocalSite/InstancePath', rootPath)
        cls.__defaults = {}
        cls.__defaults['MonitoringEnabled'] = True
        cls.__defaults['Enabled'] = True
        cls.__defaults['ControlDirectory'] = os.path.join(
            cls.__basePath, 'control', *exeName.split("/"))
        cls.__defaults['WorkDirectory'] = os.path.join(cls.__basePath, 'work',
                                                       *exeName.split("/"))
        cls.__defaults['ReconnectRetries'] = 10
        cls.__defaults['ReconnectSleep'] = 5
        cls.__properties['shifterProxy'] = ''
        cls.__properties['shifterProxyLocation'] = os.path.join(
            cls.__defaults['WorkDirectory'], '.shifterCred')
        cls.__mindName = False
        cls.__mindExtraArgs = False
        cls.__freezeTime = 0
        cls.__fastTrackEnabled = True
        cls.log = gLogger.getSubLogger(exeName, child=False)

        try:
            result = cls.initialize()
        except Exception, excp:
            gLogger.exception("Exception while initializing %s" % loadName)
            return S_ERROR("Exception while initializing: %s" % str(excp))
Beispiel #2
0
  def initialize(self):
    # Build the URLs
    self._url = self._cfg.getURL()
    if not self._url:
      return S_ERROR("Could not build service URL for %s" % self._name)
    gLogger.verbose("Service URL is %s" % self._url)
    # Load handler
    result = self._loadHandlerInit()
    if not result['OK']:
      return result
    self._handler = result['Value']
    # Initialize lock manager
    self._lockManager = LockManager(self._cfg.getMaxWaitingPetitions())
    self._initMonitoring()
    self._threadPool = ThreadPool(max(1, self._cfg.getMinThreads()),
                                  max(0, self._cfg.getMaxThreads()),
                                  self._cfg.getMaxWaitingPetitions())
    self._threadPool.daemonize()
    self._msgBroker = MessageBroker("%sMSB" % self._name, threadPool=self._threadPool)
    # Create static dict
    self._serviceInfoDict = {'serviceName': self._name,
                             'serviceSectionPath': PathFinder.getServiceSection(self._name),
                             'URL': self._cfg.getURL(),
                             'messageSender': MessageSender(self._name, self._msgBroker),
                             'validNames': self._validNames,
                             'csPaths': [PathFinder.getServiceSection(svcName) for svcName in self._validNames]
                             }
    # Call static initialization function
    try:
      self._handler['class']._rh__initializeClass(dict(self._serviceInfoDict),
                                                  self._lockManager,
                                                  self._msgBroker,
                                                  self._monitor)
      if self._handler['init']:
        for initFunc in self._handler['init']:
          gLogger.verbose("Executing initialization function")
          try:
            result = initFunc(dict(self._serviceInfoDict))
          except Exception as excp:
            gLogger.exception("Exception while calling initialization function", lException=excp)
            return S_ERROR("Exception while calling initialization function: %s" % str(excp))
          if not isReturnStructure(result):
            return S_ERROR("Service initialization function %s must return S_OK/S_ERROR" % initFunc)
          if not result['OK']:
            return S_ERROR("Error while initializing %s: %s" % (self._name, result['Message']))
    except Exception as e:
      errMsg = "Exception while initializing %s" % self._name
      gLogger.exception(e)
      gLogger.exception(errMsg)
      return S_ERROR(errMsg)

    # Load actions after the handler has initialized itself
    result = self._loadActions()
    if not result['OK']:
      return result
    self._actions = result['Value']

    gThreadScheduler.addPeriodicTask(30, self.__reportThreadPoolContents)

    return S_OK()
Beispiel #3
0
    def __init__(self, useCertificates=False):
        """c'tor

    :param self: self reference
    :param bool useCertificates: flag to enable/disable certificates
    """
        Client.__init__(self)
        ## setup logger
        self.log = gLogger.getSubLogger("RequestManagement/RequestClient")

        ## dict to store all RPC clients for easy reuse
        self.__requestRPCClientsDict = {}
        ## local if any defined
        local = PathFinder.getServiceURL("RequestManagement/localURL")
        if local:
            self.__requestRPCClientsDict.setdefault(
                "local", [self.__requestRPCClient(local)])
        ## central if any defined
        central = PathFinder.getServiceURL("RequestManagement/centralURL")
        if central:
            self.__requestRPCClientsDict.setdefault(
                "central", [self.__requestRPCClient(central)])
        ## voboxes if any defined
        voBoxUrls = fromChar(
            PathFinder.getServiceURL("RequestManagement/voBoxURLs"))
        if voBoxUrls:
            self.__requestRPCClientsDict.setdefault("voboxes", [])
            for voBoxURL in randomize(voBoxUrls):
                self.__requestRPCClientsDict["voboxes"].append(
                    self.__requestRPCClient(voBoxURL))

        self.setServer('RequestManagement/centralURL')
Beispiel #4
0
    def _ex_initialize(cls, exeName, loadName):
        cls.__properties = {
            "fullName": exeName,
            "loadName": loadName,
            "section": PathFinder.getExecutorSection(exeName),
            "loadSection": PathFinder.getExecutorSection(loadName),
            "messagesProcessed": 0,
            "reconnects": 0,
            "setup": gConfig.getValue("/DIRAC/Setup", "Unknown"),
        }
        cls.__defaults = {}
        cls.__defaults["MonitoringEnabled"] = True
        cls.__defaults["Enabled"] = True
        cls.__defaults["ControlDirectory"] = os.path.join(rootPath, "control", *exeName.split("/"))
        cls.__defaults["WorkDirectory"] = os.path.join(rootPath, "work", *exeName.split("/"))
        cls.__defaults["ReconnectRetries"] = 10
        cls.__defaults["ReconnectSleep"] = 5
        cls.__defaults["shifterProxy"] = ""
        cls.__defaults["shifterProxyLocation"] = os.path.join(cls.__defaults["WorkDirectory"], ".shifterCred")
        cls.__properties["shifterProxy"] = ""
        cls.__properties["shifterProxyLocation"] = os.path.join(cls.__defaults["WorkDirectory"], ".shifterCred")
        cls.__mindName = False
        cls.__mindExtraArgs = False
        cls.__freezeTime = 0
        cls.__fastTrackEnabled = True
        cls.log = gLogger.getSubLogger(exeName)

        try:
            result = cls.initialize()  # pylint: disable=no-member
        except Exception as excp:
            gLogger.exception("Exception while initializing %s" % loadName, lException=excp)
            return S_ERROR("Exception while initializing: %s" % str(excp))
        if not isReturnStructure(result):
            return S_ERROR("Executor %s does not return an S_OK/S_ERROR after initialization" % loadName)
        return result
Beispiel #5
0
  def __init__( self, useCertificates = False ):
    """c'tor

    :param self: self reference
    :param bool useCertificates: flag to enable/disable certificates
    """
    Client.__init__( self )
    ## setup logger
    self.log = gLogger.getSubLogger( "RequestManagement/RequestClient" )

    ## dict to store all RPC clients for easy reuse
    self.__requestRPCClientsDict = {}
    ## local if any defined
    local = PathFinder.getServiceURL( "RequestManagement/localURL" )
    if local:
      self.__requestRPCClientsDict.setdefault( "local" , [ self.__requestRPCClient( local ) ] )
    ## central if any defined
    central = PathFinder.getServiceURL( "RequestManagement/centralURL" )
    if central:
      self.__requestRPCClientsDict.setdefault( "central", [ self.__requestRPCClient( central ) ] )
    ## voboxes if any defined
    voBoxUrls = fromChar( PathFinder.getServiceURL( "RequestManagement/voBoxURLs" ) )
    if voBoxUrls:
      self.__requestRPCClientsDict.setdefault( "voboxes", [] )
      for voBoxURL in randomize( voBoxUrls ):
        self.__requestRPCClientsDict["voboxes"].append( self.__requestRPCClient( voBoxURL ) )

    self.setServer( 'RequestManagement/centralURL' )
Beispiel #6
0
  def _ex_initialize( cls, exeName, loadName ):
    cls.__properties = { 'fullName' : exeName,
                         'loadName' : loadName,
                         'section' : PathFinder.getExecutorSection( exeName ),
                         'loadSection' : PathFinder.getExecutorSection( loadName ),
                         'messagesProcessed' : 0,
                         'reconnects' : 0,
                         'setup' : gConfig.getValue( "/DIRAC/Setup", "Unknown" ) }
    cls.__basePath = gConfig.getValue( '/LocalSite/InstancePath', rootPath )
    cls.__defaults = {}
    cls.__defaults[ 'MonitoringEnabled' ] = True
    cls.__defaults[ 'Enabled' ] = True
    cls.__defaults[ 'ControlDirectory' ] = os.path.join( cls.__basePath,
                                                          'control',
                                                          *exeName.split( "/" ) )
    cls.__defaults[ 'WorkDirectory' ] = os.path.join( cls.__basePath,
                                                       'work',
                                                       *exeName.split( "/" ) )
    cls.__defaults[ 'ReconnectRetries' ] = 10
    cls.__defaults[ 'ReconnectSleep' ] = 5
    cls.__properties[ 'shifterProxy' ] = ''
    cls.__properties[ 'shifterProxyLocation' ] = os.path.join( cls.__defaults[ 'WorkDirectory' ],
                                                               '.shifterCred' )
    cls.__mindName = False
    cls.__mindExtraArgs = False
    cls.__freezeTime = 0
    cls.__fastTrackEnabled = True
    cls.log = gLogger.getSubLogger( exeName, child = False )

    try:
      result = cls.initialize()
    except Exception, excp:
      gLogger.exception( "Exception while initializing %s" % loadName )
      return S_ERROR( "Exception while initializing: %s" % str( excp ) )
Beispiel #7
0
    def initialize(self):

        self.RequestDBClient = RequestClient()

        gMonitor.registerActivity("Iteration", "Agent Loops", "ZuziaAgent",
                                  "Loops/min", gMonitor.OP_SUM)
        gMonitor.registerActivity("Attempted", "Request Processed",
                                  "ZuziaRAgent", "Requests/min",
                                  gMonitor.OP_SUM)
        gMonitor.registerActivity("Successful", "Request Forward Successful",
                                  "ZuziaAgent", "Requests/min",
                                  gMonitor.OP_SUM)
        gMonitor.registerActivity("Failed", "Request Forward Failed",
                                  "ZuziaAgent", "Requests/min",
                                  gMonitor.OP_SUM)

        self.local = PathFinder.getServiceURL("RequestManagement/localURL")
        if not self.local:
            self.local = AgentModule.am_getOption(self, 'localURL', '')
        if not self.local:
            errStr = 'The RequestManagement/localURL option must be defined.'
            gLogger.fatal(errStr)
            return S_ERROR(errStr)

        self.central = PathFinder.getServiceURL("RequestManagement/centralURL")
        if not self.central:
            errStr = 'The RequestManagement/centralURL option must be defined.'
            gLogger.fatal(errStr)
            return S_ERROR(errStr)
        return S_OK()
Beispiel #8
0
  def initialize( self ):
    #Build the URLs
    self._url = self._cfg.getURL()
    if not self._url:
      return S_ERROR( "Could not build service URL for %s" % self._name )
    gLogger.verbose( "Service URL is %s" % self._url )
    #Load handler
    result = self._loadHandlerInit()
    if not result[ 'OK' ]:
      return result
    self._handler = result[ 'Value' ]
    #Initialize lock manager
    self._lockManager = LockManager( self._cfg.getMaxWaitingPetitions() )
    self._initMonitoring()
    self._threadPool = ThreadPool( max( 1, self._cfg.getMinThreads() ),
                                   max( 0, self._cfg.getMaxThreads() ),
                                   self._cfg.getMaxWaitingPetitions() )
    self._threadPool.daemonize()
    self._msgBroker = MessageBroker( "%sMSB" % self._name, threadPool = self._threadPool )
    #Create static dict
    self._serviceInfoDict = { 'serviceName' : self._name,
                              'serviceSectionPath' : PathFinder.getServiceSection( self._name ),
                              'URL' : self._cfg.getURL(),
                              'messageSender' : MessageSender( self._name, self._msgBroker ),
                              'validNames' : self._validNames,
                              'csPaths' : [ PathFinder.getServiceSection( svcName ) for svcName in self._validNames ]
                            }
    #Call static initialization function
    try:
      self._handler[ 'class' ]._rh__initializeClass( dict( self._serviceInfoDict ),
                                                     self._lockManager,
                                                     self._msgBroker,
                                                     self._monitor )
      if self._handler[ 'init' ]:
        for initFunc in self._handler[ 'init' ]:
          gLogger.verbose( "Executing initialization function" )
          try:
            result = initFunc( dict( self._serviceInfoDict ) )
          except Exception as excp:
            gLogger.exception( "Exception while calling initialization function", lException = excp )
            return S_ERROR( "Exception while calling initialization function: %s" % str( excp ) )
          if not isReturnStructure( result ):
            return S_ERROR( "Service initialization function %s must return S_OK/S_ERROR" % initFunc )
          if not result[ 'OK' ]:
            return S_ERROR( "Error while initializing %s: %s" % ( self._name, result[ 'Message' ] ) )
    except Exception as e:
      errMsg = "Exception while initializing %s" % self._name
      gLogger.exception( e )
      gLogger.exception( errMsg )
      return S_ERROR( errMsg )

    #Load actions after the handler has initialized itself
    result = self._loadActions()
    if not result[ 'OK' ]:
      return result
    self._actions = result[ 'Value' ]

    gThreadScheduler.addPeriodicTask( 30, self.__reportThreadPoolContents )

    return S_OK()
Beispiel #9
0
    def execute(self):
        """ This agent is the smallest and (cutest) of all the DIRAC agents in existence.
    """
        gMonitor.addMark("Iteration", 1)

        central = PathFinder.getServiceURL("RequestManagement/centralURL")
        if central:
            self.central = central
        local = PathFinder.getServiceURL("RequestManagement/localURL")
        if local:
            self.local = local

        res = self.RequestDBClient.serveRequest(url=self.local)
        if not res['OK']:
            gLogger.error(
                "ZuziaAgent.execute: Failed to get request from database.",
                self.local)
            return S_OK()
        elif not res['Value']:
            gLogger.info(
                "ZuziaAgent.execute: No requests to be executed found.")
            return S_OK()

        gMonitor.addMark("Attempted", 1)

        requestString = res['Value']['RequestString']
        requestName = res['Value']['RequestName']
        gLogger.info("ZuziaAgent.execute: Obtained request %s" % requestName)

        gLogger.info("ZuziaAgent.execute: Attempting to set request to %s." %
                     self.central)
        res = self.RequestDBClient.setRequest(requestName, requestString,
                                              self.central)
        if res['OK']:
            gMonitor.addMark("Successful", 1)
            gLogger.info("ZuziaAgent.execute: Successfully put request.")
        else:
            gMonitor.addMark("Failed", 1)
            gLogger.error("ZuziaAgent.execute: Failed to set request to",
                          self.central)
            gLogger.info(
                "ZuziaAgent.execute: Attempting to set request to %s." %
                self.local)
            res = self.RequestDBClient.setRequest(requestName, requestString,
                                                  self.local)
            if res['OK']:
                gLogger.info("ZuziaAgent.execute: Successfully put request.")
            else:
                gLogger.error("ZuziaAgent.execute: Failed to set request to",
                              self.local)
                while not res['OK']:
                    gLogger.info(
                        "ZuziaAgent.execute: Attempting to set request to anywhere."
                    )
                    res = self.RequestDBClient.setRequest(
                        requestName, requestString)
                gLogger.info(
                    "ZuziaAgent.execute: Successfully put request to %s." %
                    res["Server"])
        return S_OK()
Beispiel #10
0
    def __initializeService(cls, relativeUrl, absoluteUrl):
        """
      Initialize a service.
      The work is only perform once at the first request.

      :param relativeUrl: relative URL, e.g. ``/<System>/<Component>``
      :param absoluteUrl: full URL e.g. ``https://<host>:<port>/<System>/<Component>``

      :returns: S_OK
    """
        # If the initialization was already done successfuly,
        # we can just return
        if cls.__init_done:
            return S_OK()

        # Otherwise, do the work but with a lock
        with cls.__init_lock:

            # Check again that the initialization was not done by another thread
            # while we were waiting for the lock
            if cls.__init_done:
                return S_OK()

            # Url starts with a "/", we just remove it
            serviceName = relativeUrl[1:]

            cls._startTime = datetime.utcnow()
            sLog.info("First use, initializing service...", "%s" % relativeUrl)
            cls._authManager = AuthManager(
                "%s/Authorization" % PathFinder.getServiceSection(serviceName))

            cls._initMonitoring(serviceName, absoluteUrl)

            cls._serviceName = serviceName
            cls._validNames = [serviceName]
            serviceInfo = {
                'serviceName': serviceName,
                'serviceSectionPath':
                PathFinder.getServiceSection(serviceName),
                'csPaths': [PathFinder.getServiceSection(serviceName)],
                'URL': absoluteUrl
            }
            cls._serviceInfoDict = serviceInfo

            cls.__monitorLastStatsUpdate = time.time()

            cls.initializeHandler(serviceInfo)

            cls.__init_done = True

            return S_OK()
def initializeReportGeneratorHandler( serviceInfo ):
  global gAccountingDB
  gAccountingDB = AccountingDB( readOnly = True )
  #Get data location
  reportSection = PathFinder.getServiceSection( "Accounting/ReportGenerator" )
  dataPath = gConfig.getValue( "%s/DataLocation" % reportSection, "data/accountingGraphs" )
  dataPath = dataPath.strip()
  if "/" != dataPath[0]:
    dataPath = os.path.realpath( "%s/%s" % ( gConfig.getValue( '/LocalSite/InstancePath', rootPath ), dataPath ) )
  gLogger.info( "Data will be written into %s" % dataPath )
  try:
    os.makedirs( dataPath )
  except:
    pass
  try:
    testFile = "%s/acc.jarl.test" % dataPath
    fd = file( testFile, "w" )
    fd.close()
    os.unlink( testFile )
  except IOError:
    gLogger.fatal( "Can't write to %s" % dataPath )
    return S_ERROR( "Data location is not writable" )
  gDataCache.setGraphsLocation( dataPath )
  gMonitor.registerActivity( "plotsDrawn", "Drawn plot images", "Accounting reports", "plots", gMonitor.OP_SUM )
  gMonitor.registerActivity( "reportsRequested", "Generated reports", "Accounting reports", "reports", gMonitor.OP_SUM )
  return S_OK()
Beispiel #12
0
    def initialize(self):

        self.RequestDBClient = RequestClient()
        backend = self.am_getOption('Backend', '')
        self.RequestDB = False
        if backend == 'mysql':
            from DIRAC.RequestManagementSystem.DB.RequestDBMySQL import RequestDBMySQL
            requestDB = RequestDBMySQL()
            if requestDB._connected:
                self.RequestDB = requestDB

        gMonitor.registerActivity("Iteration", "Agent Loops",
                                  "DISETForwardingAgent", "Loops/min",
                                  gMonitor.OP_SUM)
        gMonitor.registerActivity("Attempted", "Request Processed",
                                  "DISETForwardingAgent", "Requests/min",
                                  gMonitor.OP_SUM)
        gMonitor.registerActivity("Successful", "Request Forward Successful",
                                  "DISETForwardingAgent", "Requests/min",
                                  gMonitor.OP_SUM)
        gMonitor.registerActivity("Failed", "Request Forward Failed",
                                  "DISETForwardingAgent", "Requests/min",
                                  gMonitor.OP_SUM)

        self.local = PathFinder.getServiceURL("RequestManagement/localURL")
        if not self.local:
            self.local = AgentModule.am_getOption(self, 'localURL', '')
        if not self.local:
            errStr = 'The RequestManagement/localURL option must be defined.'
            gLogger.fatal(errStr)
            return S_ERROR(errStr)
        return S_OK()
Beispiel #13
0
def initializeDataLoggingHandler(serviceInfo):

    global dataPath
    global logDB
    logDB = DataLoggingDB()

    monitoringSection = PathFinder.getServiceSection(
        "DataManagement/DataLogging")
    #Get data location
    retDict = gConfig.getOption("%s/DataLocation" % monitoringSection,
                                "dataLoggingPlots")
    if not retDict['OK']:
        return retDict
    dataPath = retDict['Value'].strip()
    if "/" != dataPath[0]:
        dataPath = os.path.realpath(
            "%s/%s" %
            (gConfig.getValue('/LocalSite/InstancePath', rootPath), dataPath))
    gLogger.info("Data will be written into %s" % dataPath)
    try:
        os.makedirs(dataPath)
    except:
        pass
    try:
        testFile = "%s/mon.jarl.test" % dataPath
        fd = file(testFile, "w")
        fd.close()
        os.unlink(testFile)
    except IOError:
        gLogger.fatal("Can't write to %s" % dataPath)
        return S_ERROR("Data location is not writable")
    return S_OK()
  def initialize( self ):

    self.RequestDBClient = RequestClient()
    backend = self.am_getOption( 'Backend', '' )
    self.RequestDB = False
    if backend == 'mysql':
      from DIRAC.RequestManagementSystem.DB.RequestDBMySQL import RequestDBMySQL
      requestDB = RequestDBMySQL()
      if requestDB._connected:
        self.RequestDB = requestDB



    gMonitor.registerActivity( "Iteration", "Agent Loops", "DISETForwardingAgent", "Loops/min", gMonitor.OP_SUM )
    gMonitor.registerActivity( "Attempted", "Request Processed", "DISETForwardingAgent", "Requests/min", gMonitor.OP_SUM )
    gMonitor.registerActivity( "Successful", "Request Forward Successful", "DISETForwardingAgent", "Requests/min", gMonitor.OP_SUM )
    gMonitor.registerActivity( "Failed", "Request Forward Failed", "DISETForwardingAgent", "Requests/min", gMonitor.OP_SUM )

    self.local = PathFinder.getServiceURL( "RequestManagement/localURL" )
    if not self.local:
      self.local = AgentModule.am_getOption( self, 'localURL', '' )
    if not self.local:
      errStr = 'The RequestManagement/localURL option must be defined.'
      gLogger.fatal( errStr )
      return S_ERROR( errStr )
    return S_OK()
 def _checkServiceURL(self, serviceName, options):
     """Ensure service URL is properly configured in the CS."""
     url = self._getURL(serviceName, options)
     system = options["System"]
     module = options["Module"]
     self.log.info("Checking URLs for %s/%s" % (system, module))
     urlsConfigPath = Path.cfgPath(
         PathFinder.getSystemURLSection(system=system, setup=self.setup),
         module)
     urls = gConfig.getValue(urlsConfigPath, [])
     self.log.debug("Found configured URLs for %s: %s" % (module, urls))
     self.log.debug("This URL is %s" % url)
     runitStatus = options["RunitStatus"]
     wouldHave = "Would have " if not self.commitURLs else ""
     if runitStatus == "Run" and url not in urls:
         urls.append(url)
         message = "%sAdded URL %s to URLs for %s/%s" % (wouldHave, url,
                                                         system, module)
         self.log.info(message)
         self.accounting[serviceName + "/URL"]["Treatment"] = message
         self.csAPI.modifyValue(urlsConfigPath, ",".join(urls))
     if runitStatus == "Down" and url in urls:
         urls.remove(url)
         message = "%sRemoved URL %s from URLs for %s/%s" % (wouldHave, url,
                                                             system, module)
         self.log.info(message)
         self.accounting[serviceName + "/URL"]["Treatment"] = message
         self.csAPI.modifyValue(urlsConfigPath, ",".join(urls))
Beispiel #16
0
    def loadModules(self, modulesList, hideExceptions=False):
        """
        Load all modules required in moduleList
        """
        for modName in modulesList:
            gLogger.verbose("Checking %s" % modName)
            # if it's a executor modName name just load it and be done with it
            if "/" in modName:
                gLogger.verbose(
                    "Module %s seems to be a valid name. Try to load it!" %
                    modName)
                result = self.loadModule(modName,
                                         hideExceptions=hideExceptions)
                if not result["OK"]:
                    return result
                continue
            # Check if it's a system name
            # Look in the CS
            system = modName
            # Can this be generated with sectionFinder?
            csPath = "%s/Executors" % PathFinder.getSystemSection(system)
            gLogger.verbose("Exploring %s to discover modules" % csPath)
            result = gConfig.getSections(csPath)
            if result["OK"]:
                # Add all modules in the CS :P
                for modName in result["Value"]:
                    result = self.loadModule("%s/%s" % (system, modName),
                                             hideExceptions=hideExceptions)
                    if not result["OK"]:
                        return result
            # Look what is installed
            parentModule = None
            for rootModule in extensionsByPriority():
                if not system.endswith("System"):
                    system += "System"
                parentImport = "%s.%s.%s" % (rootModule, system,
                                             self.__csSuffix)
                # HERE!
                result = recurseImport(parentImport)
                if not result["OK"]:
                    return result
                parentModule = result["Value"]
                if parentModule:
                    break
            if not parentModule:
                continue
            parentPath = parentModule.__path__[0]
            gLogger.notice("Found modules path at %s" % parentImport)
            for entry in os.listdir(parentPath):
                if entry == "__init__.py" or not entry.endswith(".py"):
                    continue
                if not os.path.isfile(os.path.join(parentPath, entry)):
                    continue
                modName = "%s/%s" % (system, entry[:-3])
                gLogger.verbose("Trying to import %s" % modName)
                result = self.loadModule(modName,
                                         hideExceptions=hideExceptions,
                                         parentModule=parentModule)

        return S_OK()
Beispiel #17
0
  def export_sendMail( self, address, subject, body, fromAddress, avoidSpam = False ):
    """ Send an email with supplied body to the specified address using the Mail utility.

        if avoidSpam is True, then emails are first added to a set so that duplicates are removed,
        and sent every hour.
    """
    gLogger.verbose( 'Received signal to send the following mail to %s:\nSubject = %s\n%s' % ( address, subject, body ) )
    eMail = Mail()
    notificationSection = PathFinder.getServiceSection( "Framework/Notification" )
    csSection = notificationSection + '/SMTP'
    eMail._smtpHost = gConfig.getValue( '%s/Host' % csSection )
    eMail._smtpPort = gConfig.getValue( '%s/Port' % csSection )
    eMail._smtpLogin = gConfig.getValue( '%s/Login' % csSection )
    eMail._smtpPasswd = gConfig.getValue( '%s/Password' % csSection )
    eMail._smtpPtcl = gConfig.getValue( '%s/Protocol' % csSection )
    eMail._subject = subject
    eMail._message = body
    eMail._mailAddress = address
    if not fromAddress == 'None':
      eMail._fromAddress = fromAddress
    if gConfig.getValue( '%s/FromAddress' % csSection ):
      eMail._fromAddress = gConfig.getValue( '%s/FromAddress' % csSection )
    if avoidSpam:
      gMailSet.add(eMail)
      return S_OK("Mail added to gMailSet")
    else:
      result = eMail._send()
      if not result['OK']:
        gLogger.warn( 'Could not send mail with the following message:\n%s' % result['Message'] )
      else:
        gLogger.info( 'Mail sent successfully to %s with subject %s' % ( address, subject ) )
        gLogger.debug( result['Value'] )

    return result
Beispiel #18
0
  def export_sendMail( self, address, subject, body, fromAddress, avoidSpam = False ):
    """ Send an email with supplied body to the specified address using the Mail utility.

        if avoidSpam is True, then emails are first added to a set so that duplicates are removed,
        and sent every hour.
    """
    gLogger.verbose( 'Received signal to send the following mail to %s:\nSubject = %s\n%s' % ( address, subject, body ) )
    eMail = Mail()
    notificationSection = PathFinder.getServiceSection( "Framework/Notification" )
    csSection = notificationSection + '/SMTPServer'
    eMail._smtpHost = gConfig.getValue( '%s/Host' % csSection )
    eMail._smtpPort = gConfig.getValue( '%s/Port' % csSection )
    eMail._smtpLogin = gConfig.getValue( '%s/Login' % csSection )
    eMail._smtpPasswd = gConfig.getValue( '%s/Password' % csSection )
    eMail._smtpPtcl = gConfig.getValue( '%s/Protocol' % csSection )
    eMail._subject = subject
    eMail._message = body
    eMail._mailAddress = address
    if not fromAddress == 'None':
      eMail._fromAddress = fromAddress
    if gConfig.getValue( '%s/FromAddress' % csSection ):
      eMail._fromAddress = gConfig.getValue( '%s/FromAddress' % csSection )
    if avoidSpam:
      gMailSet.add(eMail)
      return S_OK("Mail added to gMailSet")
    else:
      result = eMail._send()
      if not result['OK']:
        gLogger.warn( 'Could not send mail with the following message:\n%s' % result['Message'] )
      else:
        gLogger.info( 'Mail sent successfully to %s with subject %s' % ( address, subject ) )
        gLogger.debug( result['Value'] )

    return result
 def __init__(self, url=False, useCertificates=False):
     """ Constructor of the LCGFileCatalogProxy client class
 """
     self.name = 'LFCProxy'
     self.valid = False
     try:
         if url:
             self.url = url
         else:
             url = PathFinder.getServiceURL(
                 'DataManagement/LcgFileCatalogProxy')
             if not url:
                 return
             self.url = url
         self.server = RPCClient(self.url,
                                 timeout=120,
                                 useCertificates=useCertificates)
         if not self.server:
             return
         else:
             self.valid = True
     except Exception, x:
         gLogger.exception(
             'Exception while creating connection to LcgFileCatalog proxy server',
             '', x)
         return
Beispiel #20
0
    def __init__(self, serviceData):
        """
        Init the variables for the service

        :param serviceData: dict with modName, standalone, loadName, moduleObj, classObj. e.g.:
          {'modName': 'Framework/serviceName',
          'standalone': True,
          'loadName': 'Framework/serviceName',
          'moduleObj': <module 'serviceNameHandler' from '/home/DIRAC/FrameworkSystem/Service/serviceNameHandler.pyo'>,
          'classObj': <class 'serviceNameHandler.serviceHandler'>}

        """
        self._svcData = serviceData
        self._name = serviceData["modName"]
        self._startTime = datetime.datetime.utcnow()
        self._validNames = [serviceData["modName"]]
        if serviceData["loadName"] not in self._validNames:
            self._validNames.append(serviceData["loadName"])
        self._cfg = ServiceConfiguration(list(self._validNames))
        self._standalone = serviceData["standalone"]
        self.__monitorLastStatsUpdate = time.time()
        self._stats = {"queries": 0, "connections": 0}
        self._authMgr = AuthManager(
            "%s/Authorization" %
            PathFinder.getServiceSection(serviceData["loadName"]))
        self._transportPool = getGlobalTransportPool()
        self.__cloneId = 0
        self.__maxFD = 0
        self.activityMonitoring = False
        # Check if monitoring is enabled
        if "Monitoring" in Operations().getMonitoringBackends(
                monitoringType="ServiceMonitoring"):
            self.activityMonitoring = True
Beispiel #21
0
    def initialize(self):
        ''' NotifyAgent initialization
    '''

        try:
            with sqlite3.connect(self.cacheFile) as conn:
                conn.execute(
                    '''CREATE TABLE IF NOT EXISTS ProductionManagementCache(
                      reqId VARCHAR(64) NOT NULL DEFAULT "",
                      reqType VARCHAR(64) NOT NULL DEFAULT "",
                      reqWG VARCHAR(64) NOT NULL DEFAULT "",
                      reqName VARCHAR(64) NOT NULL DEFAULT "",
                      SimCondition VARCHAR(64) NOT NULL DEFAULT "",
                      ProPath VARCHAR(64) NOT NULL DEFAULT "",
                      thegroup VARCHAR(64) NOT NULL DEFAULT "",
                      reqInform VARCHAR(64) NOT NULL DEFAULT ""
                     );''')
        except sqlite3.OperationalError:
            self.log.error('Email cache database is locked')

        self.diracAdmin = DiracAdmin()

        self.csS = PathFinder.getServiceSection(
            'ProductionManagement/ProductionRequest')

        self.fromAddress = gConfig.getValue('%s/fromAddress' % self.csS, '')
        if not self.fromAddress:
            self.log.info(
                'No fromAddress is defined, a default value will be used instead'
            )
            self.fromAddress = '*****@*****.**'

        return S_OK()
Beispiel #22
0
  def export_sendSMS( self, userName, body, fromAddress ):
    """ Send an SMS with supplied body to the specified DIRAC user using the Mail utility via an SMS switch.
    """
    gLogger.verbose( 'Received signal to send the following SMS to %s:\n%s' % ( userName, body ) )
    mobile = gConfig.getValue( '/Registry/Users/%s/Mobile' % userName, '' )
    if not mobile:
      return S_ERROR( 'No registered mobile number for %s' % userName )

    csSection = PathFinder.getServiceSection( 'Framework/Notification' )
    smsSwitch = gConfig.getValue( '%s/SMSSwitch' % csSection, '' )
    if not smsSwitch:
      return S_ERROR( 'No SMS switch is defined in CS path %s/SMSSwitch' % csSection )

    address = '%s@%s' % ( mobile, smsSwitch )
    subject = 'DIRAC SMS'
    m = Mail()
    m._subject = subject
    m._message = body
    m._mailAddress = address
    if not fromAddress == 'None':
      m._fromAddress = fromAddress
    result = m._send()
    if not result['OK']:
      gLogger.warn( 'Could not send SMS to %s with the following message:\n%s' % ( userName, result['Message'] ) )
    else:
      gLogger.info( 'SMS sent successfully to %s ' % ( userName ) )
      gLogger.debug( result['Value'] )

    return result
Beispiel #23
0
    def __init__(self, serviceData):
        """
      Init the variables for the service

      :param serviceData: dict with modName, standalone, loadName, moduleObj, classObj. e.g.:
        {'modName': 'Framework/serviceName',
        'standalone': True,
        'loadName': 'Framework/serviceName',
        'moduleObj': <module 'serviceNameHandler' from '/home/DIRAC/FrameworkSystem/Service/serviceNameHandler.pyo'>,
        'classObj': <class 'serviceNameHandler.serviceHandler'>}

        Standalone is true if there is only one service started
        If it's false, every service is linked to a different MonitoringClient
    """
        self._svcData = serviceData
        self._name = serviceData['modName']
        self._startTime = Time.dateTime()
        self._validNames = [serviceData['modName']]
        if serviceData['loadName'] not in self._validNames:
            self._validNames.append(serviceData['loadName'])
        self._cfg = ServiceConfiguration(list(self._validNames))
        if serviceData['standalone']:
            self._monitor = gMonitor
        else:
            self._monitor = MonitoringClient()
        self.__monitorLastStatsUpdate = time.time()
        self._stats = {'queries': 0, 'connections': 0}
        self._authMgr = AuthManager(
            "%s/Authorization" %
            PathFinder.getServiceSection(serviceData['loadName']))
        self._transportPool = getGlobalTransportPool()
        self.__cloneId = 0
        self.__maxFD = 0
Beispiel #24
0
    def __init__(self):
        """c'tor

        just setting validation order
        """
        self.validator = (
            self._hasRequestName,
            self._hasOwner,
            self._hasOperations,
            self._hasType,
            self._hasFiles,
            self._hasRequiredAttrs,
            self._hasChecksumAndChecksumType,
        )

        configPath = PathFinder.getAgentSection(
            "RequestManagement/RequestExecutingAgent")

        # # operation handlers over here
        opHandlersPath = "%s/%s" % (configPath, "OperationHandlers")
        opHandlers = gConfig.getSections(opHandlersPath)
        if not opHandlers["OK"]:
            gLogger.error(opHandlers["Message"])
        else:
            self.opHandlers = set(opHandlers["Value"])
Beispiel #25
0
 def __init__(self, nameList):
     self.serviceName = nameList[0]
     self.serviceURL = None
     self.nameList = nameList
     self.pathList = []
     for svcName in nameList:
         self.pathList.append(PathFinder.getServiceSection(svcName))
def initializeDataLoggingHandler( serviceInfo ):

  global dataPath
  global logDB
  logDB = DataLoggingDB()

  monitoringSection = PathFinder.getServiceSection( "DataManagement/DataLogging" )
  #Get data location
  retDict = gConfig.getOption( "%s/DataLocation" % monitoringSection, "dataLoggingPlots" )
  if not retDict[ 'OK' ]:
    return retDict
  dataPath = retDict[ 'Value' ].strip()
  if "/" != dataPath[0]:
    dataPath = os.path.realpath( "%s/%s" % ( gConfig.getValue( '/LocalSite/InstancePath', rootPath ), dataPath ) )
  gLogger.info( "Data will be written into %s" % dataPath )
  try:
    os.makedirs( dataPath )
  except:
    pass
  try:
    testFile = "%s/mon.jarl.test" % dataPath
    fd = file( testFile, "w" )
    fd.close()
    os.unlink( testFile )
  except IOError:
    gLogger.fatal( "Can't write to %s" % dataPath )
    return S_ERROR( "Data location is not writable" )
  return S_OK()
Beispiel #27
0
def initializeMonitoringHandler( serviceInfo ):
  #Check that the path is writable
  monitoringSection = PathFinder.getServiceSection( "Framework/Monitoring" )
  #Get data location
  dataPath = gConfig.getValue( "%s/DataLocation" % monitoringSection, "data/monitoring" )
  dataPath = dataPath.strip()
  if "/" != dataPath[0]:
    dataPath = os.path.realpath( "%s/%s" % ( gConfig.getValue( '/LocalSite/InstancePath', rootPath ), dataPath ) )
  gLogger.info( "Data will be written into %s" % dataPath )
  try:
    os.makedirs( dataPath )
  except:
    pass
  try:
    testFile = "%s/mon.jarl.test" % dataPath
    fd = file( testFile, "w" )
    fd.close()
    os.unlink( testFile )
  except IOError:
    gLogger.fatal( "Can't write to %s" % dataPath )
    return S_ERROR( "Data location is not writable" )
  #Define globals
  gServiceInterface.initialize( dataPath )
  if not gServiceInterface.initializeDB():
    return S_ERROR( "Can't start db engine" )
  gMonitor.registerActivity( "cachedplots", "Cached plot images", "Monitoring plots", "plots", gMonitor.OP_SUM )
  gMonitor.registerActivity( "drawnplots", "Drawn plot images", "Monitoring plots", "plots", gMonitor.OP_SUM )
  return S_OK()
Beispiel #28
0
    def discoverHandlers(self):
        """
        Force the discovery of URL, automatic call when we try to get handlers for the first time.
        You can disable the automatic call with autoDiscovery=False at initialization
        """
        gLogger.debug("Trying to auto-discover the handlers for Tornado")

        # Look in config
        diracSystems = gConfig.getSections("/Systems")
        serviceList = []
        if diracSystems["OK"]:
            for system in diracSystems["Value"]:
                try:
                    instance = PathFinder.getSystemInstance(system)
                    services = gConfig.getSections("/Systems/%s/%s/Services" % (system, instance))
                    if services["OK"]:
                        for service in services["Value"]:
                            newservice = "%s/%s" % (system, service)

                            # We search in the CS all handlers which used HTTPS as protocol
                            isHTTPS = gConfig.getValue(
                                "/Systems/%s/%s/Services/%s/Protocol" % (system, instance, service)
                            )
                            if isHTTPS and isHTTPS.lower() == "https":
                                serviceList.append(newservice)
                # On systems sometime you have things not related to services...
                except RuntimeError:
                    pass
        return self.loadHandlersByServiceName(serviceList)
Beispiel #29
0
    def export_sendSMS(self, userName, body, fromAddress):
        """ Send an SMS with supplied body to the specified DIRAC user using the Mail utility via an SMS switch.
    """
        gLogger.verbose(
            'Received signal to send the following SMS to %s:\n%s' %
            (userName, body))
        mobile = gConfig.getValue('/Registry/Users/%s/Mobile' % userName, '')
        if not mobile:
            return S_ERROR('No registered mobile number for %s' % userName)

        csSection = PathFinder.getServiceSection('Framework/Notification')
        smsSwitch = gConfig.getValue('%s/SMSSwitch' % csSection, '')
        if not smsSwitch:
            return S_ERROR('No SMS switch is defined in CS path %s/SMSSwitch' %
                           csSection)

        address = '%s@%s' % (mobile, smsSwitch)
        subject = 'DIRAC SMS'
        m = Mail()
        m._subject = subject
        m._message = body
        m._mailAddress = address
        if not fromAddress == 'None':
            m._fromAddress = fromAddress
        result = m._send()
        if not result['OK']:
            gLogger.warn(
                'Could not send SMS to %s with the following message:\n%s' %
                (userName, result['Message']))
        else:
            gLogger.info('SMS sent successfully to %s ' % (userName))
            gLogger.debug(result['Value'])

        return result
Beispiel #30
0
def initializePlottingHandler( serviceInfo ):

  #Get data location
  plottingSection = PathFinder.getServiceSection( "Framework/Plotting" )
  dataPath = gConfig.getValue( "%s/DataLocation" % plottingSection, "data/graphs" )
  dataPath = dataPath.strip()
  if "/" != dataPath[0]:
    dataPath = os.path.realpath( "%s/%s" % ( gConfig.getValue( '/LocalSite/InstancePath', rootPath ), dataPath ) )
  gLogger.info( "Data will be written into %s" % dataPath )
  try:
    os.makedirs( dataPath )
  except:
    pass
  try:
    testFile = "%s/plot__.test" % dataPath
    fd = file( testFile, "w" )
    fd.close()
    os.unlink( testFile )
  except IOError:
    gLogger.fatal( "Can't write to %s" % dataPath )
    return S_ERROR( "Data location is not writable" )

  gPlotCache.setPlotsLocation( dataPath )
  gMonitor.registerActivity( "plotsDrawn", "Drawn plot images", "Plotting requests", "plots", gMonitor.OP_SUM )
  return S_OK()
 def initializeHandler(cls, serviceInfo):
     multiPath = PathFinder.getDatabaseSection("Accounting/MultiDB")
     cls.__acDB = MultiAccountingDB(multiPath, readOnly=True)
     # Get data location
     reportSection = serviceInfo["serviceSectionPath"]
     dataPath = gConfig.getValue("%s/DataLocation" % reportSection,
                                 "data/accountingGraphs")
     dataPath = dataPath.strip()
     if "/" != dataPath[0]:
         dataPath = os.path.realpath("%s/%s" % (gConfig.getValue(
             "/LocalSite/InstancePath", rootPath), dataPath))
     gLogger.info("Data will be written into %s" % dataPath)
     mkDir(dataPath)
     try:
         testFile = "%s/acc.jarl.test" % dataPath
         with open(testFile, "w"):
             pass
         os.unlink(testFile)
     except IOError:
         gLogger.fatal("Can't write to %s" % dataPath)
         return S_ERROR("Data location is not writable")
     gDataCache.setGraphsLocation(dataPath)
     gMonitor.registerActivity("plotsDrawn", "Drawn plot images",
                               "Accounting reports", "plots",
                               gMonitor.OP_SUM)
     gMonitor.registerActivity("reportsRequested", "Generated reports",
                               "Accounting reports", "reports",
                               gMonitor.OP_SUM)
     return S_OK()
 def __init__( self, nameList ):
   self.serviceName = nameList[0]
   self.serviceURL = False
   self.nameList = nameList
   self.pathList = []
   for svcName in nameList:
     self.pathList.append( PathFinder.getServiceSection( svcName ) )
Beispiel #33
0
def initializePlottingHandler(serviceInfo):

    #Get data location
    plottingSection = PathFinder.getServiceSection("Framework/Plotting")
    dataPath = gConfig.getValue("%s/DataLocation" % plottingSection,
                                "data/graphs")
    dataPath = dataPath.strip()
    if "/" != dataPath[0]:
        dataPath = os.path.realpath(
            "%s/%s" %
            (gConfig.getValue('/LocalSite/InstancePath', rootPath), dataPath))
    gLogger.info("Data will be written into %s" % dataPath)
    try:
        os.makedirs(dataPath)
    except:
        pass
    try:
        testFile = "%s/plot__.test" % dataPath
        fd = file(testFile, "w")
        fd.close()
        os.unlink(testFile)
    except IOError:
        gLogger.fatal("Can't write to %s" % dataPath)
        return S_ERROR("Data location is not writable")

    gPlotCache.setPlotsLocation(dataPath)
    gMonitor.registerActivity("plotsDrawn", "Drawn plot images",
                              "Plotting requests", "plots", gMonitor.OP_SUM)
    return S_OK()
Beispiel #34
0
def initializeReportGeneratorHandler(serviceInfo):
    global gAccountingDB
    gAccountingDB = AccountingDB(readOnly=True)
    #Get data location
    reportSection = PathFinder.getServiceSection("Accounting/ReportGenerator")
    dataPath = gConfig.getValue("%s/DataLocation" % reportSection,
                                "data/accountingGraphs")
    dataPath = dataPath.strip()
    if "/" != dataPath[0]:
        dataPath = os.path.realpath(
            "%s/%s" %
            (gConfig.getValue('/LocalSite/InstancePath', rootPath), dataPath))
    gLogger.info("Data will be written into %s" % dataPath)
    try:
        os.makedirs(dataPath)
    except:
        pass
    try:
        testFile = "%s/acc.jarl.test" % dataPath
        fd = file(testFile, "w")
        fd.close()
        os.unlink(testFile)
    except IOError:
        gLogger.fatal("Can't write to %s" % dataPath)
        return S_ERROR("Data location is not writable")
    gDataCache.setGraphsLocation(dataPath)
    gMonitor.registerActivity("plotsDrawn", "Drawn plot images",
                              "Accounting reports", "plots", gMonitor.OP_SUM)
    gMonitor.registerActivity("reportsRequested", "Generated reports",
                              "Accounting reports", "reports", gMonitor.OP_SUM)
    return S_OK()
Beispiel #35
0
 def initialize(self):
     # Build the URLs
     self._url = self._cfg.getURL()
     if not self._url:
         return S_ERROR("Could not build service URL for %s" % self._name)
     gLogger.verbose("Service URL is %s" % self._url)
     # Load handler
     result = self._loadHandlerInit()
     if not result["OK"]:
         return result
     self._handler = result["Value"]
     # Initialize lock manager
     self._lockManager = LockManager(self._cfg.getMaxWaitingPetitions())
     self._initMonitoring()
     self._threadPool = ThreadPool(1, max(0, self._cfg.getMaxThreads()), self._cfg.getMaxWaitingPetitions())
     self._threadPool.daemonize()
     self._msgBroker = MessageBroker("%sMSB" % self._name, threadPool=self._threadPool)
     # Create static dict
     self._serviceInfoDict = {
         "serviceName": self._name,
         "serviceSectionPath": PathFinder.getServiceSection(self._name),
         "URL": self._cfg.getURL(),
         "messageSender": MessageSender(self._name, self._msgBroker),
         "validNames": self._validNames,
         "csPaths": [PathFinder.getServiceSection(svcName) for svcName in self._validNames],
     }
     # Call static initialization function
     try:
         self._handler["class"]._rh__initializeClass(
             dict(self._serviceInfoDict), self._lockManager, self._msgBroker, self._monitor
         )
         if self._handler["init"]:
             for initFunc in self._handler["init"]:
                 gLogger.verbose("Executing initialization function")
                 try:
                     result = initFunc(dict(self._serviceInfoDict))
                 except Exception, excp:
                     gLogger.exception("Exception while calling initialization function")
                     return S_ERROR("Exception while calling initialization function: %s" % str(excp))
                 if not isReturnStructure(result):
                     return S_ERROR("Service initialization function %s must return S_OK/S_ERROR" % initFunc)
                 if not result["OK"]:
                     return S_ERROR("Error while initializing %s: %s" % (self._name, result["Message"]))
     except Exception, e:
         errMsg = "Exception while initializing %s" % self._name
         gLogger.exception(errMsg)
         return S_ERROR(errMsg)
Beispiel #36
0
    def _getCSAuthorizarionSection(cls, apiName):
        """Search endpoint auth section.

        :param str apiName: API name, see :py:meth:`_getFullComponentName`

        :return: str
        """
        return "%s/Authorization" % PathFinder.getAPISection(apiName)
Beispiel #37
0
 def ftsManager( cls, timeout = 300 ):
   """ get FTSManager instance """
   if not cls.__ftsManager:
     url = PathFinder.getServiceURL( "DataManagement/FTSManager" )
     if not url:
       raise RuntimeError( "CS option DataManagement/FTSManager URL is not set!" )
     cls.__ftsManager = RPCClient( url, timeout = timeout )
   return cls.__ftsManager
Beispiel #38
0
    def _getCSAuthorizarionSection(cls, serviceName):
        """Search service auth section.

        :param str serviceName: service name, see :py:meth:`_getFullComponentName`

        :return: str
        """
        return "%s/Authorization" % PathFinder.getServiceSection(serviceName)
Beispiel #39
0
  def __init__( self):
    Client.__init__( self )
    self.setServer( "DataManagement/Test" )

    url = PathFinder.getServiceURL( "DataManagement/Test" )
    if not url:
      raise RuntimeError( "CS option DataManagement/Test URL is not set!" )
    self.testManager = RPCClient( url )
Beispiel #40
0
 def requestManager( self, timeout = 120 ):
   """ facade for RequestManager RPC client """
   if not self.__requestManager:
     url = PathFinder.getServiceURL( "RequestManagement/RequestManager" )
     if not url:
       raise RuntimeError( "CS option RequestManagement/RequestManager URL is not set!" )
     self.__requestManager = RPCClient( url, timeout = timeout )
   return self.__requestManager
Beispiel #41
0
 def requestManager( self, timeout = 120 ):
   """ facade for RequestManager RPC client """
   if not self.__requestManager:
     url = PathFinder.getServiceURL( "RequestManagement/ReqManager" )
     if not url:
       raise RuntimeError( "CS option RequestManagement/ReqManager URL is not set!" )
     self.__requestManager = RPCClient( url, timeout = timeout )
   return self.__requestManager
Beispiel #42
0
  def execute(self):
    """ This agent is the smallest and (cutest) of all the DIRAC agents in existence.
    """
    gMonitor.addMark("Iteration",1)

    central = PathFinder.getServiceURL("RequestManagement/centralURL")
    if central:
      self.central = central
    local = PathFinder.getServiceURL("RequestManagement/localURL")
    if local:
      self.local = local

    res = self.RequestDBClient.serveRequest(url=self.local)
    if not res['OK']:
      gLogger.error("ZuziaAgent.execute: Failed to get request from database.",self.local)
      return S_OK()
    elif not res['Value']:
      gLogger.info("ZuziaAgent.execute: No requests to be executed found.")
      return S_OK()

    gMonitor.addMark("Attempted",1)

    requestString = res['Value']['RequestString']
    requestName = res['Value']['RequestName']
    gLogger.info("ZuziaAgent.execute: Obtained request %s" % requestName)

    gLogger.info("ZuziaAgent.execute: Attempting to set request to %s." % self.central)
    res = self.RequestDBClient.setRequest(requestName,requestString,self.central)
    if res['OK']:
      gMonitor.addMark("Successful",1)
      gLogger.info("ZuziaAgent.execute: Successfully put request.")
    else:
      gMonitor.addMark("Failed",1)
      gLogger.error("ZuziaAgent.execute: Failed to set request to", self.central)
      gLogger.info("ZuziaAgent.execute: Attempting to set request to %s." % self.local)
      res = self.RequestDBClient.setRequest(requestName,requestString,self.local)
      if res['OK']:
        gLogger.info("ZuziaAgent.execute: Successfully put request.")
      else:
        gLogger.error("ZuziaAgent.execute: Failed to set request to", self.local)
        while not res['OK']:
          gLogger.info("ZuziaAgent.execute: Attempting to set request to anywhere.")
          res = self.RequestDBClient.setRequest(requestName,requestString)
        gLogger.info("ZuziaAgent.execute: Successfully put request to %s." % res["Server"])
    return S_OK()
Beispiel #43
0
  def __init__( self, storageName, parameters ):

    StorageBase.__init__( self, storageName, parameters )
    self.pluginName = 'Proxy'
    self.isok = True

    self.url = PathFinder.getServiceURL( "DataManagement/StorageElementProxy" )
    if not self.url:
      self.isok = False
Beispiel #44
0
    def export_sendMail(self,
                        address,
                        subject,
                        body,
                        fromAddress,
                        avoidSpam=False):
        """ Send an email with supplied body to the specified address using the Mail utility.

        :param basestring address: recipient addresses
        :param basestring subject: subject of letter
        :param basestring body: body of letter
        :param basestring fromAddress: sender address, if None, will be used default from CS
        :param bool avoidSpam: if True, then emails are first added to a set so that duplicates are removed,
               and sent every hour.

        :return: S_OK(basestring)/S_ERROR() -- basestring is status message
    """
        gLogger.verbose(
            'Received signal to send the following mail to %s:\nSubject = %s\n%s'
            % (address, subject, body))
        if gMailCache.exists(hash(address + subject + body)) and not avoidSpam:
            return S_OK(
                'Email with the same content already sent today to current addresses, come back tomorrow'
            )
        eMail = Mail()
        notificationSection = PathFinder.getServiceSection(
            "Framework/Notification")
        csSection = notificationSection + '/SMTP'
        eMail._smtpHost = gConfig.getValue('%s/Host' % csSection)
        eMail._smtpPort = gConfig.getValue('%s/Port' % csSection)
        eMail._smtpLogin = gConfig.getValue('%s/Login' % csSection)
        eMail._smtpPasswd = gConfig.getValue('%s/Password' % csSection)
        eMail._smtpPtcl = gConfig.getValue('%s/Protocol' % csSection)
        eMail._subject = subject
        eMail._message = body
        eMail._mailAddress = address
        if not fromAddress == 'None':
            eMail._fromAddress = fromAddress
        eMail._fromAddress = gConfig.getValue(
            '%s/FromAddress' % csSection) or eMail._fromAddress
        if avoidSpam and eMail in gMailSet:
            return S_OK("Mail already sent")
        else:
            result = eMail._send()
            if not result['OK']:
                gLogger.warn(
                    'Could not send mail with the following message:\n%s' %
                    result['Message'])
            else:
                gMailCache.add(hash(address + subject + body), 3600 * 24)
                gLogger.info('Mail sent successfully to %s with subject %s' %
                             (address, subject))
                gLogger.debug(result['Value'])
                if avoidSpam:
                    gMailSet.add(eMail)

        return result
Beispiel #45
0
 def initializeHandler( cls, svcInfoDict ):
   multiPath = PathFinder.getDatabaseSection( "Accounting/MultiDB" )
   cls.__acDB = MultiAccountingDB( multiPath )
   cls.__acDB.autoCompactDB()
   result = cls.__acDB.markAllPendingRecordsAsNotTaken()
   if not result[ 'OK' ]:
     return result
   gThreadScheduler.addPeriodicTask( 60, cls.__acDB.loadPendingRecords )
   return S_OK()
Beispiel #46
0
  def loadModules( self, modulesList, hideExceptions = False ):
    """
      Load all modules required in moduleList
    """
    for modName in modulesList:
      gLogger.verbose( "Checking %s" % modName )
      #if it's a executor modName name just load it and be done with it
      if modName.find( "/" ) > -1:
        gLogger.verbose( "Module %s seems to be a valid name. Try to load it!" % modName )
        result = self.loadModule( modName, hideExceptions = hideExceptions )
        if not result[ 'OK' ]:
          return result
        continue
      #Check if it's a system name
      #Look in the CS
      system = modName
      #Can this be generated with sectionFinder?
      csPath = "%s/Executors" % PathFinder.getSystemSection ( system, ( system, ) )
      gLogger.verbose( "Exploring %s to discover modules" % csPath )
      result = gConfig.getSections( csPath )
      if result[ 'OK' ]:
        #Add all modules in the CS :P
        for modName in result[ 'Value' ]:
          result = self.loadModule( "%s/%s" % ( system, modName ), hideExceptions = hideExceptions )
          if not result[ 'OK' ]:
            return result
      #Look what is installed
      parentModule = None
      for rootModule in getInstalledExtensions():
        if system.find( "System" ) != len( system ) - 6:
          parentImport = "%s.%sSystem.%s" % ( rootModule, system, self.__csSuffix )
        else:
          parentImport = "%s.%s.%s" % ( rootModule, system, self.__csSuffix )
        #HERE!
        result = self.__recurseImport( parentImport )
        if not result[ 'OK' ]:
          return result
        parentModule = result[ 'Value' ]
        if parentModule:
          break
      if not parentModule:
        continue
      parentPath = parentModule.__path__[0]
      gLogger.notice( "Found modules path at %s" % parentImport )
      for entry in os.listdir( parentPath ):
        if entry[-3:] != ".py" or entry == "__init__.py":
          continue
        if not os.path.isfile( os.path.join( parentPath, entry ) ):
          continue
        modName = "%s/%s" % ( system, entry[:-3] )
        gLogger.verbose( "Trying to import %s" % modName )
        result = self.loadModule( modName,
                                  hideExceptions = hideExceptions,
                                  parentModule = parentModule )

    return S_OK()
Beispiel #47
0
  def __init__( self, useCertificates = False ):
    """ Constructor of the RequestClient class
    """
    voBoxUrls = fromChar( PathFinder.getServiceURL( "RequestManagement/voBoxURLs" ) )
    self.voBoxUrls = []
    if voBoxUrls:
      self.voBoxUrls = randomize( voBoxUrls )

    local = PathFinder.getServiceURL( "RequestManagement/localURL" )
    self.local = False
    if local:
      self.local = local

    central = PathFinder.getServiceURL( "RequestManagement/centralURL" )
    self.central = False
    if central:
      self.central = central

    self.setServer( 'RequestManagement/centralURL' )
Beispiel #48
0
 def requestProxies( self, timeout = 120 ):
   """ get request proxies dict """
   if not self.__requestProxiesDict:
     self.__requestProxiesDict = {}
     proxiesURLs = fromChar( PathFinder.getServiceURL( "RequestManagement/ReqProxyURLs" ) )
     if not proxiesURLs:
       self.log.warn( "CS option RequestManagement/ReqProxyURLs is not set!" )
     for proxyURL in proxiesURLs:
       self.log.debug( "creating RequestProxy for url = %s" % proxyURL )
       self.__requestProxiesDict[proxyURL] = RPCClient( proxyURL, timeout = timeout )
   return self.__requestProxiesDict
Beispiel #49
0
 def __getRequestDBPath( self ):
   """ Obtain the root of the requestDB from the configuration
   """
   csSection = csSection = PathFinder.getServiceSection( "RequestManagement/RequestManager" )
   root = gConfig.getValue( '%s/Path' % csSection )
   if not root:
     diracRoot = gConfig.getValue( '/LocalSite/InstancePath', rootPath )
     root = diracRoot + '/requestdb'
   if not os.path.exists( root ):
     os.makedirs( root )
   return root
 def __init__(self,url=False,useCertificates=False):
   """ Constructor of the DataLogging client
   """
   try:
     if not url:
       self.url = PathFinder.getServiceURL('DataManagement/DataLogging')
     else:
       self.url = url
   except Exception, x:
     errStr = "DataLoggingClient.__init__: Exception while obtaining service URL."
     gLogger.exception(errStr,lException=x)
Beispiel #51
0
 def __init__( self, csSection = False ):
   threading.Thread.__init__( self )
   if csSection:
     self.csSection = csSection
   else:
     self.csSection = PathFinder.getServiceSection( "Framework/SiteMap" )
   self.refreshPeriod = self._getCSValue( "RefreshPeriod", 300 )
   self.gridsToMap = []
   self.history = []
   self.sitesData = {}
   self.refresh()
   self.setDaemon( 1 )
Beispiel #52
0
 def initialize( self ):
   self.logger = gLogger.getSubLogger( "Monitoring" )
   self.logger.debug( "Initializing Monitoring Client" )
   self.sourceDict[ 'setup' ] = gConfig.getValue( "/DIRAC/Setup" )
   self.sourceDict[ 'site' ] = DIRAC.siteName()
   if self.sourceDict[ 'componentType' ] == self.COMPONENT_SERVICE:
     self.cfgSection = PathFinder.getSystemSection( self.sourceDict[ 'componentName' ] )
   elif self.sourceDict[ 'componentType' ] == self.COMPONENT_AGENT:
     self.cfgSection = PathFinder.getAgentSection( self.sourceDict[ 'componentName' ] )
     self.setComponentLocation( Network.getFQDN() )
   elif self.sourceDict[ 'componentType' ] == self.COMPONENT_WEB:
     self.cfgSection = "/WebApp"
     self.setComponentLocation( 'http://%s' % Network.getFQDN() )
     self.setComponentName( 'WebApp' )
   elif self.sourceDict[ 'componentType' ] == self.COMPONENT_SCRIPT:
     self.cfgSection = "/Script"
   else:
     raise Exception( "Component type has not been defined" )
   gMonitoringFlusher.registerMonitoringClient( self )
   # ExitCallback.registerExitCallback( self.forceFlush )
   self.__initialized = True
Beispiel #53
0
 def initializeHandler( cls, svcInfoDict ):
   multiPath = PathFinder.getDatabaseSection( "Accounting/MultiDB" )
   cls.__acDB = MultiAccountingDB( multiPath )
   #we can run multiple services in read only mode. In that case we do not bucket
   cls.runBucketing = getServiceOption( svcInfoDict, 'RunBucketing', True )
   if cls.runBucketing:
     cls.__acDB.autoCompactDB() #pylint: disable=no-member
     result = cls.__acDB.markAllPendingRecordsAsNotTaken() #pylint: disable=no-member
     if not result[ 'OK' ]:
       return result
     gThreadScheduler.addPeriodicTask( 60, cls.__acDB.loadPendingRecords ) #pylint: disable=no-member
   return S_OK()
Beispiel #54
0
    def __getRequestDBPath():
        """ get the fs root path of the requestDB from the DIRAC configuration

    :warn: if root path doesn't exist, it will be created 
    """
        csSection = csSection = PathFinder.getServiceSection("RequestManagement/RequestManager")
        root = gConfig.getValue("%s/Path" % csSection, "requestdb")
        diracRoot = gConfig.getValue("/LocalSite/InstancePath", rootPath)
        # if the path return by the gConfig is absolute the following line does not change it,
        # otherwise it makes it relative to diracRoot
        root = os.path.join(diracRoot, root)
        if not os.path.exists(root):
            os.makedirs(root)
        return root
 def __init__( self, url = False ):
   """ Constructor of the Bookkeeping catalogue client
   """
   self.splitSize = 1000
   self.name = 'BookkeepingDB'
   self.valid = True
   try:
     if not url:
       self.url = PathFinder.getServiceURL( 'Bookkeeping/NewBookkeepingManager' )
     else:
       self.url = url
   except Exception, exceptionMessage:
     gLogger.exception( 'BookkeepingDBClient.__init__: Exception while obtaining Bookkeeping service URL.', '', exceptionMessage )
     self.valid = False
Beispiel #56
0
  def initialize(self):

    self.RequestDBClient = RequestClient()

    gMonitor.registerActivity("Iteration",          "Agent Loops",                  "ZuziaAgent",      "Loops/min",      gMonitor.OP_SUM)
    gMonitor.registerActivity("Attempted",          "Request Processed",            "ZuziaRAgent",     "Requests/min",   gMonitor.OP_SUM)
    gMonitor.registerActivity("Successful",         "Request Forward Successful",   "ZuziaAgent",      "Requests/min",   gMonitor.OP_SUM)
    gMonitor.registerActivity("Failed",             "Request Forward Failed",       "ZuziaAgent",      "Requests/min",   gMonitor.OP_SUM)

    self.local = PathFinder.getServiceURL("RequestManagement/localURL")
    if not self.local:
      self.local = AgentModule.am_getOption(self,'localURL','')
    if not self.local:
      errStr = 'The RequestManagement/localURL option must be defined.'
      gLogger.fatal(errStr)
      return S_ERROR(errStr)

    self.central = PathFinder.getServiceURL("RequestManagement/centralURL")
    if not self.central:
      errStr = 'The RequestManagement/centralURL option must be defined.'
      gLogger.fatal(errStr)
      return S_ERROR(errStr)
    return S_OK()
  def initialize( self ):

    self.section = PathFinder.getAgentSection( AGENT_NAME )
    self.RequestDB = RequestDBMySQL()
    self.TransferDB = TransferDB()
    self.DataLog = DataLoggingClient()
    self.factory = StorageFactory()
    self.rm = ReplicaManager()

    # This sets the Default Proxy to used as that defined under
    # /Operations/Shifter/DataManager
    # the shifterProxy option in the Configuration can be used to change this default.
    self.am_setOption( 'shifterProxy', 'DataManager' )

    return S_OK()